diff --git a/NOTICE-fips.txt b/NOTICE-fips.txt index ff126da07ca..229b356a30d 100644 --- a/NOTICE-fips.txt +++ b/NOTICE-fips.txt @@ -1254,11 +1254,11 @@ SOFTWARE -------------------------------------------------------------------------------- Dependency : github.com/elastic/elastic-agent-libs -Version: v0.26.2 +Version: v0.26.3-0.20251203201625-76691b894177 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-libs@v0.26.2/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-libs@v0.26.3-0.20251203201625-76691b894177/LICENSE: Apache License Version 2.0, January 2004 diff --git a/NOTICE.txt b/NOTICE.txt index fd506ba80d8..e1ff16a6ca5 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -1254,11 +1254,11 @@ SOFTWARE -------------------------------------------------------------------------------- Dependency : github.com/elastic/elastic-agent-libs -Version: v0.26.2 +Version: v0.26.3-0.20251203201625-76691b894177 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-libs@v0.26.2/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-libs@v0.26.3-0.20251203201625-76691b894177/LICENSE: Apache License Version 2.0, January 2004 diff --git a/go.mod b/go.mod index 79a1551fe45..8b48554a43b 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/elastic/cloud-on-k8s/v2 v2.0.0-20250327073047-b624240832ae github.com/elastic/elastic-agent-autodiscover v0.10.0 github.com/elastic/elastic-agent-client/v7 v7.17.2 - github.com/elastic/elastic-agent-libs v0.26.2 + github.com/elastic/elastic-agent-libs v0.26.3-0.20251203201625-76691b894177 github.com/elastic/elastic-agent-system-metrics v0.13.4 github.com/elastic/elastic-agent/internal/edot v0.0.0-20251114132921-c463803c5568 github.com/elastic/elastic-transport-go/v8 v8.8.0 diff --git a/go.sum b/go.sum index cccd6a0fa67..170e8c72846 100644 --- a/go.sum +++ b/go.sum @@ -511,8 +511,8 @@ github.com/elastic/elastic-agent-autodiscover v0.10.0 h1:WJ4zl9uSfk1kHmn2B/0byQB github.com/elastic/elastic-agent-autodiscover v0.10.0/go.mod h1:Nf3zh9FcJ9nTTswTwDTUAqXmvQllOrNliM6xmORSxwE= github.com/elastic/elastic-agent-client/v7 v7.17.2 h1:Cl2TeABqWZgW40t5fchGWT/sRk4MDDLWA0d8iHHOxLA= github.com/elastic/elastic-agent-client/v7 v7.17.2/go.mod h1:5irRFqp6HLqtu1S+OeY0jg8x7K6PLL+DW+PwVk1vJnk= -github.com/elastic/elastic-agent-libs v0.26.2 h1:zwytPWmTWSJG80oa9/5FJ6zue47ysI23eMo15LfeWy0= -github.com/elastic/elastic-agent-libs v0.26.2/go.mod h1:fc2noLqosmQorIGbatJfVeh4CL77yiP8ot16/5umeoM= +github.com/elastic/elastic-agent-libs v0.26.3-0.20251203201625-76691b894177 h1:ME5EtQjQXzd28QOCSBSOiH0jbMBmaHLZ4NVRNBmw6MU= +github.com/elastic/elastic-agent-libs v0.26.3-0.20251203201625-76691b894177/go.mod h1:fc2noLqosmQorIGbatJfVeh4CL77yiP8ot16/5umeoM= github.com/elastic/elastic-agent-system-metrics v0.13.4 h1:gX8VdlQyakPcPKFpD7uHv2QLRDyguuKfZgu0LE27V7c= github.com/elastic/elastic-agent-system-metrics v0.13.4/go.mod h1:lB8veYWYBlA9eF6TahmPN87G1IEgWlbep7QSqLSW90U= github.com/elastic/elastic-transport-go/v8 v8.8.0 h1:7k1Ua+qluFr6p1jfJjGDl97ssJS/P7cHNInzfxgBQAo= diff --git a/internal/edot/go.mod b/internal/edot/go.mod index 27c30a9c9be..c524af419aa 100644 --- a/internal/edot/go.mod +++ b/internal/edot/go.mod @@ -7,7 +7,7 @@ replace github.com/elastic/elastic-agent => ../../ require ( github.com/elastic/beats/v7 v7.0.0-alpha2.0.20251130155143-19eb3e615086 github.com/elastic/elastic-agent v0.0.0-00010101000000-000000000000 - github.com/elastic/elastic-agent-libs v0.26.2 + github.com/elastic/elastic-agent-libs v0.26.3-0.20251203201625-76691b894177 github.com/elastic/opentelemetry-collector-components/connector/elasticapmconnector v0.20.0 github.com/elastic/opentelemetry-collector-components/connector/profilingmetricsconnector v0.20.0 github.com/elastic/opentelemetry-collector-components/extension/apikeyauthextension v0.22.0 diff --git a/internal/edot/go.sum b/internal/edot/go.sum index 03cfa656ee6..6baf6409a83 100644 --- a/internal/edot/go.sum +++ b/internal/edot/go.sum @@ -438,8 +438,8 @@ github.com/elastic/elastic-agent-autodiscover v0.10.0 h1:WJ4zl9uSfk1kHmn2B/0byQB github.com/elastic/elastic-agent-autodiscover v0.10.0/go.mod h1:Nf3zh9FcJ9nTTswTwDTUAqXmvQllOrNliM6xmORSxwE= github.com/elastic/elastic-agent-client/v7 v7.17.2 h1:Cl2TeABqWZgW40t5fchGWT/sRk4MDDLWA0d8iHHOxLA= github.com/elastic/elastic-agent-client/v7 v7.17.2/go.mod h1:5irRFqp6HLqtu1S+OeY0jg8x7K6PLL+DW+PwVk1vJnk= -github.com/elastic/elastic-agent-libs v0.26.2 h1:zwytPWmTWSJG80oa9/5FJ6zue47ysI23eMo15LfeWy0= -github.com/elastic/elastic-agent-libs v0.26.2/go.mod h1:fc2noLqosmQorIGbatJfVeh4CL77yiP8ot16/5umeoM= +github.com/elastic/elastic-agent-libs v0.26.3-0.20251203201625-76691b894177 h1:ME5EtQjQXzd28QOCSBSOiH0jbMBmaHLZ4NVRNBmw6MU= +github.com/elastic/elastic-agent-libs v0.26.3-0.20251203201625-76691b894177/go.mod h1:fc2noLqosmQorIGbatJfVeh4CL77yiP8ot16/5umeoM= github.com/elastic/elastic-agent-system-metrics v0.13.4 h1:gX8VdlQyakPcPKFpD7uHv2QLRDyguuKfZgu0LE27V7c= github.com/elastic/elastic-agent-system-metrics v0.13.4/go.mod h1:lB8veYWYBlA9eF6TahmPN87G1IEgWlbep7QSqLSW90U= github.com/elastic/elastic-transport-go/v8 v8.8.0 h1:7k1Ua+qluFr6p1jfJjGDl97ssJS/P7cHNInzfxgBQAo= diff --git a/testing/integration/ess/proxy_url_test.go b/testing/integration/ess/proxy_url_test.go index 20090d44a58..b443e04c168 100644 --- a/testing/integration/ess/proxy_url_test.go +++ b/testing/integration/ess/proxy_url_test.go @@ -10,7 +10,9 @@ import ( "context" "crypto/tls" "crypto/x509" + "errors" "net" + "net/http" "net/url" "os" "path/filepath" @@ -24,14 +26,19 @@ import ( "github.com/stretchr/testify/require" "gopkg.in/yaml.v2" + "github.com/elastic/elastic-agent-libs/kibana" "github.com/elastic/elastic-agent-libs/testing/certutil" + atesting "github.com/elastic/elastic-agent/pkg/testing" integrationtest "github.com/elastic/elastic-agent/pkg/testing" "github.com/elastic/elastic-agent/pkg/testing/define" "github.com/elastic/elastic-agent/pkg/testing/tools/check" + "github.com/elastic/elastic-agent/pkg/testing/tools/fleettools" "github.com/elastic/elastic-agent/pkg/testing/tools/testcontext" + "github.com/elastic/elastic-agent/pkg/version" "github.com/elastic/elastic-agent/testing/fleetservertest" "github.com/elastic/elastic-agent/testing/integration" "github.com/elastic/elastic-agent/testing/proxytest" + "github.com/elastic/elastic-agent/testing/upgradetest" ) func TestProxyURL(t *testing.T) { @@ -792,3 +799,233 @@ func createBasicFleetPolicyData(t *testing.T, fleetHost string) (fleetservertest } return apiKey, policyData } + +// TestFleetDownloadProxyURL will test that the download proxy is used correctly for an agent upgrade. +// +// Test will target a download source that requires a proxy to be used. +// +// elastic-agent -> proxy -> artifacts-proxy -> upstream +// +// If a proxy is not used, the artifacts-proxy returns an status error code. +// This test will do the following: +// 1. Create special artifacts-proxy that requires a proxy header in order to serve artifacts +// 2. Create a policy with no proxies - that has the artifacts-proxy as the download url +// 3. Enroll an agent +// 4. Upgrade the agent +// 5. Ensure upgrade fails +// 6. Update policy to use a download proxy +// 7. Upgrade the agent +// 8. Ensure upgrade succeeds +func TestFleetDownloadProxyURL(t *testing.T) { + ctx := t.Context() + info := define.Require(t, define.Requirements{ + Group: integration.Fleet, + Stack: &define.Stack{}, + Local: false, + Sudo: true, + }) + kibClient := info.KibanaClient + fleetServerURL, err := fleettools.DefaultURL(ctx, kibClient) + require.NoError(t, err) + testUUID, err := uuid.NewV4() + require.NoError(t, err, "error generating UUID for test") + + artifactsProxy := proxytest.New(t, + proxytest.WithVerifyRequest(func(r *http.Request) error { // ensure we have proxy header + h := r.Header.Get("Forwarded") + if h == "" { + h = r.Header.Get("X-Forwarded-For") + if h == "" { + return errors.New("missing proxy header") + } + } + return nil + }), + proxytest.WithRewriteFn(func(u *url.URL) { // Send requests to real upstream source + u.Scheme = "https" + u.Host = "snapshots.elastic.co" + }), + proxytest.WithRequestLog("artifacts", t.Logf), + proxytest.WithVerboseLog()) + err = artifactsProxy.Start() + require.NoError(t, err, "Error starting artifacts proxy") + t.Cleanup(artifactsProxy.Close) + + downloadSource := kibana.DownloadSource{ + Name: "LocalArtifactsProxy-" + testUUID.String(), + Host: artifactsProxy.LocalhostURL + "/downloads/", + } + sourceResp, err := kibClient.CreateDownloadSource(ctx, downloadSource) + require.NoError(t, err, "Unable to create download source") + + // Get and process start and end fixtures + // TODO: Do we need FIPS aware tests? + startFixture, err := define.NewFixtureFromLocalBuild(t, define.Version()) + require.NoError(t, err) + err = startFixture.Prepare(ctx) + require.NoError(t, err) + startVersionInfo, err := startFixture.ExecVersion(ctx) + require.NoError(t, err) + startParsedVersion, err := version.ParseVersion(startVersionInfo.Binary.String()) + require.NoError(t, err) + + endFixture, err := atesting.NewFixture( + t, + upgradetest.EnsureSnapshot(define.Version()), + atesting.WithFetcher(atesting.ArtifactFetcher()), + ) + require.NoError(t, err) + err = endFixture.Prepare(ctx) + require.NoError(t, err) + endVersionInfo, err := endFixture.ExecVersion(ctx) + require.NoError(t, err) + + if startVersionInfo.Binary.String() == endVersionInfo.Binary.String() && + startVersionInfo.Binary.Commit == endVersionInfo.Binary.Commit { + t.Skipf("Build under test is the same as the build from the artifacts repository (version: %s) [commit: %s]", + startVersionInfo.Binary.String(), startVersionInfo.Binary.Commit) + } + if startVersionInfo.Binary.Commit == endVersionInfo.Binary.Commit { + t.Skipf("Target version has the same commit hash %q", endVersionInfo.Binary.Commit) + } + + t.Log("Creating Agent policy...") + policy := kibana.AgentPolicy{ + Name: "test-policy-" + testUUID.String(), + Namespace: "default", + Description: "Test policy " + testUUID.String(), + MonitoringEnabled: []kibana.MonitoringEnabledOption{ + kibana.MonitoringEnabledLogs, + kibana.MonitoringEnabledMetrics, + }, + DownloadSourceID: sourceResp.Item.ID, // Use artifacts-proxy as download source + AdvancedSettings: map[string]interface{}{ + "agent_download_timeout": "1m", // Force a fast initial failure timeout + }, + } + policyResp, err := kibClient.CreatePolicy(ctx, policy) + require.NoError(t, err) + enrollmentToken, err := kibClient.CreateEnrollmentAPIKey(ctx, kibana.CreateEnrollmentAPIKeyRequest{ + PolicyID: policyResp.ID, + }) + require.NoError(t, err) + + t.Log("Installing Elastic Agent...") + installOpts := atesting.InstallOpts{ + Force: true, + EnrollOpts: atesting.EnrollOpts{ + URL: fleetServerURL, + EnrollmentToken: enrollmentToken.APIKey, + }, + } + output, err := startFixture.Install(ctx, &installOpts) + t.Logf("Install agent output:\n%s", string(output)) + require.NoError(t, err) + + // TODO fast watcher config? + + t.Log("Waiting for Agent to be correct version and healthy...") + err = upgradetest.WaitHealthyAndVersion(ctx, startFixture, startVersionInfo.Binary, 2*time.Minute, 10*time.Second, t) + require.NoError(t, err) + + agentID, err := startFixture.AgentID(ctx) + require.NoError(t, err) + t.Logf("Agent ID: %q", agentID) + + t.Log("Waiting for enrolled Agent status to be online...") + require.Eventually(t, func() bool { + return check.FleetAgentStatus(ctx, t, kibClient, agentID, "online")() + }, time.Minute*2, time.Second, "Agent did not come online") + + t.Logf("Upgrading from version \"%s-%s\" to version \"%s-%s\", without proxy...", + startParsedVersion, startVersionInfo.Binary.Commit, + endVersionInfo.Binary.String(), endVersionInfo.Binary.Commit) + err = fleettools.UpgradeAgent(ctx, kibClient, agentID, endVersionInfo.Binary.String(), true) + require.NoError(t, err) + + t.Log("Ensure upgrade has failed") + require.EventuallyWithT(t, func(c *assert.CollectT) { + agent, err := kibClient.GetAgent(ctx, kibana.GetAgentRequest{ID: agentID}) + require.NoError(c, err) + require.NotNil(c, agent.UpgradeDetails) + require.Equal(c, "UPG_FAILED", agent.UpgradeDetails.State) + }, time.Minute*5, time.Second, "Unable to verify that upgrade has failed.") + + proxy := proxytest.New(t, + proxytest.WithRequestLog("proxy", t.Logf), + proxytest.WithVerboseLog()) + err = proxy.Start() + require.NoError(t, err, "error starting download proxy") + t.Cleanup(proxy.Close) + + fleetProxyResp, err := kibClient.CreateFleetProxy(ctx, kibana.ProxiesRequest{ + Name: "fleet-upgrade-test-proxy-" + testUUID.String(), + URL: proxy.LocalhostURL, + }) + require.NoError(t, err) + + // Update download source to include download proxy + downloadSource.ProxyID = fleetProxyResp.Item.ID + _, err = kibClient.UpdateDownloadSource(ctx, sourceResp.Item.ID, downloadSource) + require.NoError(t, err, "Unable to update policy with download source proxy") + // Update policy to increase download timeout to 60m (default 120m) + updatedPolicy, err := kibClient.UpdatePolicy(ctx, policyResp.ID, kibana.AgentPolicyUpdateRequest{ + Name: policy.Name, + Namespace: policy.Namespace, + AdvancedSettings: map[string]interface{}{ + "agent_download_timeout": "60m", + }, + }) + require.NoError(t, err, "Unable to update policy with new download timeout") + + t.Logf("Verify agent is online and has updated to revision %d", updatedPolicy.Revision) + require.EventuallyWithT(t, func(c *assert.CollectT) { + agentResp, err := kibClient.GetAgent(ctx, kibana.GetAgentRequest{ID: agentID}) + require.NoError(c, err) + require.Equal(c, "online", agentResp.Status) + require.Equal(c, updatedPolicy.Revision, agentResp.PolicyRevision) + }, time.Minute, time.Second, "Expected agent to be online and policy has updated") + + t.Logf("Upgrading from version \"%s-%s\" to version \"%s-%s\", with proxy...", + startParsedVersion, startVersionInfo.Binary.Commit, + endVersionInfo.Binary.String(), endVersionInfo.Binary.Commit) + err = fleettools.UpgradeAgent(ctx, kibClient, agentID, endVersionInfo.Binary.String(), true) + require.NoError(t, err) + + t.Log("Ensure upgrade starts") + require.EventuallyWithT(t, func(c *assert.CollectT) { + agent, err := kibClient.GetAgent(ctx, kibana.GetAgentRequest{ID: agentID}) + require.NoError(c, err) + require.NotNil(c, agent.UpgradeDetails) + }, time.Minute*5, time.Second, "Unable to verify that upgrade details appear.") + + t.Log("Waiting for upgrade watcher to start...") + err = upgradetest.WaitForWatcher(ctx, 5*time.Minute, 10*time.Second) + require.NoError(t, err) + t.Log("Upgrade watcher started") + + err = upgradetest.WaitHealthyAndVersion(ctx, startFixture, endVersionInfo.Binary, 2*time.Minute, 10*time.Second, t) + require.NoError(t, err) + + t.Log("Waiting for upgraded Agent status to be online...") + require.Eventually(t, func() bool { + return check.FleetAgentStatus(ctx, t, kibClient, agentID, "online")() + }, time.Minute*10, time.Second*10, "Agent did not come online") + + t.Log("Check agent version") + require.EventuallyWithT(t, func(c *assert.CollectT) { + ver, err := fleettools.GetAgentVersion(ctx, kibClient, agentID) + require.NoError(c, err) + require.Equal(c, endVersionInfo.Binary.Version, ver) + }, time.Minute*5, time.Second) + + t.Log("Waiting for upgrade watcher to finish...") + err = upgradetest.WaitForNoWatcher(ctx, 2*time.Minute, 10*time.Second, 1*time.Minute+15*time.Second) + require.NoError(t, err) + + err = upgradetest.CheckHealthyAndVersion(ctx, startFixture, endVersionInfo.Binary) + require.NoError(t, err, "Post watcher check has failed, agent may have rolled back") + + require.NotEmpty(t, artifactsProxy.ProxiedRequests(), "artifactsProxy does not have any requests") + require.NotEmpty(t, proxy.ProxiedRequests(), "proxy does not have any requests") +} diff --git a/testing/proxytest/proxytest.go b/testing/proxytest/proxytest.go index 54d748e4be9..ccd8336fc70 100644 --- a/testing/proxytest/proxytest.go +++ b/testing/proxytest/proxytest.go @@ -50,9 +50,10 @@ type Proxy struct { type Option func(o *options) type options struct { - addr string - rewriteHost func(string) string - rewriteURL func(u *url.URL) + addr string + rewriteHost func(string) string + rewriteURL func(u *url.URL) + verifyRequest func(r *http.Request) error // logFn if set will be used to log every request. logFn func(format string, a ...any) verbose bool @@ -121,6 +122,14 @@ func WithRewriteFn(f func(u *url.URL)) Option { } } +// WithVerifyRequest calls f on the request before the proxy request is made. +// If the verification returns an error, the proxy reutrns an HTTP 500 (for http proxy), or HTTP 502 (https) +func WithVerifyRequest(f func(r *http.Request) error) Option { + return func(o *options) { + o.verifyRequest = f + } +} + // WithServerTLSConfig sets the TLS config for the server. func WithServerTLSConfig(tc *tls.Config) Option { return func(o *options) { @@ -288,6 +297,12 @@ func (p *Proxy) serveHTTP(w http.ResponseWriter, r *http.Request) { func (p *Proxy) processRequest(r *http.Request) (*http.Response, error) { origURL := r.URL.String() + if p.opts.verifyRequest != nil { + if err := p.opts.verifyRequest(r); err != nil { + return nil, err + } + } + switch { case p.opts.rewriteURL != nil: p.opts.rewriteURL(r.URL) @@ -312,6 +327,14 @@ func (p *Proxy) processRequest(r *http.Request) (*http.Response, error) { // needed anyway, so remove it. r.RequestURI = "" + // Add a Forwarded header + host, _, err := net.SplitHostPort(r.RemoteAddr) + if err != nil { + p.opts.logFn("[ERROR] could not parse remote address %q: %v", r.RemoteAddr, err) + } else { + r.Header.Add("Forwarded", "for="+host) + } + return p.client.Do(r) } diff --git a/testing/proxytest/proxytest_test.go b/testing/proxytest/proxytest_test.go index f367ced2b62..d4bebd017f0 100644 --- a/testing/proxytest/proxytest_test.go +++ b/testing/proxytest/proxytest_test.go @@ -8,6 +8,7 @@ import ( "context" "crypto/tls" "crypto/x509" + "errors" "io" "net" "net/http" @@ -171,6 +172,56 @@ func TestProxy(t *testing.T) { } }, }, + { + name: "Basic scenario, request verifier passes", + setup: setup{ + fakeBackendServer: createFakeBackendServer(), + generateTestHttpClient: nil, + }, + proxyOptions: []Option{WithVerifyRequest(func(r *http.Request) error { + if r.Method == http.MethodGet { + return nil + } + return errors.New("unsupported method") + })}, + proxyStartTLS: false, + request: testRequest{ + method: http.MethodGet, + url: "http://somehost:1234/some/path/here", + body: nil, + }, + wantErr: assert.NoError, + assertFunc: func(t *testing.T, proxy *Proxy, resp *http.Response) { + assert.Equal(t, http.StatusOK, resp.StatusCode) + if assert.NotEmpty(t, proxy.ProxiedRequests(), "proxy should have captured at least 1 request") { + assert.Contains(t, proxy.ProxiedRequests()[0], "/some/path/here") + } + }, + }, + { + name: "Basic scenario, request verifier fails", + setup: setup{ + fakeBackendServer: createFakeBackendServer(), + generateTestHttpClient: nil, + }, + proxyOptions: []Option{WithVerifyRequest(func(r *http.Request) error { + if r.Method == http.MethodPost { + return nil + } + return errors.New("unsupported method") + })}, + proxyStartTLS: false, + request: testRequest{ + method: http.MethodGet, + url: "http://somehost:1234/some/path/here", + body: nil, + }, + wantErr: assert.NoError, + assertFunc: func(t *testing.T, proxy *Proxy, resp *http.Response) { + assert.Equal(t, http.StatusInternalServerError, resp.StatusCode) + assert.Empty(t, proxy.ProxiedRequests(), "proxy does not capture unverified request") + }, + }, } for _, tt := range testcases { @@ -379,7 +430,12 @@ func prepareMTLSProxyAndTargetServer(t *testing.T, targetHost string) (*Proxy, h func createFakeBackendServer() *httptest.Server { handlerF := func(writer http.ResponseWriter, request *http.Request) { - // always return HTTP 200 + if h := request.Header.Get("Forwarded"); h == "" { + // return 400 if the proxy has not added the Forwarded header. + writer.WriteHeader(http.StatusBadRequest) + return + } + // return HTTP 200 otherwise writer.WriteHeader(http.StatusOK) }