diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index f3f0dd9d28..3aa726297c 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -24,8 +24,8 @@ pipeline { RUN_ID = UUID.randomUUID().toString() } options { - timeout(time: 1, unit: 'HOURS') - buildDiscarder(logRotator(numToKeepStr: '20', artifactNumToKeepStr: '20', daysToKeepStr: '30')) + timeout(time: 90, unit: 'MINUTES') + buildDiscarder(logRotator(numToKeepStr: '200', artifactNumToKeepStr: '30', daysToKeepStr: '30')) timestamps() ansiColor('xterm') disableResume() @@ -48,6 +48,7 @@ pipeline { booleanParam(name: "notifyOnGreenBuilds", defaultValue: false, description: "If it's needed to notify to Slack with green builds.") string(name: 'SLACK_CHANNEL', defaultValue: 'observablt-bots', description: 'The Slack channel(s) where errors will be posted. For multiple channels, use a comma-separated list of channels') string(name: 'ELASTIC_AGENT_DOWNLOAD_URL', defaultValue: '', description: 'If present, it will override the download URL for the Elastic agent artifact. (I.e. https://snapshots.elastic.co/7.12.0-069dfaa4/downloads/beats/elastic-agent/elastic-agent-7.12.0-SNAPSHOT-linux-x86_64.tar.gz') + string(name: 'ELASTIC_AGENT_VERSION', defaultValue: '7.17.1-952e96f1-SNAPSHOT', description: 'SemVer version of the Elastic Agent to be used for the tests. You can use here the tag of your PR to test your changes') string(name: 'BEAT_VERSION', defaultValue: '7.17.1-952e96f1-SNAPSHOT', description: 'SemVer version of the Beat to be used for the tests. You can use here the tag of your PR to test your changes') string(name: 'ELASTIC_AGENT_STALE_VERSION', defaultValue: '7.16-SNAPSHOT', description: 'SemVer version of the stale stand-alone elastic-agent to be used for Fleet upgrade tests.') choice(name: 'LOG_LEVEL', choices: ['DEBUG', 'TRACE', 'INFO'], description: 'Log level to be used') @@ -59,7 +60,7 @@ pipeline { string(name: 'KIND_VERSION', defaultValue: '0.10.0', description: 'SemVer version of Kind to be used.') string(name: 'KUBERNETES_VERSION', defaultValue: '1.18.2', description: 'SemVer version of Kubernetes to be used.') string(name: 'GITHUB_CHECK_NAME', defaultValue: '', description: 'Name of the GitHub check to be updated. Only if this build is triggered from another parent stream.') - string(name: 'GITHUB_CHECK_REPO', defaultValue: '', description: 'Name of the GitHub repo to be updated. Only if this build is triggered from another parent stream.') + string(name: 'GITHUB_CHECK_REPO', defaultValue: 'elastic-agent', description: 'Name of the GitHub repo to be updated. Only if this build is triggered from another parent stream.') string(name: 'GITHUB_CHECK_SHA1', defaultValue: '', description: 'Git SHA for the Beats upstream project (branch or PR)') } stages { @@ -74,6 +75,7 @@ pipeline { SLACK_CHANNEL = "${params.SLACK_CHANNEL.trim()}" ELASTIC_AGENT_DOWNLOAD_URL = "${params.ELASTIC_AGENT_DOWNLOAD_URL.trim()}" BEAT_VERSION = "${params.BEAT_VERSION.trim()}" + ELASTIC_AGENT_VERSION = "${params.ELASTIC_AGENT_VERSION.trim()}" KIBANA_VERSION = "${params.KIBANA_VERSION.trim()}" STACK_VERSION = "${params.STACK_VERSION.trim()}" FORCE_SKIP_GIT_CHECKS = "${params.forceSkipGitChecks}" @@ -84,6 +86,8 @@ pipeline { KUBERNETES_VERSION = "${params.KUBERNETES_VERSION.trim()}" LOG_LEVEL = "${params.LOG_LEVEL.trim()}" TIMEOUT_FACTOR = "${params.TIMEOUT_FACTOR.trim()}" + GITHUB_CHECK_REPO = "${params.GITHUB_CHECK_REPO.trim()}" + GITHUB_CHECK_SHA1 = "${params.GITHUB_CHECK_SHA1.trim()}" } stages { stage('Checkout') { @@ -507,7 +511,7 @@ def generateFunctionalTestStep(Map args = [:]){ def envContext = [] // envContext.add("PROVIDER=${platformProvider}") envContext.add("SUITE=${suite}") - envContext.add("ELASTIC_APM_GLOBAL_LABELS=branch_name=${BRANCH_NAME},build_pr=${isPR()},build_id=${env.BUILD_ID},go_arch=${goArch},beat_version=${env.BEAT_VERSION},stack_version=${env.STACK_VERSION}") + envContext.add("ELASTIC_APM_GLOBAL_LABELS=branch_name=${BRANCH_NAME},build_pr=${isPR()},build_id=${env.BUILD_ID},go_arch=${goArch},beat_version=${env.BEAT_VERSION},elastic_agent_version=${env.ELASTIC_AGENT_VERSION},stack_version=${env.STACK_VERSION}") def stackRunner = getNodeIp("${env.WORKSPACE}/${env.BASE_DIR}", 'stack') @@ -520,9 +524,11 @@ def generateFunctionalTestStep(Map args = [:]){ unstash 'sourceEnvModified' withEnv(envContext) { // This step will help to send the APM traces to the - // APM service defined by the Otel Jenkins plugin. + // withOtelEnv is the one that uses the APM service defined by the Otel Jenkins plugin. + // withAPMEnv uses Vault to prepare the context. + // IMPORTANT: withAPMEnv is now the one in used since withOtelEnv uses a specific Opentelemetry Collector at the moment. // TODO: This will need to be integrated into the provisioned VMs - withOtelEnv() { + withAPMEnv() { // Start node, capture ip address ansible("${env.WORKSPACE}", runId, diff --git a/.ci/ansible/tasks/setup_test_script.yml b/.ci/ansible/tasks/setup_test_script.yml index 8814352056..1fe57267b0 100644 --- a/.ci/ansible/tasks/setup_test_script.yml +++ b/.ci/ansible/tasks/setup_test_script.yml @@ -11,6 +11,9 @@ - "export PROVIDER=docker" - "export DEVELOPER_MODE=true" - "export SKIP_PULL=1" + - "export STACK_VERSION={{ lookup('env', 'STACK_VERSION') or lookup('file', '{{workspace}}/.stack-version') }}" + - "export KIBANA_VERSION={{ lookup('env', 'KIBANA_VERSION') or lookup('file', '{{ workspace }}/.stack-version') }}" + - "export ELASTIC_AGENT_VERSION={{ lookup('env', 'ELASTIC_AGENT_VERSION') or lookup('file', '{{ workspace }}/.stack-version') }}" when: "'stack' == nodeLabel" - name: Extend environment for Fleet testing @@ -57,6 +60,8 @@ with_items: - "export SUITE={{ lookup('env', 'SUITE') or 'kubernetes-autodiscover' }}" - "export PROVIDER={{ lookup('env', 'PROVIDER') or 'docker' }}" + - "export KIND_VERSION={{ lookup('env', 'KIND_VERSION') }}" + - "export KUBERNETES_VERSION={{ lookup('env', 'KUBERNETES_VERSION') }}" when: - suite is defined - suite == "kubernetes-autodiscover" @@ -71,6 +76,10 @@ with_items: - "export SUITE={{ lookup('env', 'SUITE') or 'helm' }}" - "export PROVIDER={{ lookup('env', 'PROVIDER') or 'docker' }}" + - "export HELM_CHART_VERSION={{ lookup('env', 'HELM_CHART_VERSION') }}" + - "export HELM_VERSION={{ lookup('env', 'HELM_VERSION') }}" + - "export KIND_VERSION={{ lookup('env', 'KIND_VERSION') }}" + - "export KUBERNETES_VERSION={{ lookup('env', 'KUBERNETES_VERSION') }}" when: - suite is defined - suite == "helm" @@ -85,6 +94,9 @@ with_items: - "export STACK_VERSION={{ lookup('file', '{{workspace}}/.stack-version') or '8.0.0-SNAPSHOT' }}" - "export BEAT_VERSION={{ lookup('file', '{{workspace}}/.stack-version') or '8.0.0-SNAPSHOT' }}" + - "export ELASTIC_AGENT_VERSION={{ lookup('file', '{{workspace}}/.stack-version') or '8.0.0-SNAPSHOT' }}" + - "export GITHUB_CHECK_SHA1={{ lookup('env', 'GITHUB_CHECK_SHA1') }}" + - "export GITHUB_CHECK_REPO={{ lookup('env', 'GITHUB_CHECK_REPO') }}" - "export ELASTIC_APM_GLOBAL_LABELS={{ lookup('env', 'ELASTIC_APM_GLOBAL_LABELS') }}" - "export PATH=$PATH:/opt/go/{{golang_version}}/bin:/usr/local/bin" @@ -110,4 +122,5 @@ FORMAT=junit:${REPORT}.xml \ STACK_VERSION=${STACK_VERSION} \ BEAT_VERSION=${BEAT_VERSION} \ + ELASTIC_AGENT_VERSION=${ELASTIC_AGENT_VERSION} \ make --no-print-directory -C "${BASE_DIR}/e2e/_suites/${SUITE}" functional-test diff --git a/.ci/bump-stack-version.sh b/.ci/bump-stack-version.sh index 6514fd53bf..17b24dab90 100755 --- a/.ci/bump-stack-version.sh +++ b/.ci/bump-stack-version.sh @@ -36,6 +36,7 @@ git add $FILE echo "Update stack with version ${VERSION} in Jenkinsfile" FILE="./.ci/Jenkinsfile" ${SED} -E -e "s#(name: 'BEAT_VERSION', defaultValue: ')[0-9]+\.[0-9]+\.[0-9]+(-[a-f0-9]{8})?#\1${VERSION}#g" $FILE +${SED} -E -e "s#(name: 'ELASTIC_AGENT_VERSION', defaultValue: ')[0-9]+\.[0-9]+\.[0-9]+(-[a-f0-9]{8})?#\1${VERSION}#g" $FILE ${SED} -E -e "s#(name: 'STACK_VERSION', defaultValue: ')[0-9]+\.[0-9]+\.[0-9]+(-[a-f0-9]{8})?#\1${VERSION}#g" $FILE git add $FILE diff --git a/.ci/scripts/functional-test.sh b/.ci/scripts/functional-test.sh index bfeffd5f70..94984456a0 100755 --- a/.ci/scripts/functional-test.sh +++ b/.ci/scripts/functional-test.sh @@ -14,6 +14,7 @@ set -euxo pipefail # - TAGS - that's the tags to be tested. Default '' which means all of them. # - STACK_VERSION - that's the version of the stack to be tested. Default is stored in '.stack-version'. # - BEAT_VERSION - that's the version of the Beat to be tested. Default is stored in '.stack-version'. +# - ELASTIC_AGENT_VERSION - that's the version of the Elastic Agent to be tested. Default is stored in '.stack-version'. # BASE_VERSION="$(cat $(pwd)/.stack-version)" @@ -22,6 +23,7 @@ SUITE=${1:-''} TAGS=${2:-''} STACK_VERSION=${3:-"${BASE_VERSION}"} BEAT_VERSION=${4:-"${BASE_VERSION}"} +ELASTIC_AGENT_VERSION=${5:-"${BASE_VERSION}"} GOARCH=${GOARCH:-"amd64"} ## Install the required dependencies for the given SUITE @@ -32,4 +34,4 @@ mkdir -p outputs REPORT="$(pwd)/outputs/TEST-${GOARCH}-${SUITE}" -TAGS="${TAGS}" FORMAT=junit:${REPORT}.xml GOARCH=${GOARCH} STACK_VERSION=${STACK_VERSION} BEAT_VERSION=${BEAT_VERSION} make --no-print-directory -C e2e/_suites/${SUITE} functional-test +TAGS="${TAGS}" FORMAT=junit:${REPORT}.xml GOARCH=${GOARCH} STACK_VERSION=${STACK_VERSION} BEAT_VERSION=${BEAT_VERSION} ELASTIC_AGENT_VERSION=${ELASTIC_AGENT_VERSION} make --no-print-directory -C e2e/_suites/${SUITE} functional-test diff --git a/Makefile b/Makefile index 6bb5ccfbb6..8080147bd0 100644 --- a/Makefile +++ b/Makefile @@ -37,7 +37,7 @@ notice: -depsOut "" .PHONY: unit-test -unit-test: test-report-setup unit-test-dir-cli unit-test-dir-internal +unit-test: test-report-setup unit-test-dir-cli unit-test-dir-internal unit-test-dir-pkg # See https://pkg.go.dev/gotest.tools/gotestsum/#readme-junit-xml-output .PHONY: unit-test-suite-% diff --git a/e2e/README.md b/e2e/README.md index 532a141eb9..6f46bf469c 100644 --- a/e2e/README.md +++ b/e2e/README.md @@ -105,7 +105,8 @@ We are going to enumerate the variables that will affect the product versions us > Related to this compatibility matrix too, it's also remarkable that Kibana **Fleet** plugin should not allow to enroll an agent with a version higher than kibana (See https://github.com/elastic/kibana/blob/fed9a4fddcc0087ee9eca6582a2a84e001890f08/x-pack/test/fleet_api_integration/apis/agents/enroll.ts#L99). #### Fleet -- `BEAT_VERSION`. Set this environment variable to the proper version of the Elastic Agent to be used in the current execution. Default: See https://github.com/elastic/e2e-testing/blob/70b1d3ddaf39567aeb4c322054b93ad7ce53e825/.ci/Jenkinsfile#L44 +- `BEAT_VERSION`. Set this environment variable to the proper version of the Beats to be used in the current execution. Default: See https://github.com/elastic/e2e-testing/blob/70b1d3ddaf39567aeb4c322054b93ad7ce53e825/.ci/Jenkinsfile#L44 +- `ELASTIC_AGENT_VERSION`. Set this environment variable to the proper version of the Elastic Agent to be used in the current execution. Default: See https://github.com/elastic/e2e-testing/blob/70b1d3ddaf39567aeb4c322054b93ad7ce53e825/.ci/Jenkinsfile#L44 - `ELASTIC_AGENT_DOWNLOAD_URL`. Set this environment variable if you know the bucket URL for an Elastic Agent artifact generated by the CI, i.e. for a pull request. It will take precedence over the `BEAT_VERSION` variable. Default empty: See https://github.com/elastic/e2e-testing/blob/0446248bae1ff604219735998841a21a7576bfdd/.ci/Jenkinsfile#L35 - `ELASTIC_AGENT_STALE_VERSION`. Set this environment variable to the proper version of the Elastic Agent to be used in the upgrade tests, representing the version to be upgraded. Default: See https://github.com/elastic/e2e-testing/blob/b8d0cb09d575f90f447fe3331b6df0a185c01c89/.ci/Jenkinsfile#L38 @@ -117,6 +118,7 @@ We are going to enumerate the variables that will affect the product versions us #### Kubernetes autodiscover charts - `BEAT_VERSION`. Set this environment variable to the proper version of the Beat to be used in the current execution. Default: See https://github.com/elastic/e2e-testing/blob/70b1d3ddaf39567aeb4c322054b93ad7ce53e825/.ci/Jenkinsfile#L44 +- `ELASTIC_AGENT_VERSION`. Set this environment variable to the proper version of the Elastic Agent to be used in the current execution. Default: See https://github.com/elastic/e2e-testing/blob/70b1d3ddaf39567aeb4c322054b93ad7ce53e825/.ci/Jenkinsfile#L44 - `KIND_VERSION`. Set this environment variable to the proper version of Kind (Kubernetes in Docker) to be used in the current execution. Default: See https://github.com/elastic/e2e-testing/blob/0446248bae1ff604219735998841a21a7576bfdd/.ci/Jenkinsfile#L45 - `KUBERNETES_VERSION`. Set this environment variable to the proper version of Kubernetes to be used in the current execution. Default: See https://github.com/elastic/e2e-testing/blob/0446248bae1ff604219735998841a21a7576bfdd/.ci/Jenkinsfile#L46 @@ -127,7 +129,8 @@ The following environment variables affect how the tests are run in both the CI - `ELASTIC_APM_ENVIRONMENT`: Set this environment variable to `ci` to send APM data to Elastic Cloud. Otherwise, the framework will spin up local APM Server and Kibana instances. For the CI, it will read credentials from Vault. Default value: `local`. - `SKIP_PULL`: Set this environment variable to prevent the test suite to pull Docker images and/or external dependencies for all components. Default: `false` - `BEATS_LOCAL_PATH`: Set this environment variable to the base path to your local clone of Beats if it's needed to use the binary snapshots produced by your local build instead of the official releases. The snapshots will be fetched from the `${BEATS_LOCAL_PATH}/${THE_BEAT}/build/distributions` local directory. This variable is intended to be used by Beats developers, when testing locally the artifacts generated its own build. Default: empty. -- `GITHUB_CHECK_SHA1`: Set this environment variable to the git commit in the Beats repo to use the binary snapshots produced by Beats CI instead of the official releases. The snapshots will be downloaded from a bucket in Google Cloud Storage. This variable is used by the Beats repository, when testing the artifacts generated by the packaging job. Default: empty. +- `GITHUB_CHECK_SHA1`: Set this environment variable to the git commit in the right repository to use the binary snapshots produced by the CI instead of the official releases. The snapshots will be downloaded from a bucket in Google Cloud Storage. This variable is used by the repository, when testing the artifacts generated by the packaging job. Default: empty. +- `GITHUB_CHECK_REPO`: Set this environment variable to the name of the Github repository where the above git SHA commit lives. Default: elastic-agent. - `LOG_LEVEL`: Set this environment variable to `TRACE`, `DEBUG`, `INFO`, `WARN`, `ERROR` or `FATAL` to set the log level in the project. Default: `INFO`. - `DEVELOPER_MODE`: Set this environment variable to `true` to activate developer mode, which means not destroying the services provisioned by the test framework. Default: `false`. - `KIBANA_VERSION`. Set this environment variable to the proper version of the Kibana instance to be used in the current execution, which should be used for the Docker tag of the kibana instance. It will refer to an image related to a Kibana PR, under the Observability-CI namespace. Default is empty @@ -154,6 +157,7 @@ git checkout main TAGS="fleet_mode_agent" \ TIMEOUT_FACTOR=3 LOG_LEVEL=TRACE \ BEAT_VERSION="7.10.1" \ + ELASTIC_AGENT_VERSION="7.10.1" \ make -C e2e/_suites/fleet functional-test ``` Or running by feature file: @@ -163,6 +167,7 @@ git checkout main FEATURES="fleet_mode_agent.feature" \ TIMEOUT_FACTOR=3 LOG_LEVEL=TRACE \ BEAT_VERSION="7.10.1" \ + ELASTIC_AGENT_VERSION="7.10.1" \ make -C e2e/_suites/fleet functional-test ``` @@ -214,8 +219,8 @@ To do so: 1. In the input parameters form, keep the Beat version (for Fleet) as is, to use each branch's default version. 1. In the input parameters form, keep the stack version (for Fleet) as is, to use each branch's default version. 1. In the input parameters form, set the `GITHUB_CHECK_NAME` to `E2E Tests`. This value will appear as the label for the Github check for the E2E tests. -1. In the input parameters form, set the `GITHUB_CHECK_REPO` to `beats`. 1. In the input parameters form, set the `GITHUB_CHECK_SHA1` to the `SHA1` of the last commit in your pull request. This value will allow us to modify the mergeable status of that commit with the Github check. Besides that, it will set the specific directory in the GCP bucket to look up the CI binaries. +1. In the input parameters form, set the `GITHUB_CHECK_REPO` to `elastic-agent` or `beats`, depending where the aforementioned SHA1 belongs. 1. Click the `Build` button at the bottom of the parameters form. ## Noticing the test framework diff --git a/e2e/_suites/fleet/README.md b/e2e/_suites/fleet/README.md index 4bbcefbd47..c8629a404d 100644 --- a/e2e/_suites/fleet/README.md +++ b/e2e/_suites/fleet/README.md @@ -88,6 +88,8 @@ This is an example of the optional configuration: export ELASTIC_AGENT_DOWNLOAD_URL="https://snapshots.elastic.co/7.12.0-069dfaa4/downloads/beats/elastic-agent/elastic-agent-7.12.0-SNAPSHOT-linux-x86_64.tar.gz" # This environment variable will use the its value as the Docker tag produced by Beats CI (Please look up Google Cloud Storage CI bucket). export GITHUB_CHECK_SHA1="78a762c76080aafa34c52386341b590dac24e2df" + # This environment variable will use the name of the repository where the commit lives + export GITHUB_CHECK_REPO="beats" ``` 3. Define the proper Docker images to be used in tests (Optional). diff --git a/e2e/_suites/fleet/fleet.go b/e2e/_suites/fleet/fleet.go index 5bd4ac3f9a..9f8f7a94a5 100644 --- a/e2e/_suites/fleet/fleet.go +++ b/e2e/_suites/fleet/fleet.go @@ -21,7 +21,6 @@ import ( "github.com/cenkalti/backoff/v4" "github.com/cucumber/godog" - elasticversion "github.com/elastic/e2e-testing/internal" "github.com/elastic/e2e-testing/internal/common" "github.com/elastic/e2e-testing/internal/deploy" "github.com/elastic/e2e-testing/internal/elasticsearch" @@ -29,6 +28,7 @@ import ( "github.com/elastic/e2e-testing/internal/kibana" "github.com/elastic/e2e-testing/internal/shell" "github.com/elastic/e2e-testing/internal/utils" + "github.com/elastic/e2e-testing/pkg/downloads" "github.com/pkg/errors" log "github.com/sirupsen/logrus" @@ -158,7 +158,7 @@ func (fts *FleetTestSuite) beforeScenario() { fts.StandAlone = false fts.ElasticAgentStopped = false - fts.Version = common.BeatVersion + fts.Version = common.ElasticAgentVersion waitForPolicy := func() error { policy, err := fts.kibanaClient.CreatePolicy(fts.currentContext) @@ -350,7 +350,7 @@ func (fts *FleetTestSuite) anStaleAgentIsDeployedToFleetWithInstaller(version, i common.AgentStaleVersion = shell.GetEnv("ELASTIC_AGENT_STALE_VERSION", common.AgentStaleVersion) // check if stale version is an alias - v, err := elasticversion.GetElasticArtifactVersion(common.AgentStaleVersion) + v, err := downloads.GetElasticArtifactVersion(common.AgentStaleVersion) if err != nil { log.WithFields(log.Fields{ "error": err, @@ -360,8 +360,7 @@ func (fts *FleetTestSuite) anStaleAgentIsDeployedToFleetWithInstaller(version, i } common.AgentStaleVersion = v - useCISnapshots := elasticversion.GithubCommitSha1 != "" - if useCISnapshots && !strings.HasSuffix(common.AgentStaleVersion, "-SNAPSHOT") { + if downloads.UseElasticAgentCISnapshots() && !strings.HasSuffix(common.AgentStaleVersion, "-SNAPSHOT") { common.AgentStaleVersion += "-SNAPSHOT" } @@ -369,7 +368,7 @@ func (fts *FleetTestSuite) anStaleAgentIsDeployedToFleetWithInstaller(version, i case "stale": version = common.AgentStaleVersion case "latest": - version = common.BeatVersion + version = common.ElasticAgentVersion default: version = common.AgentStaleVersion } @@ -386,7 +385,7 @@ func (fts *FleetTestSuite) installCerts() error { err := agentInstaller.InstallCerts(fts.currentContext) if err != nil { log.WithFields(log.Fields{ - "agentVersion": common.BeatVersion, + "agentVersion": common.ElasticAgentVersion, "agentStaleVersion": common.AgentStaleVersion, "error": err, "installer": agentInstaller, @@ -403,9 +402,9 @@ func (fts *FleetTestSuite) anAgentIsUpgraded(desiredVersion string) error { case "stale": desiredVersion = common.AgentStaleVersion case "latest": - desiredVersion = common.BeatVersion + desiredVersion = common.ElasticAgentVersion default: - desiredVersion = common.BeatVersion + desiredVersion = common.ElasticAgentVersion } agentService := deploy.NewServiceRequest(common.ElasticAgentServiceName) @@ -418,7 +417,7 @@ func (fts *FleetTestSuite) agentInVersion(version string) error { case "stale": version = common.AgentStaleVersion case "latest": - version = elasticversion.GetSnapshotVersion(common.BeatVersion) + version = downloads.GetSnapshotVersion(common.ElasticAgentVersion) } agentInVersionFn := func() error { @@ -678,7 +677,7 @@ func bootstrapFleet(ctx context.Context, env map[string]string) error { for k, v := range env { fleetServerEnv[k] = v } - fleetServerEnv["elasticAgentTag"] = common.BeatVersion + fleetServerEnv["elasticAgentTag"] = common.ElasticAgentVersion fleetServerEnv["fleetServerMode"] = "1" fleetServerEnv["fleetServerPort"] = "8220" fleetServerEnv["fleetInsecure"] = "1" diff --git a/e2e/_suites/fleet/ingest_manager_test.go b/e2e/_suites/fleet/ingest_manager_test.go index 193c362868..ec8140de79 100644 --- a/e2e/_suites/fleet/ingest_manager_test.go +++ b/e2e/_suites/fleet/ingest_manager_test.go @@ -132,11 +132,11 @@ func InitializeIngestManagerTestSuite(ctx *godog.TestSuiteContext) { // cyclic imports since common.defaults now imports deploy module if !shell.GetEnvBool("SKIP_PULL") && common.Provider != "remote" { images := []string{ - "docker.elastic.co/beats/elastic-agent:" + common.BeatVersion, - "docker.elastic.co/beats/elastic-agent-ubi8:" + common.BeatVersion, + "docker.elastic.co/beats/elastic-agent:" + common.ElasticAgentVersion, + "docker.elastic.co/beats/elastic-agent-ubi8:" + common.ElasticAgentVersion, "docker.elastic.co/elasticsearch/elasticsearch:" + common.StackVersion, - "docker.elastic.co/observability-ci/elastic-agent:" + common.BeatVersion, - "docker.elastic.co/observability-ci/elastic-agent-ubi8:" + common.BeatVersion, + "docker.elastic.co/observability-ci/elastic-agent:" + common.ElasticAgentVersion, + "docker.elastic.co/observability-ci/elastic-agent-ubi8:" + common.ElasticAgentVersion, "docker.elastic.co/observability-ci/elasticsearch:" + common.StackVersion, "docker.elastic.co/observability-ci/kibana:" + common.KibanaVersion, } diff --git a/e2e/_suites/fleet/stand-alone.go b/e2e/_suites/fleet/stand-alone.go index 9f37039449..f6ad50357d 100644 --- a/e2e/_suites/fleet/stand-alone.go +++ b/e2e/_suites/fleet/stand-alone.go @@ -10,13 +10,12 @@ import ( "time" "github.com/cenkalti/backoff/v4" - elasticversion "github.com/elastic/e2e-testing/internal" "github.com/elastic/e2e-testing/internal/common" "github.com/elastic/e2e-testing/internal/deploy" "github.com/elastic/e2e-testing/internal/installer" "github.com/elastic/e2e-testing/internal/kibana" - "github.com/elastic/e2e-testing/internal/shell" "github.com/elastic/e2e-testing/internal/utils" + "github.com/elastic/e2e-testing/pkg/downloads" "github.com/elastic/e2e-testing/internal/elasticsearch" log "github.com/sirupsen/logrus" @@ -83,7 +82,7 @@ func (fts *FleetTestSuite) startStandAloneAgent(image string, flavour string, en fts.StandAlone = true log.Trace("Deploying an agent to Fleet") - dockerImageTag := common.BeatVersion + dockerImageTag := common.ElasticAgentVersion common.ProfileEnv["elasticAgentDockerNamespace"] = deploy.GetDockerNamespaceEnvVar("beats") common.ProfileEnv["elasticAgentDockerImageSuffix"] = "" @@ -91,10 +90,7 @@ func (fts *FleetTestSuite) startStandAloneAgent(image string, flavour string, en common.ProfileEnv["elasticAgentDockerImageSuffix"] = "-" + image } - useCISnapshots := elasticversion.GithubCommitSha1 != "" - beatsLocalPath := shell.GetEnv("BEATS_LOCAL_PATH", "") - - if useCISnapshots || beatsLocalPath != "" { + if downloads.UseElasticAgentCISnapshots() || downloads.BeatsLocalPath != "" { // load the docker images that were already: // a. downloaded from the GCP bucket // b. fetched from the local beats binaries diff --git a/e2e/_suites/kubernetes-autodiscover/README.md b/e2e/_suites/kubernetes-autodiscover/README.md index bc8662faf7..e182c28775 100644 --- a/e2e/_suites/kubernetes-autodiscover/README.md +++ b/e2e/_suites/kubernetes-autodiscover/README.md @@ -105,6 +105,7 @@ This is an example of the optional configuration: ```shell # Depending on the versions used, export BEAT_VERSION=7.12.0 # version of beats to use + export ELASTIC_AGENT_VERSION=7.12.0 # version of Elastic Agent to use export GITHUB_CHECK_SHA1=0123456789 # to select snapshots built by beats-ci export KUBERNETES_VERSION="1.18.2" # version of the cluster to be passed to kind ``` diff --git a/e2e/_suites/kubernetes-autodiscover/autodiscover_test.go b/e2e/_suites/kubernetes-autodiscover/autodiscover_test.go index b35cd02bbd..beace6205d 100644 --- a/e2e/_suites/kubernetes-autodiscover/autodiscover_test.go +++ b/e2e/_suites/kubernetes-autodiscover/autodiscover_test.go @@ -28,12 +28,12 @@ import ( "go.elastic.co/apm" "github.com/elastic/e2e-testing/cli/config" - elasticversion "github.com/elastic/e2e-testing/internal" "github.com/elastic/e2e-testing/internal/common" "github.com/elastic/e2e-testing/internal/deploy" "github.com/elastic/e2e-testing/internal/kubernetes" "github.com/elastic/e2e-testing/internal/shell" "github.com/elastic/e2e-testing/internal/utils" + "github.com/elastic/e2e-testing/pkg/downloads" ) var beatVersions = map[string]string{} @@ -76,7 +76,7 @@ func (m *podsManager) executeTemplateFor(podName string, writer io.Writer, optio return false }, "beats_namespace": func() string { - return deploy.GetDockerNamespaceEnvVar("beats") + return deploy.GetDockerNamespaceEnvVarForRepository(podName, "beats") }, "beats_version": func() string { return beatVersions[podName] @@ -133,14 +133,21 @@ func (m *podsManager) configureDockerImage(podName string) error { return nil } - beatVersion := elasticversion.GetSnapshotVersion(common.BeatVersion) + "-amd64" + v := common.BeatVersion + if strings.EqualFold(podName, "elastic-agent") { + v = common.ElasticAgentVersion + } + beatVersion := downloads.GetSnapshotVersion(v) + "-amd64" + + ciSnapshotsFn := downloads.UseBeatsCISnapshots + if strings.EqualFold(podName, "elastic-agent") { + ciSnapshotsFn = downloads.UseElasticAgentCISnapshots + } - useCISnapshots := elasticversion.GithubCommitSha1 != "" - beatsLocalPath := shell.GetEnv("BEATS_LOCAL_PATH", "") - if useCISnapshots || beatsLocalPath != "" { + if ciSnapshotsFn() || downloads.BeatsLocalPath != "" { log.Debugf("Configuring Docker image for %s", podName) - _, imagePath, err := elasticversion.FetchElasticArtifact(m.ctx, podName, common.BeatVersion, "linux", "amd64", "tar.gz", true, true) + _, imagePath, err := downloads.FetchElasticArtifact(m.ctx, podName, v, "linux", "amd64", "tar.gz", true, true) if err != nil { return err } @@ -153,7 +160,7 @@ func (m *podsManager) configureDockerImage(podName string) error { // tag the image with the proper docker tag, including platform err = deploy.TagImage( - "docker.elastic.co/beats/"+podName+":"+elasticversion.GetSnapshotVersion(common.BeatVersionBase), + "docker.elastic.co/beats/"+podName+":"+downloads.GetSnapshotVersion(common.BeatVersionBase), "docker.elastic.co/observability-ci/"+podName+":"+beatVersion, ) if err != nil { diff --git a/internal/common/defaults.go b/internal/common/defaults.go index 043200240e..8f46baf404 100644 --- a/internal/common/defaults.go +++ b/internal/common/defaults.go @@ -5,8 +5,8 @@ package common import ( - elasticversion "github.com/elastic/e2e-testing/internal" "github.com/elastic/e2e-testing/internal/shell" + "github.com/elastic/e2e-testing/pkg/downloads" log "github.com/sirupsen/logrus" ) @@ -42,6 +42,10 @@ var BeatVersionBase = "7.17.1-952e96f1-SNAPSHOT" // It can be overriden by BEAT_VERSION env var var BeatVersion = BeatVersionBase +// ElasticAgentVersion is the version of the Elastic Agent to use +// It can be overriden by ELASTIC_AGENT_VERSION env var +var ElasticAgentVersion = BeatVersionBase + // DeveloperMode if enabled will keep deployments around after test runs var DeveloperMode = false @@ -81,7 +85,7 @@ func init() { // supporting lazy-loading the versions when needed. Basically, the CLI part does not // need to load them func InitVersions() { - v, err := elasticversion.GetElasticArtifactVersion(BeatVersionBase) + v, err := downloads.GetElasticArtifactVersion(BeatVersionBase) if err != nil { log.WithFields(log.Fields{ "error": err, @@ -93,7 +97,7 @@ func InitVersions() { BeatVersion = shell.GetEnv("BEAT_VERSION", BeatVersionBase) // check if version is an alias - v, err = elasticversion.GetElasticArtifactVersion(BeatVersion) + v, err = downloads.GetElasticArtifactVersion(BeatVersion) if err != nil { log.WithFields(log.Fields{ "error": err, @@ -111,10 +115,33 @@ func InitVersions() { }).Trace("Beat Version provided: will be used as fallback") fallbackVersion = BeatVersion } - BeatVersion = elasticversion.CheckPRVersion(BeatVersion, fallbackVersion) + BeatVersion = downloads.CheckPRVersion(BeatVersion, fallbackVersion) + + ElasticAgentVersion = shell.GetEnv("ELASTIC_AGENT_VERSION", BeatVersionBase) + + // check if version is an alias + v, err = downloads.GetElasticArtifactVersion(ElasticAgentVersion) + if err != nil { + log.WithFields(log.Fields{ + "error": err, + "version": ElasticAgentVersion, + }).Fatal("Failed to get Elastic Agent version, aborting") + } + ElasticAgentVersion = v + + // detects if the ElasticAgentVersion is set by the GITHUB_CHECK_SHA1 variable + fallbackVersion = BeatVersionBase + if ElasticAgentVersion != BeatVersionBase { + log.WithFields(log.Fields{ + "BeatVersionBase": BeatVersionBase, + "ElasticAgentVersion": ElasticAgentVersion, + }).Trace("Elastic Agent Version provided: will be used as fallback") + fallbackVersion = ElasticAgentVersion + } + ElasticAgentVersion = downloads.CheckPRVersion(ElasticAgentVersion, fallbackVersion) StackVersion = shell.GetEnv("STACK_VERSION", BeatVersionBase) - v, err = elasticversion.GetElasticArtifactVersion(StackVersion) + v, err = downloads.GetElasticArtifactVersion(StackVersion) if err != nil { log.WithFields(log.Fields{ "error": err, @@ -127,7 +154,7 @@ func InitVersions() { if KibanaVersion == "" { // we want to deploy a released version for Kibana // if not set, let's use StackVersion - KibanaVersion, err = elasticversion.GetElasticArtifactVersion(StackVersion) + KibanaVersion, err = downloads.GetElasticArtifactVersion(StackVersion) if err != nil { log.WithFields(log.Fields{ "error": err, @@ -137,9 +164,10 @@ func InitVersions() { } log.WithFields(log.Fields{ - "BeatVersionBase": BeatVersionBase, - "BeatVersion": BeatVersion, - "StackVersion": StackVersion, - "KibanaVersion": KibanaVersion, + "BeatVersionBase": BeatVersionBase, + "BeatVersion": BeatVersion, + "ElasticAgentVersion": ElasticAgentVersion, + "StackVersion": StackVersion, + "KibanaVersion": KibanaVersion, }).Info("Initial artifact versions defined") } diff --git a/internal/deploy/docker.go b/internal/deploy/docker.go index fb137ec729..978506629a 100644 --- a/internal/deploy/docker.go +++ b/internal/deploy/docker.go @@ -10,8 +10,8 @@ import ( "path/filepath" "strings" - elasticversion "github.com/elastic/e2e-testing/internal" "github.com/elastic/e2e-testing/internal/shell" + "github.com/elastic/e2e-testing/pkg/downloads" log "github.com/sirupsen/logrus" "go.elastic.co/apm" ) @@ -278,9 +278,19 @@ func (c *dockerDeploymentManifest) Stop(ctx context.Context, service ServiceRequ // the images produced by local Beats build, or not. // If an error occurred reading the environment, will return the passed namespace as fallback func GetDockerNamespaceEnvVar(fallback string) string { - beatsLocalPath := shell.GetEnv("BEATS_LOCAL_PATH", "") - useCISnapshots := elasticversion.GithubCommitSha1 != "" - if useCISnapshots || beatsLocalPath != "" { + return GetDockerNamespaceEnvVarForRepository("elastic-agent", fallback) +} + +// GetDockerNamespaceEnvVarForRepository returns the Docker namespace whether we use one of the CI snapshots or +// the images produced by local Beats build, or not. +// If an error occurred reading the environment, will return the passed namespace as fallback +func GetDockerNamespaceEnvVarForRepository(repo string, fallback string) string { + ciSnapshotsFn := downloads.UseBeatsCISnapshots + if strings.EqualFold(repo, "elastic-agent") { + ciSnapshotsFn = downloads.UseElasticAgentCISnapshots + } + + if ciSnapshotsFn() || downloads.BeatsLocalPath != "" { return "observability-ci" } return fallback diff --git a/internal/deploy/docker_test.go b/internal/deploy/docker_test.go index 1626596507..ee8a69534d 100644 --- a/internal/deploy/docker_test.go +++ b/internal/deploy/docker_test.go @@ -7,8 +7,7 @@ package deploy import ( "testing" - elasticversion "github.com/elastic/e2e-testing/internal" - + "github.com/elastic/e2e-testing/pkg/downloads" "github.com/stretchr/testify/assert" ) @@ -19,10 +18,40 @@ func TestGetDockerNamespaceEnvVar(t *testing.T) { }) t.Run("Returns Observability CI when environment variable is set", func(t *testing.T) { - elasticversion.GithubCommitSha1 = "0123456789" - defer func() { elasticversion.GithubCommitSha1 = "" }() + downloads.GithubCommitSha1 = "0123456789" + defer func() { downloads.GithubCommitSha1 = "" }() namespace := GetDockerNamespaceEnvVar("beats") assert.True(t, namespace == "observability-ci") }) } + +func TestGetDockerNamespaceEnvVarForRepository(t *testing.T) { + t.Run("Returns fallback when environment variables are not set", func(t *testing.T) { + namespace := GetDockerNamespaceEnvVarForRepository("filebeat", "beats") + assert.True(t, namespace == "beats") + }) + + t.Run("Returns Observability CI when environment variable is set for Beats", func(t *testing.T) { + downloads.GithubCommitSha1 = "0123456789" + downloads.GithubRepository = "beats" + defer func() { + downloads.GithubCommitSha1 = "" + downloads.GithubRepository = "elastic-agent" + }() + + namespace := GetDockerNamespaceEnvVarForRepository("filebeat", "beats") + assert.True(t, namespace == "observability-ci") + }) + + t.Run("Returns Observability CI when environment variable is set for elastic-agent", func(t *testing.T) { + downloads.GithubCommitSha1 = "0123456789" + defer func() { + downloads.GithubCommitSha1 = "" + downloads.GithubRepository = "elastic-agent" + }() + + namespace := GetDockerNamespaceEnvVarForRepository("elastic-agent", "beats") + assert.True(t, namespace == "observability-ci") + }) +} diff --git a/internal/deploy/elastic_package.go b/internal/deploy/elastic_package.go index edf5c9c01b..6e52a2fae8 100644 --- a/internal/deploy/elastic_package.go +++ b/internal/deploy/elastic_package.go @@ -41,12 +41,12 @@ func newElasticPackage() Deployment { // Add adds services deployment: the first service in the list must be the profile in which to deploy the service func (ep *EPServiceManager) Add(ctx context.Context, profile ServiceRequest, services []ServiceRequest, env map[string]string) error { - version := common.BeatVersion + version := common.ElasticAgentVersion span, _ := apm.StartSpanOptions(ctx, "Adding elastic-agent to Elastic-Package deployment", "elastic-package.elastic-agent.add", apm.SpanOptions{ Parent: apm.SpanFromContext(ctx).TraceContext(), }) - span.Context.SetLabel("beatVersion", version) + span.Context.SetLabel("elasticAgentVersion", version) span.Context.SetLabel("profile", profile) span.Context.SetLabel("services", services) defer span.End() diff --git a/internal/installer/elasticagent_deb.go b/internal/installer/elasticagent_deb.go index f157150045..a805bcc2cd 100644 --- a/internal/installer/elasticagent_deb.go +++ b/internal/installer/elasticagent_deb.go @@ -9,11 +9,11 @@ import ( "fmt" "strings" - elasticversion "github.com/elastic/e2e-testing/internal" "github.com/elastic/e2e-testing/internal/common" "github.com/elastic/e2e-testing/internal/deploy" "github.com/elastic/e2e-testing/internal/kibana" "github.com/elastic/e2e-testing/internal/utils" + "github.com/elastic/e2e-testing/pkg/downloads" log "github.com/sirupsen/logrus" "go.elastic.co/apm" ) @@ -132,7 +132,7 @@ func (i *elasticAgentDEBPackage) Postinstall(ctx context.Context) error { // Preinstall executes operations before installing a DEB package func (i *elasticAgentDEBPackage) Preinstall(ctx context.Context) error { - installArtifactFn := func(ctx context.Context, artifact string) error { + installArtifactFn := func(ctx context.Context, artifact string, version string, useCISnapshots bool) error { span, _ := apm.StartSpanOptions(ctx, "Pre-install "+artifact, artifact+".debian.pre-install", apm.SpanOptions{ Parent: apm.SpanFromContext(ctx).TraceContext(), }) @@ -142,11 +142,11 @@ func (i *elasticAgentDEBPackage) Preinstall(ctx context.Context) error { arch := utils.GetArchitecture() extension := "deb" - binaryName, binaryPath, err := elasticversion.FetchElasticArtifact(ctx, artifact, common.BeatVersion, os, arch, extension, false, true) + binaryName, binaryPath, err := downloads.FetchElasticArtifactForSnapshots(ctx, useCISnapshots, artifact, version, os, arch, extension, false, true) if err != nil { log.WithFields(log.Fields{ "artifact": artifact, - "version": common.BeatVersion, + "version": version, "os": os, "arch": arch, "extension": extension, @@ -170,15 +170,15 @@ func (i *elasticAgentDEBPackage) Preinstall(ctx context.Context) error { for _, bp := range i.service.BackgroundProcesses { if strings.EqualFold(bp, "filebeat") || strings.EqualFold(bp, "metricbeat") { - // pre-install the dependant binary first - err := installArtifactFn(ctx, bp) + // pre-install the dependant binary first, using the stack version + err := installArtifactFn(ctx, bp, common.BeatVersion, downloads.UseBeatsCISnapshots()) if err != nil { return err } } } - return installArtifactFn(ctx, "elastic-agent") + return installArtifactFn(ctx, "elastic-agent", common.ElasticAgentVersion, downloads.UseElasticAgentCISnapshots()) } // Restart will restart a service diff --git a/internal/installer/elasticagent_docker.go b/internal/installer/elasticagent_docker.go index b9a8843681..5e723856ae 100644 --- a/internal/installer/elasticagent_docker.go +++ b/internal/installer/elasticagent_docker.go @@ -8,10 +8,10 @@ import ( "context" "fmt" - elasticversion "github.com/elastic/e2e-testing/internal" "github.com/elastic/e2e-testing/internal/common" "github.com/elastic/e2e-testing/internal/deploy" "github.com/elastic/e2e-testing/internal/utils" + "github.com/elastic/e2e-testing/pkg/downloads" log "github.com/sirupsen/logrus" "go.elastic.co/apm" ) @@ -99,11 +99,11 @@ func (i *elasticAgentDockerPackage) Preinstall(ctx context.Context) error { arch := utils.GetArchitecture() extension := "tar.gz" - _, binaryPath, err := elasticversion.FetchElasticArtifact(ctx, artifact, common.BeatVersion, os, arch, extension, true, true) + _, binaryPath, err := downloads.FetchElasticArtifact(ctx, artifact, common.ElasticAgentVersion, os, arch, extension, true, true) if err != nil { log.WithFields(log.Fields{ "artifact": artifact, - "version": common.BeatVersion, + "version": common.ElasticAgentVersion, "os": os, "arch": arch, "extension": extension, @@ -119,10 +119,10 @@ func (i *elasticAgentDockerPackage) Preinstall(ctx context.Context) error { // we need to tag the loaded image because its tag relates to the target branch return deploy.TagImage( - fmt.Sprintf("docker.elastic.co/beats/%s:%s", artifact, elasticversion.GetSnapshotVersion(common.BeatVersionBase)), - fmt.Sprintf("docker.elastic.co/observability-ci/%s:%s-%s", artifact, elasticversion.GetSnapshotVersion(common.BeatVersion), arch), + fmt.Sprintf("docker.elastic.co/beats/%s:%s", artifact, downloads.GetSnapshotVersion(common.BeatVersionBase)), + fmt.Sprintf("docker.elastic.co/observability-ci/%s:%s-%s", artifact, downloads.GetSnapshotVersion(common.ElasticAgentVersion), arch), // tagging including git commit and snapshot - fmt.Sprintf("docker.elastic.co/observability-ci/%s:%s-%s", artifact, elasticversion.GetFullVersion(common.BeatVersion), arch), + fmt.Sprintf("docker.elastic.co/observability-ci/%s:%s-%s", artifact, downloads.GetFullVersion(common.ElasticAgentVersion), arch), ) } diff --git a/internal/installer/elasticagent_rpm.go b/internal/installer/elasticagent_rpm.go index 04d84d75ef..c3c9daab61 100644 --- a/internal/installer/elasticagent_rpm.go +++ b/internal/installer/elasticagent_rpm.go @@ -9,11 +9,11 @@ import ( "fmt" "strings" - elasticversion "github.com/elastic/e2e-testing/internal" "github.com/elastic/e2e-testing/internal/common" "github.com/elastic/e2e-testing/internal/deploy" "github.com/elastic/e2e-testing/internal/kibana" "github.com/elastic/e2e-testing/internal/utils" + "github.com/elastic/e2e-testing/pkg/downloads" log "github.com/sirupsen/logrus" "go.elastic.co/apm" ) @@ -133,7 +133,7 @@ func (i *elasticAgentRPMPackage) Postinstall(ctx context.Context) error { // Preinstall executes operations before installing a RPM package func (i *elasticAgentRPMPackage) Preinstall(ctx context.Context) error { - installArtifactFn := func(ctx context.Context, artifact string) error { + installArtifactFn := func(ctx context.Context, artifact string, version string, useCISnapshots bool) error { span, _ := apm.StartSpanOptions(ctx, "Pre-install "+artifact, artifact+".rpm.pre-install", apm.SpanOptions{ Parent: apm.SpanFromContext(ctx).TraceContext(), }) @@ -146,16 +146,16 @@ func (i *elasticAgentRPMPackage) Preinstall(ctx context.Context) error { } extension := "rpm" - binaryName, binaryPath, err := elasticversion.FetchElasticArtifact(ctx, artifact, common.BeatVersion, os, arch, extension, false, true) + binaryName, binaryPath, err := downloads.FetchElasticArtifactForSnapshots(ctx, useCISnapshots, artifact, version, os, arch, extension, false, true) if err != nil { log.WithFields(log.Fields{ "artifact": artifact, - "version": common.BeatVersion, + "version": version, "os": os, "arch": arch, "extension": extension, "error": err, - }).Error("Could not download the binary for the agent") + }).Error("Could not download the binary") return err } @@ -174,15 +174,15 @@ func (i *elasticAgentRPMPackage) Preinstall(ctx context.Context) error { for _, bp := range i.service.BackgroundProcesses { if strings.EqualFold(bp, "filebeat") || strings.EqualFold(bp, "metricbeat") { - // pre-install the dependant binary first - err := installArtifactFn(ctx, bp) + // pre-install the dependant binary first, using the stack version + err := installArtifactFn(ctx, bp, common.BeatVersion, downloads.UseBeatsCISnapshots()) if err != nil { return err } } } - return installArtifactFn(ctx, "elastic-agent") + return installArtifactFn(ctx, "elastic-agent", common.ElasticAgentVersion, downloads.UseElasticAgentCISnapshots()) } // Restart will restart a service diff --git a/internal/installer/elasticagent_tar.go b/internal/installer/elasticagent_tar.go index 9e69bf7ec2..99f92d17f4 100644 --- a/internal/installer/elasticagent_tar.go +++ b/internal/installer/elasticagent_tar.go @@ -10,12 +10,12 @@ import ( "os" "strings" - elasticversion "github.com/elastic/e2e-testing/internal" "github.com/elastic/e2e-testing/internal/common" "github.com/elastic/e2e-testing/internal/deploy" "github.com/elastic/e2e-testing/internal/io" "github.com/elastic/e2e-testing/internal/kibana" "github.com/elastic/e2e-testing/internal/utils" + "github.com/elastic/e2e-testing/pkg/downloads" log "github.com/sirupsen/logrus" "go.elastic.co/apm" ) @@ -108,8 +108,8 @@ func (i *elasticAgentTARPackage) Postinstall(ctx context.Context) error { // Preinstall executes operations before installing a TAR package func (i *elasticAgentTARPackage) Preinstall(ctx context.Context) error { - installArtifactFn := func(ctx context.Context, artifact string) error { - span, _ := apm.StartSpanOptions(ctx, "Pre-install operations for the Elastic Agent", "elastic-agent.tar.pre-install", apm.SpanOptions{ + installArtifactFn := func(ctx context.Context, artifact string, version string, useCISnapshots bool) error { + span, _ := apm.StartSpanOptions(ctx, "Pre-install "+artifact, artifact+".tar.pre-install", apm.SpanOptions{ Parent: apm.SpanFromContext(ctx).TraceContext(), }) defer span.End() @@ -129,16 +129,16 @@ func (i *elasticAgentTARPackage) Preinstall(ctx context.Context) error { } log.Trace("Cleared previously downloaded artifacts") } - _, binaryPath, err := elasticversion.FetchElasticArtifact(ctx, artifact, common.BeatVersion, runningOS, arch, extension, false, true) + _, binaryPath, err := downloads.FetchElasticArtifactForSnapshots(ctx, useCISnapshots, artifact, version, runningOS, arch, extension, false, true) if err != nil { log.WithFields(log.Fields{ "artifact": artifact, - "version": common.BeatVersion, + "version": version, "os": runningOS, "arch": arch, "extension": extension, "error": err, - }).Error("Could not download the binary for the agent") + }).Error("Could not download the binary") return err } @@ -147,7 +147,7 @@ func (i *elasticAgentTARPackage) Preinstall(ctx context.Context) error { return err } - output, _ := i.Exec(ctx, []string{"mv", fmt.Sprintf("%s-%s-%s-%s", artifact, elasticversion.GetSnapshotVersion(common.BeatVersion), runningOS, arch), artifact}) + output, _ := i.Exec(ctx, []string{"mv", fmt.Sprintf("%s-%s-%s-%s", artifact, downloads.GetSnapshotVersion(version), runningOS, arch), artifact}) log.WithFields(log.Fields{ "output": output, "artifact": artifact, @@ -158,14 +158,14 @@ func (i *elasticAgentTARPackage) Preinstall(ctx context.Context) error { for _, bp := range i.service.BackgroundProcesses { if strings.EqualFold(bp, "filebeat") || strings.EqualFold(bp, "metricbeat") { // pre-install the dependant binary first - err := installArtifactFn(ctx, bp) + err := installArtifactFn(ctx, bp, common.BeatVersion, downloads.UseBeatsCISnapshots()) if err != nil { return err } } } - return installArtifactFn(ctx, "elastic-agent") + return installArtifactFn(ctx, "elastic-agent", common.ElasticAgentVersion, downloads.UseElasticAgentCISnapshots()) } diff --git a/internal/installer/elasticagent_tar_macos.go b/internal/installer/elasticagent_tar_macos.go index 93401428d3..72f4e1e1b8 100644 --- a/internal/installer/elasticagent_tar_macos.go +++ b/internal/installer/elasticagent_tar_macos.go @@ -9,11 +9,11 @@ import ( "fmt" "runtime" - elasticversion "github.com/elastic/e2e-testing/internal" "github.com/elastic/e2e-testing/internal/common" "github.com/elastic/e2e-testing/internal/deploy" "github.com/elastic/e2e-testing/internal/kibana" "github.com/elastic/e2e-testing/internal/utils" + "github.com/elastic/e2e-testing/pkg/downloads" log "github.com/sirupsen/logrus" "go.elastic.co/apm" ) @@ -117,11 +117,11 @@ func (i *elasticAgentTARDarwinPackage) Preinstall(ctx context.Context) error { } extension := "tar.gz" - _, binaryPath, err := elasticversion.FetchElasticArtifact(ctx, artifact, common.BeatVersion, os, arch, extension, false, true) + _, binaryPath, err := downloads.FetchElasticArtifact(ctx, artifact, common.ElasticAgentVersion, os, arch, extension, false, true) if err != nil { log.WithFields(log.Fields{ "artifact": artifact, - "version": common.BeatVersion, + "version": common.ElasticAgentVersion, "os": os, "arch": arch, "extension": extension, @@ -135,7 +135,7 @@ func (i *elasticAgentTARDarwinPackage) Preinstall(ctx context.Context) error { return err } - output, _ := i.Exec(ctx, []string{"mv", fmt.Sprintf("/%s-%s-%s-%s", artifact, elasticversion.GetSnapshotVersion(common.BeatVersion), os, arch), "/elastic-agent"}) + output, _ := i.Exec(ctx, []string{"mv", fmt.Sprintf("/%s-%s-%s-%s", artifact, downloads.GetSnapshotVersion(common.ElasticAgentVersion), os, arch), "/elastic-agent"}) log.WithField("output", output).Trace("Moved elastic-agent") return nil } diff --git a/internal/installer/elasticagent_zip.go b/internal/installer/elasticagent_zip.go index 8696c728b5..9265ce58f5 100644 --- a/internal/installer/elasticagent_zip.go +++ b/internal/installer/elasticagent_zip.go @@ -9,10 +9,10 @@ import ( "fmt" "strings" - elasticversion "github.com/elastic/e2e-testing/internal" "github.com/elastic/e2e-testing/internal/common" "github.com/elastic/e2e-testing/internal/deploy" "github.com/elastic/e2e-testing/internal/kibana" + "github.com/elastic/e2e-testing/pkg/downloads" log "github.com/sirupsen/logrus" "go.elastic.co/apm" ) @@ -110,11 +110,11 @@ func (i *elasticAgentZIPPackage) Preinstall(ctx context.Context) error { arch := "x86_64" extension := "zip" - _, binaryPath, err := elasticversion.FetchElasticArtifact(ctx, artifact, common.BeatVersion, os, arch, extension, false, true) + _, binaryPath, err := downloads.FetchElasticArtifact(ctx, artifact, common.ElasticAgentVersion, os, arch, extension, false, true) if err != nil { log.WithFields(log.Fields{ "artifact": artifact, - "version": common.BeatVersion, + "version": common.ElasticAgentVersion, "os": os, "arch": arch, "extension": extension, @@ -135,7 +135,7 @@ func (i *elasticAgentZIPPackage) Preinstall(ctx context.Context) error { return err } - output, _ := i.Exec(ctx, []string{"powershell.exe", "Move-Item", "-Force", "-Path", fmt.Sprintf("C:\\%s-%s-%s-%s", artifact, elasticversion.GetSnapshotVersion(common.BeatVersion), os, arch), "-Destination", "C:\\elastic-agent"}) + output, _ := i.Exec(ctx, []string{"powershell.exe", "Move-Item", "-Force", "-Path", fmt.Sprintf("C:\\%s-%s-%s-%s", artifact, downloads.GetSnapshotVersion(common.ElasticAgentVersion), os, arch), "-Destination", "C:\\elastic-agent"}) log.WithField("output", output).Trace("Moved elastic-agent") return nil } diff --git a/internal/utils/utils.go b/internal/utils/utils.go index f0adc405b2..13c28262c8 100644 --- a/internal/utils/utils.go +++ b/internal/utils/utils.go @@ -26,6 +26,13 @@ const charset = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" //nolint:unused var seededRand = rand.New(rand.NewSource(time.Now().UnixNano())) +// DownloadRequest struct contains download details ad path and URL +type DownloadRequest struct { + URL string + DownloadPath string + UnsanitizedFilePath string +} + // GetArchitecture retrieves if the underlying system platform is arm64 or amd64 func GetArchitecture() string { arch, present := os.LookupEnv("GOARCH") @@ -40,36 +47,41 @@ func GetArchitecture() string { // DownloadFile will download a url and store it in a temporary path. // It writes to the destination file as it downloads it, without // loading the entire file into memory. -func DownloadFile(url string) (string, error) { - tempParentDir := filepath.Join(os.TempDir(), uuid.NewString()) - internalio.MkdirAll(tempParentDir) +func DownloadFile(downloadRequest *DownloadRequest) error { + var filePath string + if downloadRequest.DownloadPath == "" { + tempParentDir := filepath.Join(os.TempDir(), uuid.NewString()) + internalio.MkdirAll(tempParentDir) + filePath = filepath.Join(tempParentDir, uuid.NewString()) + downloadRequest.DownloadPath = filePath + } else { + filePath = filepath.Join(downloadRequest.DownloadPath, uuid.NewString()) + } - tempFile, err := os.Create(filepath.Join(tempParentDir, uuid.NewString())) + tempFile, err := os.Create(filePath) if err != nil { log.WithFields(log.Fields{ "error": err, - "url": url, + "url": downloadRequest.URL, }).Error("Error creating file") - return "", err + return err } defer tempFile.Close() - filepathFull := tempFile.Name() - + downloadRequest.UnsanitizedFilePath = tempFile.Name() exp := GetExponentialBackOff(3) retryCount := 1 var fileReader io.ReadCloser - download := func() error { - resp, err := http.Get(url) + resp, err := http.Get(downloadRequest.URL) if err != nil { log.WithFields(log.Fields{ "elapsedTime": exp.GetElapsedTime(), "error": err, - "path": filepathFull, + "path": downloadRequest.UnsanitizedFilePath, "retry": retryCount, - "url": url, + "url": downloadRequest.URL, }).Warn("Could not download the file") retryCount++ @@ -80,8 +92,8 @@ func DownloadFile(url string) (string, error) { log.WithFields(log.Fields{ "elapsedTime": exp.GetElapsedTime(), "retries": retryCount, - "path": filepathFull, - "url": url, + "path": downloadRequest.UnsanitizedFilePath, + "url": downloadRequest.URL, }).Trace("File downloaded") fileReader = resp.Body @@ -90,13 +102,13 @@ func DownloadFile(url string) (string, error) { } log.WithFields(log.Fields{ - "url": url, - "path": filepathFull, + "url": downloadRequest.URL, + "path": downloadRequest.UnsanitizedFilePath, }).Trace("Downloading file") err = backoff.Retry(download, exp) if err != nil { - return "", err + return err } defer fileReader.Close() @@ -104,16 +116,16 @@ func DownloadFile(url string) (string, error) { if err != nil { log.WithFields(log.Fields{ "error": err, - "url": url, - "path": filepathFull, + "url": downloadRequest.URL, + "path": downloadRequest.UnsanitizedFilePath, }).Error("Could not write file") - return filepathFull, err + return err } _ = os.Chmod(tempFile.Name(), 0666) - return filepathFull, nil + return nil } // IsCommit returns true if the string matches commit format diff --git a/internal/utils/utils_test.go b/internal/utils/utils_test.go index 82098c5459..44fbc18043 100644 --- a/internal/utils/utils_test.go +++ b/internal/utils/utils_test.go @@ -9,9 +9,14 @@ import ( ) func TestDownloadFile(t *testing.T) { - f, err := DownloadFile("https://www.elastic.co/robots.txt") + var dRequest = DownloadRequest{ + URL: "https://www.elastic.co/robots.txt", + DownloadPath: "", + } + err := DownloadFile(&dRequest) assert.Nil(t, err) - defer os.Remove(filepath.Dir(f)) + assert.NotEmpty(t, dRequest.UnsanitizedFilePath) + defer os.Remove(filepath.Dir(dRequest.UnsanitizedFilePath)) } func TestGetArchitecture(t *testing.T) { diff --git a/internal/_testresources/beats/x-pack/elastic-agent/build/distributions/elastic-agent-BEATS_VERSION-aarch64.rpm b/pkg/_testresources/beats/x-pack/elastic-agent/build/distributions/elastic-agent-BEATS_VERSION-aarch64.rpm similarity index 100% rename from internal/_testresources/beats/x-pack/elastic-agent/build/distributions/elastic-agent-BEATS_VERSION-aarch64.rpm rename to pkg/_testresources/beats/x-pack/elastic-agent/build/distributions/elastic-agent-BEATS_VERSION-aarch64.rpm diff --git a/internal/_testresources/beats/x-pack/elastic-agent/build/distributions/elastic-agent-BEATS_VERSION-amd64.deb b/pkg/_testresources/beats/x-pack/elastic-agent/build/distributions/elastic-agent-BEATS_VERSION-amd64.deb similarity index 100% rename from internal/_testresources/beats/x-pack/elastic-agent/build/distributions/elastic-agent-BEATS_VERSION-amd64.deb rename to pkg/_testresources/beats/x-pack/elastic-agent/build/distributions/elastic-agent-BEATS_VERSION-amd64.deb diff --git a/internal/_testresources/beats/x-pack/elastic-agent/build/distributions/elastic-agent-BEATS_VERSION-arm64.deb b/pkg/_testresources/beats/x-pack/elastic-agent/build/distributions/elastic-agent-BEATS_VERSION-arm64.deb similarity index 100% rename from internal/_testresources/beats/x-pack/elastic-agent/build/distributions/elastic-agent-BEATS_VERSION-arm64.deb rename to pkg/_testresources/beats/x-pack/elastic-agent/build/distributions/elastic-agent-BEATS_VERSION-arm64.deb diff --git a/internal/_testresources/beats/x-pack/elastic-agent/build/distributions/elastic-agent-BEATS_VERSION-linux-amd64.docker.tar.gz b/pkg/_testresources/beats/x-pack/elastic-agent/build/distributions/elastic-agent-BEATS_VERSION-linux-amd64.docker.tar.gz similarity index 100% rename from internal/_testresources/beats/x-pack/elastic-agent/build/distributions/elastic-agent-BEATS_VERSION-linux-amd64.docker.tar.gz rename to pkg/_testresources/beats/x-pack/elastic-agent/build/distributions/elastic-agent-BEATS_VERSION-linux-amd64.docker.tar.gz diff --git a/internal/_testresources/beats/x-pack/elastic-agent/build/distributions/elastic-agent-BEATS_VERSION-linux-amd64.tar.gz b/pkg/_testresources/beats/x-pack/elastic-agent/build/distributions/elastic-agent-BEATS_VERSION-linux-amd64.tar.gz similarity index 100% rename from internal/_testresources/beats/x-pack/elastic-agent/build/distributions/elastic-agent-BEATS_VERSION-linux-amd64.tar.gz rename to pkg/_testresources/beats/x-pack/elastic-agent/build/distributions/elastic-agent-BEATS_VERSION-linux-amd64.tar.gz diff --git a/internal/_testresources/beats/x-pack/elastic-agent/build/distributions/elastic-agent-BEATS_VERSION-linux-arm64.docker.tar.gz b/pkg/_testresources/beats/x-pack/elastic-agent/build/distributions/elastic-agent-BEATS_VERSION-linux-arm64.docker.tar.gz similarity index 100% rename from internal/_testresources/beats/x-pack/elastic-agent/build/distributions/elastic-agent-BEATS_VERSION-linux-arm64.docker.tar.gz rename to pkg/_testresources/beats/x-pack/elastic-agent/build/distributions/elastic-agent-BEATS_VERSION-linux-arm64.docker.tar.gz diff --git a/internal/_testresources/beats/x-pack/elastic-agent/build/distributions/elastic-agent-BEATS_VERSION-linux-arm64.tar.gz b/pkg/_testresources/beats/x-pack/elastic-agent/build/distributions/elastic-agent-BEATS_VERSION-linux-arm64.tar.gz similarity index 100% rename from internal/_testresources/beats/x-pack/elastic-agent/build/distributions/elastic-agent-BEATS_VERSION-linux-arm64.tar.gz rename to pkg/_testresources/beats/x-pack/elastic-agent/build/distributions/elastic-agent-BEATS_VERSION-linux-arm64.tar.gz diff --git a/internal/_testresources/beats/x-pack/elastic-agent/build/distributions/elastic-agent-BEATS_VERSION-linux-x86_64.tar.gz b/pkg/_testresources/beats/x-pack/elastic-agent/build/distributions/elastic-agent-BEATS_VERSION-linux-x86_64.tar.gz similarity index 100% rename from internal/_testresources/beats/x-pack/elastic-agent/build/distributions/elastic-agent-BEATS_VERSION-linux-x86_64.tar.gz rename to pkg/_testresources/beats/x-pack/elastic-agent/build/distributions/elastic-agent-BEATS_VERSION-linux-x86_64.tar.gz diff --git a/internal/_testresources/beats/x-pack/elastic-agent/build/distributions/elastic-agent-BEATS_VERSION-x86_64.rpm b/pkg/_testresources/beats/x-pack/elastic-agent/build/distributions/elastic-agent-BEATS_VERSION-x86_64.rpm similarity index 100% rename from internal/_testresources/beats/x-pack/elastic-agent/build/distributions/elastic-agent-BEATS_VERSION-x86_64.rpm rename to pkg/_testresources/beats/x-pack/elastic-agent/build/distributions/elastic-agent-BEATS_VERSION-x86_64.rpm diff --git a/internal/_testresources/beats/x-pack/elastic-agent/build/distributions/elastic-agent-ubi8-BEATS_VERSION-linux-amd64.docker.tar.gz b/pkg/_testresources/beats/x-pack/elastic-agent/build/distributions/elastic-agent-ubi8-BEATS_VERSION-linux-amd64.docker.tar.gz similarity index 100% rename from internal/_testresources/beats/x-pack/elastic-agent/build/distributions/elastic-agent-ubi8-BEATS_VERSION-linux-amd64.docker.tar.gz rename to pkg/_testresources/beats/x-pack/elastic-agent/build/distributions/elastic-agent-ubi8-BEATS_VERSION-linux-amd64.docker.tar.gz diff --git a/internal/_testresources/beats/x-pack/elastic-agent/build/distributions/elastic-agent-ubi8-BEATS_VERSION-linux-arm64.docker.tar.gz b/pkg/_testresources/beats/x-pack/elastic-agent/build/distributions/elastic-agent-ubi8-BEATS_VERSION-linux-arm64.docker.tar.gz similarity index 100% rename from internal/_testresources/beats/x-pack/elastic-agent/build/distributions/elastic-agent-ubi8-BEATS_VERSION-linux-arm64.docker.tar.gz rename to pkg/_testresources/beats/x-pack/elastic-agent/build/distributions/elastic-agent-ubi8-BEATS_VERSION-linux-arm64.docker.tar.gz diff --git a/internal/_testresources/gcp/commits.json b/pkg/_testresources/gcp/commits.json similarity index 100% rename from internal/_testresources/gcp/commits.json rename to pkg/_testresources/gcp/commits.json diff --git a/internal/_testresources/gcp/nextPageParam.json b/pkg/_testresources/gcp/nextPageParam.json similarity index 100% rename from internal/_testresources/gcp/nextPageParam.json rename to pkg/_testresources/gcp/nextPageParam.json diff --git a/internal/_testresources/gcp/snapshots.json b/pkg/_testresources/gcp/snapshots.json similarity index 100% rename from internal/_testresources/gcp/snapshots.json rename to pkg/_testresources/gcp/snapshots.json diff --git a/pkg/downloads/buckets.go b/pkg/downloads/buckets.go new file mode 100644 index 0000000000..8b94a07914 --- /dev/null +++ b/pkg/downloads/buckets.go @@ -0,0 +1,183 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package downloads + +import ( + "fmt" + "strings" + + log "github.com/sirupsen/logrus" +) + +// BeatsCIArtifactsBase name of the bucket used to store the artifacts generated on Beats CI +const BeatsCIArtifactsBase = "beats-ci-artifacts" + +// FleetCIArtifactsBase name of the bucket used to store the artifacts generated on FLeet CI +const FleetCIArtifactsBase = "fleet-ci-artifacts" + +// BucketURLResolver interface to resolve URL for artifacts in a bucket +type BucketURLResolver interface { + Resolve() (string, string, string) +} + +// BeatsLegacyURLResolver resolver for legacy Beats projects, such as metricbeat, filebeat, etc +// The Elastic Agent must use the project resolver +type BeatsLegacyURLResolver struct { + Bucket string + Beat string + Variant string + FileName string +} + +// NewBeatsLegacyURLResolver creates a new resolver for Beats projects +// The Elastic Agent must use the project resolver +func NewBeatsLegacyURLResolver(beat string, fileName string, variant string) *BeatsLegacyURLResolver { + return &BeatsLegacyURLResolver{ + Bucket: BeatsCIArtifactsBase, + Beat: beat, + FileName: fileName, + Variant: variant, + } +} + +// Resolve returns the bucket, prefix and object for Beats artifacts +func (r *BeatsLegacyURLResolver) Resolve() (string, string, string) { + artifact := r.Beat + fileName := r.FileName + + if strings.EqualFold(r.Variant, "ubi8") { + artifact = strings.ReplaceAll(artifact, "-ubi8", "") + } + + prefix := fmt.Sprintf("snapshots/%s", artifact) + object := fileName + + ciSnapshotsFn := UseBeatsCISnapshots + if strings.EqualFold(artifact, "elastic-agent") { + ciSnapshotsFn = UseElasticAgentCISnapshots + } + + // the commit SHA will identify univocally the artifact in the GCP storage bucket + if ciSnapshotsFn() { + prefix = fmt.Sprintf("commits/%s", GithubCommitSha1) + object = artifact + "/" + fileName + } + + log.WithFields(log.Fields{ + "beat": r.Beat, + "bucket": r.Bucket, + "fileName": r.FileName, + "object": object, + "prefix": prefix, + "variant": r.Variant, + }).Debug("Resolving URL from Beats Legacy resolver") + + return r.Bucket, prefix, object +} + +// BeatsURLResolver resolver for Beats projects, such as metricbeat, filebeat, etc +// The Elastic Agent must use the project resolver +type BeatsURLResolver struct { + Bucket string + Beat string + Variant string + FileName string +} + +// NewBeatsURLResolver creates a new resolver for Beats projects +// The Elastic Agent must use the project resolver +func NewBeatsURLResolver(beat string, fileName string, variant string) *BeatsURLResolver { + return &BeatsURLResolver{ + Bucket: BeatsCIArtifactsBase, + Beat: beat, + FileName: fileName, + Variant: variant, + } +} + +// Resolve returns the bucket, prefix and object for Beats artifacts +func (r *BeatsURLResolver) Resolve() (string, string, string) { + artifact := r.Beat + fileName := r.FileName + + if strings.EqualFold(r.Variant, "ubi8") { + artifact = strings.ReplaceAll(artifact, "-ubi8", "") + } + + prefix := fmt.Sprintf("beats/snapshots/%s", artifact) + object := fileName + + ciSnapshotsFn := UseBeatsCISnapshots + if strings.EqualFold(artifact, "elastic-agent") { + ciSnapshotsFn = UseElasticAgentCISnapshots + } + + // the commit SHA will identify univocally the artifact in the GCP storage bucket + if ciSnapshotsFn() { + prefix = fmt.Sprintf("beats/commits/%s", GithubCommitSha1) + object = artifact + "/" + fileName + } + + log.WithFields(log.Fields{ + "beat": r.Beat, + "bucket": r.Bucket, + "fileName": r.FileName, + "object": object, + "prefix": prefix, + "variant": r.Variant, + }).Debug("Resolving URL from Beats resolver") + + return r.Bucket, prefix, object +} + +// ProjectURLResolver resolver for Elastic projects, such as elastic-agent, fleet-server, etc. +// The Elastic Agent and Fleet Server must use the project resolver +type ProjectURLResolver struct { + Bucket string + Project string + FileName string + Variant string +} + +// NewProjectURLResolver creates a new resolver for Elastic projects +// The Elastic Agent and Fleet Server must use the project resolver +func NewProjectURLResolver(bucketBase string, project string, fileName string, variant string) *ProjectURLResolver { + return &ProjectURLResolver{ + Bucket: bucketBase, + Project: project, + FileName: fileName, + Variant: variant, + } +} + +// Resolve returns the bucket, prefix and object for Elastic artifacts +func (r *ProjectURLResolver) Resolve() (string, string, string) { + artifact := r.Project + + if strings.EqualFold(r.Variant, "ubi8") { + artifact = strings.ReplaceAll(artifact, "-ubi8", "") + } + + prefix := fmt.Sprintf("%s/snapshots", artifact) + + ciSnapshotsFn := UseBeatsCISnapshots + if strings.EqualFold(artifact, "elastic-agent") { + ciSnapshotsFn = UseElasticAgentCISnapshots + } + + // the commit SHA will identify univocally the artifact in the GCP storage bucket + if ciSnapshotsFn() { + prefix = fmt.Sprintf("%s/commits/%s", artifact, GithubCommitSha1) + } + + log.WithFields(log.Fields{ + "bucket": r.Bucket, + "object": r.FileName, + "prefix": prefix, + "project": artifact, + }).Debug("Resolving URL from Project resolver") + + return r.Bucket, prefix, r.FileName +} diff --git a/pkg/downloads/buckets_test.go b/pkg/downloads/buckets_test.go new file mode 100644 index 0000000000..338e2288f3 --- /dev/null +++ b/pkg/downloads/buckets_test.go @@ -0,0 +1,303 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package downloads + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestBeatsLegacyURLResolver(t *testing.T) { + beat := "metricbeat" + defer func() { GithubRepository = "elastic-agent" }() + + t.Run("Fetching snapshots bucket for RPM package", func(t *testing.T) { + resolver := NewBeatsLegacyURLResolver(beat, beat+"-"+testVersion+"-x86_64.rpm", "") + + bucket, prefix, object := resolver.Resolve() + + assert.Equal(t, bucket, "beats-ci-artifacts") + assert.Equal(t, prefix, "snapshots/"+beat) + assert.Equal(t, object, beat+"-"+testVersion+"-x86_64.rpm") + }) + + t.Run("Fetching snapshots bucket for DEB package", func(t *testing.T) { + resolver := NewBeatsLegacyURLResolver(beat, beat+"-"+testVersion+"-amd64.deb", "") + + bucket, prefix, object := resolver.Resolve() + + assert.Equal(t, bucket, "beats-ci-artifacts") + assert.Equal(t, prefix, "snapshots/"+beat) + assert.Equal(t, object, beat+"-"+testVersion+"-amd64.deb") + }) + + t.Run("Fetching snapshots bucket for TAR package adds OS to fileName and object", func(t *testing.T) { + resolver := NewBeatsLegacyURLResolver(beat, beat+"-"+testVersion+"-linux-x86_64.tar.gz", "") + + bucket, prefix, object := resolver.Resolve() + + assert.Equal(t, bucket, "beats-ci-artifacts") + assert.Equal(t, prefix, "snapshots/"+beat) + assert.Equal(t, object, beat+"-"+testVersion+"-linux-x86_64.tar.gz") + }) + + t.Run("Fetching commits bucket for RPM package", func(t *testing.T) { + GithubCommitSha1 = "0123456789" + GithubRepository = "beats" + defer func() { + GithubCommitSha1 = "" + GithubRepository = "" + }() + + resolver := NewBeatsLegacyURLResolver(beat, beat+"-"+testVersion+"-x86_64.rpm", "") + + bucket, prefix, object := resolver.Resolve() + + assert.Equal(t, bucket, "beats-ci-artifacts") + assert.Equal(t, prefix, "commits/0123456789") + assert.Equal(t, object, beat+"/"+beat+"-"+testVersion+"-x86_64.rpm") + }) + + t.Run("Fetching commits bucket for DEB package", func(t *testing.T) { + GithubCommitSha1 = "0123456789" + GithubRepository = "beats" + defer func() { + GithubCommitSha1 = "" + GithubRepository = "" + }() + + resolver := NewBeatsLegacyURLResolver(beat, beat+"-"+testVersion+"-amd64.deb", "") + + bucket, prefix, object := resolver.Resolve() + + assert.Equal(t, bucket, "beats-ci-artifacts") + assert.Equal(t, prefix, "commits/0123456789") + assert.Equal(t, object, beat+"/"+beat+"-"+testVersion+"-amd64.deb") + }) + + t.Run("Fetching commits bucket for TAR package adds OS to fileName and object", func(t *testing.T) { + GithubCommitSha1 = "0123456789" + GithubRepository = "beats" + defer func() { + GithubCommitSha1 = "" + GithubRepository = "" + }() + + resolver := NewBeatsLegacyURLResolver(beat, beat+"-"+testVersion+"-linux-x86_64.tar.gz", "") + + bucket, prefix, object := resolver.Resolve() + + assert.Equal(t, bucket, "beats-ci-artifacts") + assert.Equal(t, prefix, "commits/0123456789") + assert.Equal(t, object, beat+"/"+beat+"-"+testVersion+"-linux-x86_64.tar.gz") + }) + + t.Run("Fetching commits bucket for ubi8 Docker image", func(t *testing.T) { + GithubCommitSha1 = "0123456789" + GithubRepository = "beats" + defer func() { + GithubCommitSha1 = "" + GithubRepository = "" + }() + + resolver := NewBeatsLegacyURLResolver(beat, beat+"-ubi8-"+testVersion+"-linux-x86_64.tar.gz", "ubi8") + + bucket, prefix, object := resolver.Resolve() + + assert.Equal(t, bucket, "beats-ci-artifacts") + assert.Equal(t, prefix, "commits/0123456789") + assert.Equal(t, object, beat+"/"+beat+"-ubi8-"+testVersion+"-linux-x86_64.tar.gz") + }) +} + +func TestBeatsURLResolver(t *testing.T) { + beat := "metricbeat" + defer func() { GithubRepository = "elastic-agent" }() + + t.Run("Fetching snapshots bucket for RPM package", func(t *testing.T) { + resolver := NewBeatsURLResolver(beat, beat+"-"+testVersion+"-x86_64.rpm", "") + + bucket, prefix, object := resolver.Resolve() + + assert.Equal(t, bucket, "beats-ci-artifacts") + assert.Equal(t, prefix, "beats/snapshots/"+beat) + assert.Equal(t, object, beat+"-"+testVersion+"-x86_64.rpm") + }) + + t.Run("Fetching snapshots bucket for DEB package", func(t *testing.T) { + resolver := NewBeatsURLResolver(beat, beat+"-"+testVersion+"-amd64.deb", "") + + bucket, prefix, object := resolver.Resolve() + + assert.Equal(t, bucket, "beats-ci-artifacts") + assert.Equal(t, prefix, "beats/snapshots/"+beat) + assert.Equal(t, object, beat+"-"+testVersion+"-amd64.deb") + }) + + t.Run("Fetching snapshots bucket for TAR package adds OS to fileName and object", func(t *testing.T) { + resolver := NewBeatsURLResolver(beat, beat+"-"+testVersion+"-linux-x86_64.tar.gz", "") + + bucket, prefix, object := resolver.Resolve() + + assert.Equal(t, bucket, "beats-ci-artifacts") + assert.Equal(t, prefix, "beats/snapshots/"+beat) + assert.Equal(t, object, beat+"-"+testVersion+"-linux-x86_64.tar.gz") + }) + + t.Run("Fetching commits bucket for RPM package", func(t *testing.T) { + GithubCommitSha1 = "0123456789" + GithubRepository = "beats" + defer func() { + GithubCommitSha1 = "" + GithubRepository = "" + }() + + resolver := NewBeatsURLResolver(beat, beat+"-"+testVersion+"-x86_64.rpm", "") + + bucket, prefix, object := resolver.Resolve() + + assert.Equal(t, bucket, "beats-ci-artifacts") + assert.Equal(t, prefix, "beats/commits/0123456789") + assert.Equal(t, object, beat+"/"+beat+"-"+testVersion+"-x86_64.rpm") + }) + + t.Run("Fetching commits bucket for DEB package", func(t *testing.T) { + GithubCommitSha1 = "0123456789" + GithubRepository = "beats" + defer func() { + GithubCommitSha1 = "" + GithubRepository = "" + }() + + resolver := NewBeatsURLResolver(beat, beat+"-"+testVersion+"-amd64.deb", "") + + bucket, prefix, object := resolver.Resolve() + + assert.Equal(t, bucket, "beats-ci-artifacts") + assert.Equal(t, prefix, "beats/commits/0123456789") + assert.Equal(t, object, beat+"/"+beat+"-"+testVersion+"-amd64.deb") + }) + + t.Run("Fetching commits bucket for TAR package adds OS to fileName and object", func(t *testing.T) { + GithubCommitSha1 = "0123456789" + GithubRepository = "beats" + defer func() { + GithubCommitSha1 = "" + GithubRepository = "" + }() + + resolver := NewBeatsURLResolver(beat, beat+"-"+testVersion+"-linux-x86_64.tar.gz", "") + + bucket, prefix, object := resolver.Resolve() + + assert.Equal(t, bucket, "beats-ci-artifacts") + assert.Equal(t, prefix, "beats/commits/0123456789") + assert.Equal(t, object, beat+"/"+beat+"-"+testVersion+"-linux-x86_64.tar.gz", "") + }) + + t.Run("Fetching commits bucket for ubi8 Docker image", func(t *testing.T) { + GithubCommitSha1 = "0123456789" + GithubRepository = "beats" + defer func() { + GithubCommitSha1 = "" + GithubRepository = "" + }() + + resolver := NewBeatsURLResolver(beat, beat+"-ubi8-"+testVersion+"-linux-x86_64.tar.gz", "ubi8") + + bucket, prefix, object := resolver.Resolve() + + assert.Equal(t, bucket, "beats-ci-artifacts") + assert.Equal(t, prefix, "beats/commits/0123456789") + assert.Equal(t, object, beat+"/"+beat+"-ubi8-"+testVersion+"-linux-x86_64.tar.gz") + }) +} + +func TestProjectURLResolver(t *testing.T) { + project := "elastic-agent" + + t.Run("Fetching snapshots bucket for RPM package", func(t *testing.T) { + resolver := NewProjectURLResolver(BeatsCIArtifactsBase, project, project+"-"+testVersion+"-x86_64.rpm", "") + + bucket, prefix, object := resolver.Resolve() + + assert.Equal(t, bucket, "beats-ci-artifacts") + assert.Equal(t, prefix, project+"/snapshots") + assert.Equal(t, object, project+"-"+testVersion+"-x86_64.rpm") + }) + + t.Run("Fetching snapshots bucket for DEB package", func(t *testing.T) { + resolver := NewProjectURLResolver(BeatsCIArtifactsBase, project, project+"-"+testVersion+"-amd64.deb", "") + + bucket, prefix, object := resolver.Resolve() + + assert.Equal(t, bucket, "beats-ci-artifacts") + assert.Equal(t, prefix, project+"/snapshots") + assert.Equal(t, object, project+"-"+testVersion+"-amd64.deb") + }) + + t.Run("Fetching snapshots bucket for TAR package adds OS to fileName and object", func(t *testing.T) { + resolver := NewProjectURLResolver(BeatsCIArtifactsBase, project, project+"-"+testVersion+"-linux-x86_64.tar.gz", "") + + bucket, prefix, object := resolver.Resolve() + + assert.Equal(t, bucket, "beats-ci-artifacts") + assert.Equal(t, prefix, project+"/snapshots") + assert.Equal(t, object, project+"-"+testVersion+"-linux-x86_64.tar.gz") + }) + + t.Run("Fetching commits bucket for RPM package", func(t *testing.T) { + GithubCommitSha1 = "0123456789" + defer func() { GithubCommitSha1 = "" }() + + resolver := NewProjectURLResolver(BeatsCIArtifactsBase, project, project+"-"+testVersion+"-x86_64.rpm", "") + + bucket, prefix, object := resolver.Resolve() + + assert.Equal(t, bucket, "beats-ci-artifacts") + assert.Equal(t, prefix, project+"/commits/0123456789") + assert.Equal(t, object, project+"-"+testVersion+"-x86_64.rpm") + }) + + t.Run("Fetching commits bucket for DEB package", func(t *testing.T) { + GithubCommitSha1 = "0123456789" + defer func() { GithubCommitSha1 = "" }() + + resolver := NewProjectURLResolver(BeatsCIArtifactsBase, project, project+"-"+testVersion+"-amd64.deb", "") + + bucket, prefix, object := resolver.Resolve() + + assert.Equal(t, bucket, "beats-ci-artifacts") + assert.Equal(t, prefix, project+"/commits/0123456789") + assert.Equal(t, object, project+"-"+testVersion+"-amd64.deb") + }) + + t.Run("Fetching Elastic Agent commits bucket for TAR package adds OS to fileName and object", func(t *testing.T) { + GithubCommitSha1 = "0123456789" + defer func() { GithubCommitSha1 = "" }() + + resolver := NewProjectURLResolver(BeatsCIArtifactsBase, project, project+"-"+testVersion+"-linux-x86_64.tar.gz", "") + + bucket, prefix, object := resolver.Resolve() + + assert.Equal(t, bucket, "beats-ci-artifacts") + assert.Equal(t, prefix, project+"/commits/0123456789") + assert.Equal(t, object, project+"-"+testVersion+"-linux-x86_64.tar.gz") + }) + + t.Run("Fetching Elastic Agent commits bucket for ubi8 Docker image", func(t *testing.T) { + GithubCommitSha1 = "0123456789" + defer func() { GithubCommitSha1 = "" }() + + resolver := NewProjectURLResolver(BeatsCIArtifactsBase, project, project+"-ubi8-"+testVersion+"-linux-x86_64.tar.gz", "ubi8") + + bucket, prefix, object := resolver.Resolve() + + assert.Equal(t, bucket, "beats-ci-artifacts") + assert.Equal(t, prefix, project+"/commits/0123456789") + assert.Equal(t, object, project+"-ubi8-"+testVersion+"-linux-x86_64.tar.gz") + }) +} diff --git a/internal/versions.go b/pkg/downloads/versions.go similarity index 67% rename from internal/versions.go rename to pkg/downloads/versions.go index f7dad4b7af..43a837a062 100644 --- a/internal/versions.go +++ b/pkg/downloads/versions.go @@ -2,7 +2,7 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. -package internal +package downloads import ( "context" @@ -24,6 +24,10 @@ import ( log "github.com/sirupsen/logrus" ) +// BeatsLocalPath is the path to a local copy of the Beats git repository +// It can be overriden by BEATS_LOCAL_PATH env var. Using the empty string as a default. +var BeatsLocalPath = "" + // to avoid downloading the same artifacts, we are adding this map to cache the URL of the downloaded binaries, using as key // the URL of the artifact. If another installer is trying to download the same URL, it will return the location of the // already downloaded artifact. @@ -37,8 +41,18 @@ var elasticVersionsCache = map[string]string{} // GithubCommitSha1 represents the value of the "GITHUB_CHECK_SHA1" environment variable var GithubCommitSha1 string +// GithubRepository represents the value of the "GITHUB_CHECK_REPO" environment variable +// Default is "elastic-agent" +var GithubRepository string + func init() { GithubCommitSha1 = shell.GetEnv("GITHUB_CHECK_SHA1", "") + GithubRepository = shell.GetEnv("GITHUB_CHECK_REPO", "elastic-agent") + + BeatsLocalPath = shell.GetEnv("BEATS_LOCAL_PATH", BeatsLocalPath) + if BeatsLocalPath != "" { + log.Infof(`Beats local path will be used for artifacts. Please make sure all binaries are properly built in their "build/distributions" folder: %s`, BeatsLocalPath) + } } // elasticVersion represents a version @@ -73,8 +87,15 @@ func CheckPRVersion(version string, fallbackVersion string) string { // FetchElasticArtifact fetches an artifact from the right repository, returning binary name, path and error func FetchElasticArtifact(ctx context.Context, artifact string, version string, os string, arch string, extension string, isDocker bool, xpack bool) (string, string, error) { + useCISnapshots := GithubCommitSha1 != "" + + return FetchElasticArtifactForSnapshots(ctx, useCISnapshots, artifact, version, os, arch, extension, isDocker, xpack) +} + +// FetchElasticArtifactForSnapshots fetches an artifact from the right repository, returning binary name, path and error +func FetchElasticArtifactForSnapshots(ctx context.Context, useCISnapshots bool, artifact string, version string, os string, arch string, extension string, isDocker bool, xpack bool) (string, string, error) { binaryName := buildArtifactName(artifact, version, os, arch, extension, isDocker) - binaryPath, err := fetchBeatsBinary(ctx, binaryName, artifact, version, utils.TimeoutFactor, xpack) + binaryPath, err := FetchProjectBinaryForSnapshots(ctx, useCISnapshots, artifact, binaryName, artifact, version, utils.TimeoutFactor, xpack, "", false) if err != nil { log.WithFields(log.Fields{ "artifact": artifact, @@ -83,7 +104,7 @@ func FetchElasticArtifact(ctx context.Context, artifact string, version string, "arch": arch, "extension": extension, "error": err, - }).Error("Could not download the binary for the agent") + }).Error("Could not download the binary for the Elastic artifact") return "", "", err } @@ -97,10 +118,11 @@ func GetCommitVersion(version string) string { // GetElasticArtifactURL returns the URL of a released artifact, which its full name is defined in the first argument, // from Elastic's artifact repository, building the JSON path query based on the full name +// It also returns the URL of the sha512 file of the released artifact. // i.e. GetElasticArtifactURL("elastic-agent-$VERSION-$ARCH.deb", "elastic-agent", "$VERSION") // i.e. GetElasticArtifactURL("elastic-agent-$VERSION-x86_64.rpm", "elastic-agent","$VERSION") // i.e. GetElasticArtifactURL("elastic-agent-$VERSION-linux-$ARCH.tar.gz", "elastic-agent","$VERSION") -func GetElasticArtifactURL(artifactName string, artifact string, version string) (string, error) { +func GetElasticArtifactURL(artifactName string, artifact string, version string) (string, string, error) { exp := utils.GetExponentialBackOff(time.Minute) retryCount := 1 @@ -152,7 +174,7 @@ func GetElasticArtifactURL(artifactName string, artifact string, version string) err := backoff.Retry(apiStatus, exp) if err != nil { - return "", err + return "", "", err } jsonParsed, err := gabs.ParseJSON([]byte(body)) @@ -162,7 +184,7 @@ func GetElasticArtifactURL(artifactName string, artifact string, version string) "artifactName": artifactName, "version": tmpVersion, }).Error("Could not parse the response body for the artifact") - return "", err + return "", "", err } log.WithFields(log.Fields{ @@ -182,8 +204,9 @@ func GetElasticArtifactURL(artifactName string, artifact string, version string) // we need to get keys with dots using Search instead of Path downloadObject := packagesObject.Search(artifactName) downloadURL := downloadObject.Path("url").Data().(string) + downloadshaURL := downloadObject.Path("sha_url").Data().(string) - return downloadURL, nil + return downloadURL, downloadshaURL, nil } // GetElasticArtifactVersion returns the current version: @@ -302,6 +325,27 @@ func SnapshotHasCommit(s string) bool { return re.MatchString(s) } +// UseBeatsCISnapshots check if CI snapshots should be used for the Beats, where the given SHA commit +// lives in the beats repository +func UseBeatsCISnapshots() bool { + return useCISnapshots("beats") +} + +// UseElasticAgentCISnapshots check if CI snapshots should be used for the Elastic Agent, where the given SHA commit +// lives in the elastic-agent repository +func UseElasticAgentCISnapshots() bool { + return useCISnapshots("elastic-agent") +} + +// useCISnapshots check if CI snapshots should be used, passing a function that evaluates the repository in which +// the given Sha commit has context. I.e. a commit in the elastic-agent repository should pass a function that +func useCISnapshots(repository string) bool { + if GithubCommitSha1 != "" && strings.EqualFold(GithubRepository, repository) { + return true + } + return false +} + // buildArtifactName builds the artifact name from the different coordinates for the artifact func buildArtifactName(artifact string, artifactVersion string, OS string, arch string, extension string, isDocker bool) string { dockerString := "" @@ -321,8 +365,7 @@ func buildArtifactName(artifact string, artifactVersion string, OS string, arch } } - beatsLocalPath := shell.GetEnv("BEATS_LOCAL_PATH", "") - if beatsLocalPath != "" && isDocker { + if BeatsLocalPath != "" && isDocker { dockerString = ".docker" return fmt.Sprintf("%s-%s-%s-%s%s.%s", artifact, artifactVersion, OS, arch, dockerString, lowerCaseExtension) } @@ -339,22 +382,42 @@ func buildArtifactName(artifact string, artifactVersion string, OS string, arch } -// fetchBeatsBinary it downloads the binary and returns the location of the downloaded file +// FetchBeatsBinary it downloads the binary and returns the location of the downloaded file // If the environment variable BEATS_LOCAL_PATH is set, then the artifact // to be used will be defined by the local snapshot produced by the local build. // Else, if the environment variable GITHUB_CHECK_SHA1 is set, then the artifact // to be downloaded will be defined by the snapshot produced by the Beats CI for that commit. -func fetchBeatsBinary(ctx context.Context, artifactName string, artifact string, version string, timeoutFactor int, xpack bool) (string, error) { - beatsLocalPath := shell.GetEnv("BEATS_LOCAL_PATH", "") - if beatsLocalPath != "" { - span, _ := apm.StartSpanOptions(ctx, "Fetching Beats binary", "beats.local.fetch-binary", apm.SpanOptions{ +func FetchBeatsBinary(ctx context.Context, artifactName string, artifact string, version string, timeoutFactor int, xpack bool, downloadPath string, downloadSHAFile bool) (string, error) { + return FetchProjectBinary(ctx, "beats", artifactName, artifact, version, timeoutFactor, xpack, downloadPath, downloadSHAFile) +} + +// FetchProjectBinary it downloads the binary and returns the location of the downloaded file +// If the environment variable BEATS_LOCAL_PATH is set, then the artifact +// to be used will be defined by the local snapshot produced by the local build. +// Else, if the environment variable GITHUB_CHECK_SHA1 is set, then the artifact +// to be downloaded will be defined by the snapshot produced by the Beats CI for that commit. +func FetchProjectBinary(ctx context.Context, project string, artifactName string, artifact string, version string, timeoutFactor int, xpack bool, downloadPath string, downloadSHAFile bool) (string, error) { + useCISnapshots := GithubCommitSha1 != "" + + return FetchProjectBinaryForSnapshots(ctx, useCISnapshots, project, artifactName, artifact, version, timeoutFactor, xpack, downloadPath, downloadSHAFile) +} + +// FetchProjectBinaryForSnapshots it downloads the binary and returns the location of the downloaded file +// If the environment variable BEATS_LOCAL_PATH is set, then the artifact +// to be used will be defined by the local snapshot produced by the local build. +// Else, if the useCISnapshots argument is set to true, then the artifact +// to be downloaded will be defined by the snapshot produced by the Beats CI or Fleet CI for that commit. +func FetchProjectBinaryForSnapshots(ctx context.Context, useCISnapshots bool, project string, artifactName string, artifact string, version string, timeoutFactor int, xpack bool, downloadPath string, downloadSHAFile bool) (string, error) { + if BeatsLocalPath != "" && project == "beats" { + span, _ := apm.StartSpanOptions(ctx, "Fetching Project binary", "project.local.fetch-binary", apm.SpanOptions{ Parent: apm.SpanFromContext(ctx).TraceContext(), }) + span.Context.SetLabel("project", project) defer span.End() - distributions := path.Join(beatsLocalPath, artifact, "build", "distributions") + distributions := path.Join(BeatsLocalPath, artifact, "build", "distributions") if xpack { - distributions = path.Join(beatsLocalPath, "x-pack", artifact, "build", "distributions") + distributions = path.Join(BeatsLocalPath, "x-pack", artifact, "build", "distributions") } log.Debugf("Using local snapshots for the %s: %s", artifact, distributions) @@ -369,9 +432,15 @@ func fetchBeatsBinary(ctx context.Context, artifactName string, artifact string, } handleDownload := func(URL string) (string, error) { - span, _ := apm.StartSpanOptions(ctx, "Fetching Beats binary", "beats.url.fetch-binary", apm.SpanOptions{ + name := artifactName + downloadRequest := utils.DownloadRequest{ + DownloadPath: downloadPath, + URL: URL, + } + span, _ := apm.StartSpanOptions(ctx, "Fetching Project binary", "project.url.fetch-binary", apm.SpanOptions{ Parent: apm.SpanFromContext(ctx).TraceContext(), }) + span.Context.SetLabel("project", project) defer span.End() if val, ok := binariesCache[URL]; ok { @@ -382,20 +451,23 @@ func fetchBeatsBinary(ctx context.Context, artifactName string, artifact string, return val, nil } - filePathFull, err := utils.DownloadFile(URL) + err := utils.DownloadFile(&downloadRequest) if err != nil { - return filePathFull, err + return downloadRequest.UnsanitizedFilePath, err } + if strings.HasSuffix(URL, ".sha512") { + name = fmt.Sprintf("%s.sha512", name) + } // use artifact name as file name to avoid having URL params in the name - sanitizedFilePath := filepath.Join(path.Dir(filePathFull), artifactName) - err = os.Rename(filePathFull, sanitizedFilePath) + sanitizedFilePath := filepath.Join(path.Dir(downloadRequest.UnsanitizedFilePath), name) + err = os.Rename(downloadRequest.UnsanitizedFilePath, sanitizedFilePath) if err != nil { log.WithFields(log.Fields{ - "fileName": filePathFull, + "fileName": downloadRequest.UnsanitizedFilePath, "sanitizedFileName": sanitizedFilePath, }).Warn("Could not sanitize downloaded file name. Keeping old name") - sanitizedFilePath = filePathFull + sanitizedFilePath = downloadRequest.UnsanitizedFilePath } binariesCache[URL] = sanitizedFilePath @@ -403,10 +475,9 @@ func fetchBeatsBinary(ctx context.Context, artifactName string, artifact string, return sanitizedFilePath, nil } - var downloadURL string + var downloadURL, downloadShaURL string var err error - useCISnapshots := GithubCommitSha1 != "" if useCISnapshots { span, _ := apm.StartSpanOptions(ctx, "Fetching Beats binary", "beats.gcp.fetch-binary", apm.SpanOptions{ Parent: apm.SpanFromContext(ctx).TraceContext(), @@ -415,24 +486,63 @@ func fetchBeatsBinary(ctx context.Context, artifactName string, artifact string, log.Debugf("Using CI snapshots for %s", artifact) - bucket, prefix, object := getGCPBucketCoordinates(artifactName, artifact) - maxTimeout := time.Duration(timeoutFactor) * time.Minute - downloadURL, err = getObjectURLFromBucket(bucket, prefix, object, maxTimeout) + variant := "" + if strings.HasSuffix(artifact, "-ubi8") { + variant = "ubi8" + } + + // look up the bucket in this particular order: + // 1. the project layout (elastic-agent, fleet-server) + // 2. the new beats layout (beats) + // 3. the legacy Beats layout (commits/snapshots) + resolvers := []BucketURLResolver{ + NewProjectURLResolver(FleetCIArtifactsBase, project, artifactName, variant), + NewProjectURLResolver(BeatsCIArtifactsBase, project, artifactName, variant), + NewBeatsURLResolver(artifact, artifactName, variant), + NewBeatsLegacyURLResolver(artifact, artifactName, variant), + } + + downloadURL, err = getObjectURLFromResolvers(resolvers, maxTimeout) if err != nil { return "", err } + downloadLocation, err := handleDownload(downloadURL) + + // check if sha file should be downloaded, else return + if !downloadSHAFile { + return downloadLocation, err + } + + sha512ArtifactName := fmt.Sprintf("%s.sha512", artifactName) + sha512Resolvers := []BucketURLResolver{ + NewProjectURLResolver(FleetCIArtifactsBase, project, sha512ArtifactName, variant), + NewProjectURLResolver(BeatsCIArtifactsBase, project, sha512ArtifactName, variant), + NewBeatsURLResolver(artifact, sha512ArtifactName, variant), + NewBeatsLegacyURLResolver(artifact, sha512ArtifactName, variant), + } + + downloadURL, err = getObjectURLFromResolvers(sha512Resolvers, maxTimeout) + if err != nil { + return "", err + } return handleDownload(downloadURL) } - downloadURL, err = GetElasticArtifactURL(artifactName, artifact, version) + downloadURL, downloadShaURL, err = GetElasticArtifactURL(artifactName, artifact, version) if err != nil { return "", err } - - return handleDownload(downloadURL) + downloadLocation, err := handleDownload(downloadURL) + if err != nil { + return "", err + } + if downloadSHAFile && downloadShaURL != "" { + downloadLocation, err = handleDownload(downloadShaURL) + } + return downloadLocation, err } func getBucketSearchNextPageParam(jsonParsed *gabs.Container) string { @@ -445,28 +555,29 @@ func getBucketSearchNextPageParam(jsonParsed *gabs.Container) string { return "&pageToken=" + nextPageToken } -// getGCPBucketCoordinates it calculates the bucket path in GCP -func getGCPBucketCoordinates(fileName string, artifact string) (string, string, string) { - bucket := "beats-ci-artifacts" - - if strings.HasSuffix(artifact, "-ubi8") { - artifact = strings.ReplaceAll(artifact, "-ubi8", "") - } +// getObjectURLFromResolvers extracts the media URL for the desired artifact from the +// Google Cloud Storage bucket used by the CI to push snapshots +func getObjectURLFromResolvers(resolvers []BucketURLResolver, maxtimeout time.Duration) (string, error) { + for i, resolver := range resolvers { + bucket, prefix, object := resolver.Resolve() - prefix := fmt.Sprintf("snapshots/%s", artifact) - object := fileName + downloadURL, err := getObjectURLFromBucket(bucket, prefix, object, maxtimeout) + if err != nil { + if i < len(resolvers)-1 { + log.WithFields(log.Fields{ + "resolver": resolver, + }).Warn("Object not found. Trying with another artifact resolver") + continue + } else { + log.Error("Object not found. There is no other artifact resolver") + return "", err + } + } - // the commit SHA will identify univocally the artifact in the GCP storage bucket - if GithubCommitSha1 != "" { - log.WithFields(log.Fields{ - "commit": GithubCommitSha1, - "file": fileName, - }).Debug("Using CI snapshots for a commit") - prefix = fmt.Sprintf("commits/%s", GithubCommitSha1) - object = artifact + "/" + fileName + return downloadURL, nil } - return bucket, prefix, object + return "", fmt.Errorf("the artifact was not found") } // getObjectURLFromBucket extracts the media URL for the desired artifact from the diff --git a/internal/versions_test.go b/pkg/downloads/versions_test.go similarity index 77% rename from internal/versions_test.go rename to pkg/downloads/versions_test.go index 47300192b9..5d8405720f 100644 --- a/internal/versions_test.go +++ b/pkg/downloads/versions_test.go @@ -2,7 +2,7 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. -package internal +package downloads import ( "context" @@ -22,6 +22,8 @@ var testVersion = "BEATS_VERSION" var ubi8VersionPrefix = artifact + "-ubi8-" + testVersion var versionPrefix = artifact + "-" + testVersion +var testResourcesBasePath = path.Join("..", "_testresources") + const bucket = "beats-ci-artifacts" const commits = "commits" const snapshots = "snapshots" @@ -31,19 +33,19 @@ var commitsJSON *gabs.Container var snapshotsJSON *gabs.Container func init() { - nextTokenParamContent, err := ioutil.ReadFile(path.Join("_testresources", "gcp", "nextPageParam.json")) + nextTokenParamContent, err := ioutil.ReadFile(path.Join(testResourcesBasePath, "gcp", "nextPageParam.json")) if err != nil { os.Exit(1) } nextTokenParamJSON, _ = gabs.ParseJSON([]byte(nextTokenParamContent)) - commitsContent, err := ioutil.ReadFile(path.Join("_testresources", "gcp", "commits.json")) + commitsContent, err := ioutil.ReadFile(path.Join(testResourcesBasePath, "gcp", "commits.json")) if err != nil { os.Exit(1) } commitsJSON, _ = gabs.ParseJSON([]byte(commitsContent)) - snapshotsContent, err := ioutil.ReadFile(path.Join("_testresources", "gcp", "snapshots.json")) + snapshotsContent, err := ioutil.ReadFile(path.Join(testResourcesBasePath, "gcp", "snapshots.json")) if err != nil { os.Exit(1) } @@ -199,8 +201,8 @@ func TestBuildArtifactName(t *testing.T) { }) t.Run("For Docker from local repository (amd64)", func(t *testing.T) { - defer os.Unsetenv("BEATS_LOCAL_PATH") - os.Setenv("BEATS_LOCAL_PATH", "/tmp") + defer func() { BeatsLocalPath = "" }() + BeatsLocalPath = "/tmp" artifact = "elastic-agent" arch := "amd64" @@ -214,8 +216,8 @@ func TestBuildArtifactName(t *testing.T) { assert.Equal(t, expectedFileName, artifactName) }) t.Run("For Docker from local repository (arm64)", func(t *testing.T) { - defer os.Unsetenv("BEATS_LOCAL_PATH") - os.Setenv("BEATS_LOCAL_PATH", "/tmp") + defer func() { BeatsLocalPath = "" }() + BeatsLocalPath = "/tmp" artifact = "elastic-agent" arch := "arm64" @@ -230,8 +232,8 @@ func TestBuildArtifactName(t *testing.T) { }) t.Run("For Docker UBI8 from local repository (amd64)", func(t *testing.T) { - defer os.Unsetenv("BEATS_LOCAL_PATH") - os.Setenv("BEATS_LOCAL_PATH", "/tmp") + defer func() { BeatsLocalPath = "" }() + BeatsLocalPath = "/tmp" artifact = "elastic-agent-ubi8" arch := "amd64" @@ -245,8 +247,8 @@ func TestBuildArtifactName(t *testing.T) { assert.Equal(t, expectedFileName, artifactName) }) t.Run("For Docker UBI8 from local repository (arm64)", func(t *testing.T) { - defer os.Unsetenv("BEATS_LOCAL_PATH") - os.Setenv("BEATS_LOCAL_PATH", "/tmp") + defer func() { BeatsLocalPath = "" }() + BeatsLocalPath = "/tmp" artifact = "elastic-agent-ubi8" arch := "arm64" @@ -375,142 +377,142 @@ func TestCheckPRVersion(t *testing.T) { func TestFetchBeatsBinaryFromLocalPath(t *testing.T) { artifact := "elastic-agent" - beatsDir := path.Join("_testresources", "beats") + beatsDir := path.Join(testResourcesBasePath, "beats") distributionsDir, _ := filepath.Abs(path.Join(beatsDir, "x-pack", "elastic-agent", "build", "distributions")) version := testVersion ctx := context.Background() t.Run("Fetching non-existent binary from local Beats dir throws an error", func(t *testing.T) { - defer os.Unsetenv("BEATS_LOCAL_PATH") - os.Setenv("BEATS_LOCAL_PATH", beatsDir) + defer func() { BeatsLocalPath = "" }() + BeatsLocalPath = beatsDir - _, err := fetchBeatsBinary(ctx, "foo_fileName", artifact, version, utils.TimeoutFactor, true) + _, err := FetchBeatsBinary(ctx, "foo_fileName", artifact, version, utils.TimeoutFactor, true, "", false) assert.NotNil(t, err) }) t.Run("Fetching RPM binary (amd64) from local Beats dir", func(t *testing.T) { - defer os.Unsetenv("BEATS_LOCAL_PATH") - os.Setenv("BEATS_LOCAL_PATH", beatsDir) + defer func() { BeatsLocalPath = "" }() + BeatsLocalPath = beatsDir artifactName := versionPrefix + "-x86_64.rpm" expectedFilePath := path.Join(distributionsDir, artifactName) - downloadedFilePath, err := fetchBeatsBinary(ctx, artifactName, artifact, version, utils.TimeoutFactor, true) + downloadedFilePath, err := FetchBeatsBinary(ctx, artifactName, artifact, version, utils.TimeoutFactor, true, "", false) assert.Nil(t, err) assert.Equal(t, downloadedFilePath, expectedFilePath) }) t.Run("Fetching RPM binary (arm64) from local Beats dir", func(t *testing.T) { - defer os.Unsetenv("BEATS_LOCAL_PATH") - os.Setenv("BEATS_LOCAL_PATH", beatsDir) + defer func() { BeatsLocalPath = "" }() + BeatsLocalPath = beatsDir artifactName := versionPrefix + "-aarch64.rpm" expectedFilePath := path.Join(distributionsDir, artifactName) - downloadedFilePath, err := fetchBeatsBinary(ctx, artifactName, artifact, version, utils.TimeoutFactor, true) + downloadedFilePath, err := FetchBeatsBinary(ctx, artifactName, artifact, version, utils.TimeoutFactor, true, "", false) assert.Nil(t, err) assert.Equal(t, downloadedFilePath, expectedFilePath) }) t.Run("Fetching DEB binary (amd64) from local Beats dir", func(t *testing.T) { - defer os.Unsetenv("BEATS_LOCAL_PATH") - os.Setenv("BEATS_LOCAL_PATH", beatsDir) + defer func() { BeatsLocalPath = "" }() + BeatsLocalPath = beatsDir artifactName := versionPrefix + "-amd64.deb" expectedFilePath := path.Join(distributionsDir, artifactName) - downloadedFilePath, err := fetchBeatsBinary(ctx, artifactName, artifact, version, utils.TimeoutFactor, true) + downloadedFilePath, err := FetchBeatsBinary(ctx, artifactName, artifact, version, utils.TimeoutFactor, true, "", false) assert.Nil(t, err) assert.Equal(t, downloadedFilePath, expectedFilePath) }) t.Run("Fetching DEB binary (arm64) from local Beats dir", func(t *testing.T) { - defer os.Unsetenv("BEATS_LOCAL_PATH") - os.Setenv("BEATS_LOCAL_PATH", beatsDir) + defer func() { BeatsLocalPath = "" }() + BeatsLocalPath = beatsDir artifactName := versionPrefix + "-arm64.deb" expectedFilePath := path.Join(distributionsDir, artifactName) - downloadedFilePath, err := fetchBeatsBinary(ctx, artifactName, artifact, version, utils.TimeoutFactor, true) + downloadedFilePath, err := FetchBeatsBinary(ctx, artifactName, artifact, version, utils.TimeoutFactor, true, "", false) assert.Nil(t, err) assert.Equal(t, downloadedFilePath, expectedFilePath) }) t.Run("Fetching TAR binary (amd64) from local Beats dir", func(t *testing.T) { - defer os.Unsetenv("BEATS_LOCAL_PATH") - os.Setenv("BEATS_LOCAL_PATH", beatsDir) + defer func() { BeatsLocalPath = "" }() + BeatsLocalPath = beatsDir artifactName := versionPrefix + "-linux-amd64.tar.gz" expectedFilePath := path.Join(distributionsDir, artifactName) - downloadedFilePath, err := fetchBeatsBinary(ctx, artifactName, artifact, version, utils.TimeoutFactor, true) + downloadedFilePath, err := FetchBeatsBinary(ctx, artifactName, artifact, version, utils.TimeoutFactor, true, "", false) assert.Nil(t, err) assert.Equal(t, downloadedFilePath, expectedFilePath) }) t.Run("Fetching TAR binary (x86_64) from local Beats dir", func(t *testing.T) { - defer os.Unsetenv("BEATS_LOCAL_PATH") - os.Setenv("BEATS_LOCAL_PATH", beatsDir) + defer func() { BeatsLocalPath = "" }() + BeatsLocalPath = beatsDir artifactName := versionPrefix + "-linux-x86_64.tar.gz" expectedFilePath := path.Join(distributionsDir, artifactName) - downloadedFilePath, err := fetchBeatsBinary(ctx, artifactName, artifact, version, utils.TimeoutFactor, true) + downloadedFilePath, err := FetchBeatsBinary(ctx, artifactName, artifact, version, utils.TimeoutFactor, true, "", false) assert.Nil(t, err) assert.Equal(t, downloadedFilePath, expectedFilePath) }) t.Run("Fetching TAR binary (arm64) from local Beats dir", func(t *testing.T) { - defer os.Unsetenv("BEATS_LOCAL_PATH") - os.Setenv("BEATS_LOCAL_PATH", beatsDir) + defer func() { BeatsLocalPath = "" }() + BeatsLocalPath = beatsDir artifactName := versionPrefix + "-linux-arm64.tar.gz" expectedFilePath := path.Join(distributionsDir, artifactName) - downloadedFilePath, err := fetchBeatsBinary(ctx, artifactName, artifact, version, utils.TimeoutFactor, true) + downloadedFilePath, err := FetchBeatsBinary(ctx, artifactName, artifact, version, utils.TimeoutFactor, true, "", false) assert.Nil(t, err) assert.Equal(t, downloadedFilePath, expectedFilePath) }) t.Run("Fetching Docker binary (amd64) from local Beats dir", func(t *testing.T) { - defer os.Unsetenv("BEATS_LOCAL_PATH") - os.Setenv("BEATS_LOCAL_PATH", beatsDir) + defer func() { BeatsLocalPath = "" }() + BeatsLocalPath = beatsDir artifactName := versionPrefix + "-linux-amd64.docker.tar.gz" expectedFilePath := path.Join(distributionsDir, artifactName) - downloadedFilePath, err := fetchBeatsBinary(ctx, artifactName, artifact, version, utils.TimeoutFactor, true) + downloadedFilePath, err := FetchBeatsBinary(ctx, artifactName, artifact, version, utils.TimeoutFactor, true, "", false) assert.Nil(t, err) assert.Equal(t, downloadedFilePath, expectedFilePath) }) t.Run("Fetching Docker binary (arm64) from local Beats dir", func(t *testing.T) { - defer os.Unsetenv("BEATS_LOCAL_PATH") - os.Setenv("BEATS_LOCAL_PATH", beatsDir) + defer func() { BeatsLocalPath = "" }() + BeatsLocalPath = beatsDir artifactName := versionPrefix + "-linux-arm64.docker.tar.gz" expectedFilePath := path.Join(distributionsDir, artifactName) - downloadedFilePath, err := fetchBeatsBinary(ctx, artifactName, artifact, version, utils.TimeoutFactor, true) + downloadedFilePath, err := FetchBeatsBinary(ctx, artifactName, artifact, version, utils.TimeoutFactor, true, "", false) assert.Nil(t, err) assert.Equal(t, downloadedFilePath, expectedFilePath) }) t.Run("Fetching ubi8 Docker binary (amd64) from local Beats dir", func(t *testing.T) { - defer os.Unsetenv("BEATS_LOCAL_PATH") - os.Setenv("BEATS_LOCAL_PATH", beatsDir) + defer func() { BeatsLocalPath = "" }() + BeatsLocalPath = beatsDir artifactName := ubi8VersionPrefix + "-linux-amd64.docker.tar.gz" expectedFilePath := path.Join(distributionsDir, artifactName) - downloadedFilePath, err := fetchBeatsBinary(ctx, artifactName, artifact, version, utils.TimeoutFactor, true) + downloadedFilePath, err := FetchBeatsBinary(ctx, artifactName, artifact, version, utils.TimeoutFactor, true, "", false) assert.Nil(t, err) assert.Equal(t, downloadedFilePath, expectedFilePath) }) t.Run("Fetching ubi8 Docker binary (arm64) from local Beats dir", func(t *testing.T) { - defer os.Unsetenv("BEATS_LOCAL_PATH") - os.Setenv("BEATS_LOCAL_PATH", beatsDir) + defer func() { BeatsLocalPath = "" }() + BeatsLocalPath = beatsDir artifactName := ubi8VersionPrefix + "-linux-arm64.docker.tar.gz" expectedFilePath := path.Join(distributionsDir, artifactName) - downloadedFilePath, err := fetchBeatsBinary(ctx, artifactName, artifact, version, utils.TimeoutFactor, true) + downloadedFilePath, err := FetchBeatsBinary(ctx, artifactName, artifact, version, utils.TimeoutFactor, true, "", false) assert.Nil(t, err) assert.Equal(t, downloadedFilePath, expectedFilePath) }) @@ -549,89 +551,6 @@ func TestGetBucketSearchNextPageParam_HasNoMorePages(t *testing.T) { assert.True(t, param == "") } -func TestGetGCPBucketCoordinates_Commits(t *testing.T) { - artifact := "elastic-agent" - - t.Run("Fetching commits bucket for RPM package", func(t *testing.T) { - GithubCommitSha1 = "0123456789" - defer func() { GithubCommitSha1 = "" }() - - fileName := "elastic-agent-" + testVersion + "-x86_64.rpm" - - bucket, prefix, object := getGCPBucketCoordinates(fileName, artifact) - assert.Equal(t, bucket, "beats-ci-artifacts") - assert.Equal(t, prefix, "commits/0123456789") - assert.Equal(t, object, "elastic-agent/elastic-agent-"+testVersion+"-x86_64.rpm") - }) - - t.Run("Fetching commits bucket for DEB package", func(t *testing.T) { - GithubCommitSha1 = "0123456789" - defer func() { GithubCommitSha1 = "" }() - - fileName := "elastic-agent-" + testVersion + "-amd64.deb" - - bucket, prefix, object := getGCPBucketCoordinates(fileName, artifact) - assert.Equal(t, bucket, "beats-ci-artifacts") - assert.Equal(t, prefix, "commits/0123456789") - assert.Equal(t, object, "elastic-agent/elastic-agent-"+testVersion+"-amd64.deb") - }) - - t.Run("Fetching commits bucket for TAR package adds OS to fileName and object", func(t *testing.T) { - GithubCommitSha1 = "0123456789" - defer func() { GithubCommitSha1 = "" }() - - fileName := "elastic-agent-" + testVersion + "-linux-x86_64.tar.gz" - - bucket, prefix, object := getGCPBucketCoordinates(fileName, artifact) - assert.Equal(t, bucket, "beats-ci-artifacts") - assert.Equal(t, prefix, "commits/0123456789") - assert.Equal(t, object, "elastic-agent/elastic-agent-"+testVersion+"-linux-x86_64.tar.gz") - }) - - t.Run("Fetching commits bucket for ubi8 Docker image", func(t *testing.T) { - GithubCommitSha1 = "0123456789" - defer func() { GithubCommitSha1 = "" }() - - fileName := "elastic-agent-ubi8-" + testVersion + "-x86_64.tar.gz" - - bucket, prefix, object := getGCPBucketCoordinates(fileName, "elastic-agent-ubi8") - assert.Equal(t, bucket, "beats-ci-artifacts") - assert.Equal(t, prefix, "commits/0123456789") - assert.Equal(t, object, "elastic-agent/elastic-agent-ubi8-"+testVersion+"-x86_64.tar.gz") - }) -} - -func TestGetGCPBucketCoordinates_Snapshots(t *testing.T) { - artifact := "elastic-agent" - - t.Run("Fetching snapshots bucket for RPM package", func(t *testing.T) { - fileName := "elastic-agent-" + testVersion + "-x86_64.rpm" - - bucket, prefix, object := getGCPBucketCoordinates(fileName, artifact) - assert.Equal(t, bucket, "beats-ci-artifacts") - assert.Equal(t, prefix, "snapshots/elastic-agent") - assert.Equal(t, object, "elastic-agent-"+testVersion+"-x86_64.rpm") - }) - - t.Run("Fetching snapshots bucket for DEB package", func(t *testing.T) { - fileName := "elastic-agent-" + testVersion + "-amd64.deb" - - bucket, prefix, object := getGCPBucketCoordinates(fileName, artifact) - assert.Equal(t, bucket, "beats-ci-artifacts") - assert.Equal(t, prefix, "snapshots/elastic-agent") - assert.Equal(t, object, "elastic-agent-"+testVersion+"-amd64.deb") - }) - - t.Run("Fetching snapshots bucket for TAR package adds OS to fileName and object", func(t *testing.T) { - fileName := "elastic-agent-" + testVersion + "-linux-x86_64.tar.gz" - - bucket, prefix, object := getGCPBucketCoordinates(fileName, artifact) - assert.Equal(t, bucket, "beats-ci-artifacts") - assert.Equal(t, prefix, "snapshots/elastic-agent") - assert.Equal(t, object, "elastic-agent-"+testVersion+"-linux-x86_64.tar.gz") - }) -} - func Test_GetCommitVersion(t *testing.T) { t.Run("GetCommitVersion without git commit", func(t *testing.T) { v := GetCommitVersion("1.2.3-SNAPSHOT")