From 5f9b8a8f51624c5ad0ee665d633d9ee98f86bab2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Wed, 22 Sep 2021 19:02:43 +0200 Subject: [PATCH 1/7] chore: define variable for BeatsLocalPath --- e2e/_suites/fleet/stand-alone.go | 4 +--- .../autodiscover_test.go | 4 +--- internal/deploy/docker.go | 3 +-- internal/versions.go | 19 +++++++++++++------ 4 files changed, 16 insertions(+), 14 deletions(-) diff --git a/e2e/_suites/fleet/stand-alone.go b/e2e/_suites/fleet/stand-alone.go index 6fba00ccfa..f58528a3fa 100644 --- a/e2e/_suites/fleet/stand-alone.go +++ b/e2e/_suites/fleet/stand-alone.go @@ -15,7 +15,6 @@ import ( "github.com/elastic/e2e-testing/internal/deploy" "github.com/elastic/e2e-testing/internal/installer" "github.com/elastic/e2e-testing/internal/kibana" - "github.com/elastic/e2e-testing/internal/shell" "github.com/elastic/e2e-testing/internal/utils" "github.com/elastic/e2e-testing/internal/elasticsearch" @@ -92,9 +91,8 @@ func (fts *FleetTestSuite) startStandAloneAgent(image string, flavour string, en } useCISnapshots := elasticversion.GithubCommitSha1 != "" - beatsLocalPath := shell.GetEnv("BEATS_LOCAL_PATH", "") - if useCISnapshots || beatsLocalPath != "" { + if useCISnapshots || elasticversion.BeatsLocalPath != "" { // load the docker images that were already: // a. downloaded from the GCP bucket // b. fetched from the local beats binaries diff --git a/e2e/_suites/kubernetes-autodiscover/autodiscover_test.go b/e2e/_suites/kubernetes-autodiscover/autodiscover_test.go index d4d0f4e57b..63b0d2d19b 100644 --- a/e2e/_suites/kubernetes-autodiscover/autodiscover_test.go +++ b/e2e/_suites/kubernetes-autodiscover/autodiscover_test.go @@ -32,7 +32,6 @@ import ( "github.com/elastic/e2e-testing/internal/common" "github.com/elastic/e2e-testing/internal/deploy" "github.com/elastic/e2e-testing/internal/kubernetes" - "github.com/elastic/e2e-testing/internal/shell" "github.com/elastic/e2e-testing/internal/utils" ) @@ -136,8 +135,7 @@ func (m *podsManager) configureDockerImage(podName string) error { beatVersion := elasticversion.GetSnapshotVersion(common.BeatVersion) + "-amd64" useCISnapshots := elasticversion.GithubCommitSha1 != "" - beatsLocalPath := shell.GetEnv("BEATS_LOCAL_PATH", "") - if useCISnapshots || beatsLocalPath != "" { + if useCISnapshots || elasticversion.BeatsLocalPath != "" { log.Debugf("Configuring Docker image for %s", podName) _, imagePath, err := elasticversion.FetchElasticArtifact(m.ctx, podName, common.BeatVersion, "linux", "amd64", "tar.gz", true, true) diff --git a/internal/deploy/docker.go b/internal/deploy/docker.go index fb137ec729..91e97cd37f 100644 --- a/internal/deploy/docker.go +++ b/internal/deploy/docker.go @@ -278,9 +278,8 @@ func (c *dockerDeploymentManifest) Stop(ctx context.Context, service ServiceRequ // the images produced by local Beats build, or not. // If an error occurred reading the environment, will return the passed namespace as fallback func GetDockerNamespaceEnvVar(fallback string) string { - beatsLocalPath := shell.GetEnv("BEATS_LOCAL_PATH", "") useCISnapshots := elasticversion.GithubCommitSha1 != "" - if useCISnapshots || beatsLocalPath != "" { + if useCISnapshots || elasticversion.BeatsLocalPath != "" { return "observability-ci" } return fallback diff --git a/internal/versions.go b/internal/versions.go index f7dad4b7af..5611dea0f8 100644 --- a/internal/versions.go +++ b/internal/versions.go @@ -24,6 +24,10 @@ import ( log "github.com/sirupsen/logrus" ) +// BeatsLocalPath is the path to a local copy of the Beats git repository +// It can be overriden by BEATS_LOCAL_PATH env var. Using the empty string as a default. +var BeatsLocalPath = "" + // to avoid downloading the same artifacts, we are adding this map to cache the URL of the downloaded binaries, using as key // the URL of the artifact. If another installer is trying to download the same URL, it will return the location of the // already downloaded artifact. @@ -39,6 +43,11 @@ var GithubCommitSha1 string func init() { GithubCommitSha1 = shell.GetEnv("GITHUB_CHECK_SHA1", "") + + BeatsLocalPath = shell.GetEnv("BEATS_LOCAL_PATH", BeatsLocalPath) + if BeatsLocalPath != "" { + log.Infof(`Beats local path will be used for artifacts. Please make sure all binaries are properly built in their "build/distributions" folder: %s`, BeatsLocalPath) + } } // elasticVersion represents a version @@ -321,8 +330,7 @@ func buildArtifactName(artifact string, artifactVersion string, OS string, arch } } - beatsLocalPath := shell.GetEnv("BEATS_LOCAL_PATH", "") - if beatsLocalPath != "" && isDocker { + if BeatsLocalPath != "" && isDocker { dockerString = ".docker" return fmt.Sprintf("%s-%s-%s-%s%s.%s", artifact, artifactVersion, OS, arch, dockerString, lowerCaseExtension) } @@ -345,16 +353,15 @@ func buildArtifactName(artifact string, artifactVersion string, OS string, arch // Else, if the environment variable GITHUB_CHECK_SHA1 is set, then the artifact // to be downloaded will be defined by the snapshot produced by the Beats CI for that commit. func fetchBeatsBinary(ctx context.Context, artifactName string, artifact string, version string, timeoutFactor int, xpack bool) (string, error) { - beatsLocalPath := shell.GetEnv("BEATS_LOCAL_PATH", "") - if beatsLocalPath != "" { + if BeatsLocalPath != "" { span, _ := apm.StartSpanOptions(ctx, "Fetching Beats binary", "beats.local.fetch-binary", apm.SpanOptions{ Parent: apm.SpanFromContext(ctx).TraceContext(), }) defer span.End() - distributions := path.Join(beatsLocalPath, artifact, "build", "distributions") + distributions := path.Join(BeatsLocalPath, artifact, "build", "distributions") if xpack { - distributions = path.Join(beatsLocalPath, "x-pack", artifact, "build", "distributions") + distributions = path.Join(BeatsLocalPath, "x-pack", artifact, "build", "distributions") } log.Debugf("Using local snapshots for the %s: %s", artifact, distributions) From 70458097e7c79fe9d732df4a61a645f8ab0dd9d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Fri, 1 Oct 2021 08:02:43 +0200 Subject: [PATCH 2/7] chore: add JJB for the new regular pipeline --- .ci/jobs/fleet-server-e2e-tests.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 .ci/jobs/fleet-server-e2e-tests.yml diff --git a/.ci/jobs/fleet-server-e2e-tests.yml b/.ci/jobs/fleet-server-e2e-tests.yml new file mode 100644 index 0000000000..46db114cd7 --- /dev/null +++ b/.ci/jobs/fleet-server-e2e-tests.yml @@ -0,0 +1,18 @@ +--- +- job: + name: e2e-tests/e2e-testing-fleet-server + display-name: Fleet Server e2e tests Pipeline + description: Jenkins pipeline to run the end2end tests for Fleet Server + project-type: pipeline + pipeline-scm: + script-path: .ci/e2eFleetServer.groovy + scm: + - git: + url: git@github.com:elastic/e2e-testing.git + wipe-workspace: 'True' + name: origin + shallow-clone: true + credentials-id: f6c7695a-671e-4f4f-a331-acdce44ff9ba + branches: + - "master" + triggers: [] From ae2eccedccc962807e221d4d1d3f8bb5900f2863 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Fri, 1 Oct 2021 08:24:07 +0200 Subject: [PATCH 3/7] chore: add regular pipeline --- .ci/e2eFleetServer.groovy | 176 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 176 insertions(+) create mode 100644 .ci/e2eFleetServer.groovy diff --git a/.ci/e2eFleetServer.groovy b/.ci/e2eFleetServer.groovy new file mode 100644 index 0000000000..8857a05ddb --- /dev/null +++ b/.ci/e2eFleetServer.groovy @@ -0,0 +1,176 @@ +#!/usr/bin/env groovy + +@Library('apm@current') _ + +pipeline { + agent none + environment { + REPO = 'fleet-server' + ELASTIC_REPO = "elastic/${env.REPO}" + BASE_DIR = "src/github.com/${env.ELASTIC_REPO}" + BEATS_REPO = 'beats' + BEATS_ELASTIC_REPO = "elastic/${env.BEATS_REPO}" + BEATS_BASE_DIR = "src/github.com/${env.BEATS_ELASTIC_REPO}" + E2E_REPO = 'e2e-testing' + E2E_ELASTIC_REPO = "elastic/${env.E2E_REPO}" + E2E_BASE_DIR = "src/github.com/${env.E2E_ELASTIC_REPO}" + DOCKER_REGISTRY = 'docker.elastic.co' + DOCKER_REGISTRY_NAMESPACE = 'observability-ci' + DOCKER_ELASTIC_SECRET = 'secret/observability-team/ci/docker-registry/prod' + GITHUB_APP_SECRET = 'secret/observability-team/ci/github-app' + GITHUB_CHECK_E2E_TESTS_NAME = 'E2E Tests' + JOB_GIT_CREDENTIALS = "2a9602aa-ab9f-4e52-baf3-b71ca88469c7-UserAndToken" + PIPELINE_LOG_LEVEL = "INFO" + AGENT_DROP_PATH = "/tmp/agent-drop-path" + } + options { + timeout(time: 3, unit: 'HOURS') + buildDiscarder(logRotator(numToKeepStr: '20', artifactNumToKeepStr: '20', daysToKeepStr: '30')) + timestamps() + ansiColor('xterm') + disableResume() + durabilityHint('PERFORMANCE_OPTIMIZED') + disableConcurrentBuilds() + } + // http://JENKINS_URL/generic-webhook-trigger/invoke + // Pull requests events: https://docs.github.com/en/developers/webhooks-and-events/github-event-types#pullrequestevent + triggers { + GenericTrigger( + genericVariables: [ + [key: 'GT_REPO', value: '$.repository.full_name'], + [key: 'GT_PR', value: '$.issue.number'], + [key: 'GT_BODY', value: '$.comment.body'], + [key: 'GT_COMMENT_ID', value: '$.comment.id'] + ], + genericHeaderVariables: [ + [key: 'x-github-event', regexpFilter: 'comment'] + ], + causeString: 'Triggered on #$GT_PR via comment: $GT_BODY', + printContributedVariables: false, + printPostContent: false, + silentResponse: true, + regexpFilterText: '$GT_REPO$GT_BODY', + regexpFilterExpression: '^elastic/fleet-server/run-fleet-e2e-tests$' + ) + } + parameters { + string(name: 'fleet_server_pr', defaultValue: "master", description: "PR ID to use to run the E2E tests (e.g 10000)") + } + stages { + stage('Initialize'){ + agent { label 'ubuntu-20 && immutable' } + options { skipDefaultCheckout() } + environment { + HOME = "${env.WORKSPACE}/${BASE_DIR}" + } + stages { + stage('Check permissions') { + steps { + checkPermissions() + setEnvVar('E2E_BASE_BRANCH', getE2EBaseBranch()) + sh(label:'Prepare Agent Drop path', script: 'mkdir -p ${AGENT_DROP_PATH}') + } + } + stage('Process GitHub Event') { + options { skipDefaultCheckout() } + parallel { + stage('Build Fleet Server') { + options { skipDefaultCheckout() } + steps { + gitCheckout(basedir: BASE_DIR, branch: 'master', githubNotifyFirstTimeContributor: true, repo: "git@github.com:${env.ELASTIC_REPO}.git", credentialsId: env.JOB_GIT_CREDENTIALS) + dir("${BASE_DIR}") { + withGoEnv(){ + sh(label: 'Build Fleet Server', script: "make release") + sh(label: 'Copy binaries to Agent Drop path', script: 'cp build/distributions/* ${AGENT_DROP_PATH}') + } + } + } + } + stage('Build Elastic Agent Dependencies') { + options { skipDefaultCheckout() } + steps { + gitCheckout(basedir: BEATS_BASE_DIR, branch: 'master', githubNotifyFirstTimeContributor: true, repo: "git@github.com:${env.BEATS_ELASTIC_REPO}.git", credentialsId: env.JOB_GIT_CREDENTIALS) + dir("${BEATS_BASE_DIR}/x-pack/elastic-agent") { + withGoEnv(){ + withEnv(["DEV=true", "SNAPSHOT=true", "PLATFORMS='+all linux/amd64'"]) { + sh(label: 'Build Fleet Server', script: 'mage package') + } + } + } + dir("${BEATS_BASE_DIR}/x-pack") { + sh(label:'Copy Filebeat binaries to Agent Drop path', script: 'cp filebeat/build/distributions/* ${AGENT_DROP_PATH}') + sh(label:'Copy Heartbeat binaries to Agent Drop path', script: 'cp metricbeat/build/distributions/* ${AGENT_DROP_PATH}') + sh(label:'Copy Metricbeat binaries to Agent Drop path', script: 'cp heartbeat/build/distributions/* ${AGENT_DROP_PATH}') + } + } + } + } + } + stage('Build Elastic Agent including Fleet Server') { + options { skipDefaultCheckout() } + steps { + dir("${BEATS_BASE_DIR}/x-pack/elastic-agent") { + withGoEnv(){ + withEnv(["AGENT_DROP_PATH='${env.AGENT_DROP_PATH}'", "DEV=true", "SNAPSHOT=true", "PLATFORMS='+all linux/amd64'"]) { + sh(label: 'Build Fleet Server', script: 'mage package') + } + } + } + } + } + stage('Run E2E Tests') { + options { skipDefaultCheckout() } + steps { + gitCheckout(basedir: E2E_BASE_DIR, branch: "${env.E2E_BASE_BRANCH}", githubNotifyFirstTimeContributor: true, repo: 'git@github.com:elastic/e2e-testing.git', credentialsId: env.JOB_GIT_CREDENTIALS) + dockerLogin(secret: "${DOCKER_ELASTIC_SECRET}", registry: "${DOCKER_REGISTRY}") + dir("${E2E_BASE_DIR}") { + withGoEnv(){ + withEnv(["BEATS_LOCAL_PATH='${env.BEATS_BASE_DIR}"]) { + sh(label: 'Run E2E Tests', script: './.ci/scripts/fleet-tests.sh ') + } + } + } + } + } + } + } + } +} + +def checkPermissions(){ + if(env.GT_PR){ + if(!githubPrCheckApproved(changeId: "${env.GT_PR}", org: 'elastic', repo: 'kibana')){ + error("Only PRs from Elasticians can be tested with Fleet E2E tests") + } + + if(!hasCommentAuthorWritePermissions(repoName: "${env.ELASTIC_REPO}", commentId: env.GT_COMMENT_ID)){ + error("Only Elasticians can trigger Fleet E2E tests") + } + } +} + +def getE2EBaseBranch() { + // we need a second API request, as the issue_comment API does not retrieve data about the pull request + // See https://docs.github.com/en/developers/webhooks-and-events/webhook-events-and-payloads#issue_comment + def prID = getID() + def token = githubAppToken(secret: "${env.GITHUB_APP_SECRET}") + + def pullRequest = githubApiCall(token: token, url: "https://api.github.com/repos/${env.ELASTIC_REPO}/pulls/${prID}") + def baseRef = pullRequest?.base?.ref + //def headSha = pullRequest?.head?.sha + + if (!baseRef.endsWith('.x')) { + // use maintenance branches mode (i.e. 7.16 translates to 7.16.x) + return baseRef + '.x' + } + + return baseRef +} + +def getID(){ + if(env.GT_PR){ + return "${env.GT_PR}" + } + + return "${params.fleet_server_pr}" +} From 380aa920a070172fe2b17ec8e72ccd91afe2154f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Fri, 1 Oct 2021 10:31:17 +0200 Subject: [PATCH 4/7] fix: update path initialisation in unit tests --- internal/versions_test.go | 64 +++++++++++++++++++-------------------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/internal/versions_test.go b/internal/versions_test.go index 0b44ddaace..5bd66e87f6 100644 --- a/internal/versions_test.go +++ b/internal/versions_test.go @@ -199,8 +199,8 @@ func TestBuildArtifactName(t *testing.T) { }) t.Run("For Docker from local repository (amd64)", func(t *testing.T) { - defer os.Unsetenv("BEATS_LOCAL_PATH") - os.Setenv("BEATS_LOCAL_PATH", "/tmp") + defer func() { BeatsLocalPath = "" }() + BeatsLocalPath = "/tmp" artifact = "elastic-agent" arch := "amd64" @@ -214,8 +214,8 @@ func TestBuildArtifactName(t *testing.T) { assert.Equal(t, expectedFileName, artifactName) }) t.Run("For Docker from local repository (arm64)", func(t *testing.T) { - defer os.Unsetenv("BEATS_LOCAL_PATH") - os.Setenv("BEATS_LOCAL_PATH", "/tmp") + defer func() { BeatsLocalPath = "" }() + BeatsLocalPath = "/tmp" artifact = "elastic-agent" arch := "arm64" @@ -230,8 +230,8 @@ func TestBuildArtifactName(t *testing.T) { }) t.Run("For Docker UBI8 from local repository (amd64)", func(t *testing.T) { - defer os.Unsetenv("BEATS_LOCAL_PATH") - os.Setenv("BEATS_LOCAL_PATH", "/tmp") + defer func() { BeatsLocalPath = "" }() + BeatsLocalPath = "/tmp" artifact = "elastic-agent-ubi8" arch := "amd64" @@ -245,8 +245,8 @@ func TestBuildArtifactName(t *testing.T) { assert.Equal(t, expectedFileName, artifactName) }) t.Run("For Docker UBI8 from local repository (arm64)", func(t *testing.T) { - defer os.Unsetenv("BEATS_LOCAL_PATH") - os.Setenv("BEATS_LOCAL_PATH", "/tmp") + defer func() { BeatsLocalPath = "" }() + BeatsLocalPath = "/tmp" artifact = "elastic-agent-ubi8" arch := "arm64" @@ -382,16 +382,16 @@ func TestFetchBeatsBinaryFromLocalPath(t *testing.T) { ctx := context.Background() t.Run("Fetching non-existent binary from local Beats dir throws an error", func(t *testing.T) { - defer os.Unsetenv("BEATS_LOCAL_PATH") - os.Setenv("BEATS_LOCAL_PATH", beatsDir) + defer func() { BeatsLocalPath = "" }() + BeatsLocalPath = beatsDir _, err := fetchBeatsBinary(ctx, "foo_fileName", artifact, version, utils.TimeoutFactor, true) assert.NotNil(t, err) }) t.Run("Fetching RPM binary (amd64) from local Beats dir", func(t *testing.T) { - defer os.Unsetenv("BEATS_LOCAL_PATH") - os.Setenv("BEATS_LOCAL_PATH", beatsDir) + defer func() { BeatsLocalPath = "" }() + BeatsLocalPath = beatsDir artifactName := versionPrefix + "-x86_64.rpm" expectedFilePath := path.Join(distributionsDir, artifactName) @@ -401,8 +401,8 @@ func TestFetchBeatsBinaryFromLocalPath(t *testing.T) { assert.Equal(t, downloadedFilePath, expectedFilePath) }) t.Run("Fetching RPM binary (arm64) from local Beats dir", func(t *testing.T) { - defer os.Unsetenv("BEATS_LOCAL_PATH") - os.Setenv("BEATS_LOCAL_PATH", beatsDir) + defer func() { BeatsLocalPath = "" }() + BeatsLocalPath = beatsDir artifactName := versionPrefix + "-aarch64.rpm" expectedFilePath := path.Join(distributionsDir, artifactName) @@ -413,8 +413,8 @@ func TestFetchBeatsBinaryFromLocalPath(t *testing.T) { }) t.Run("Fetching DEB binary (amd64) from local Beats dir", func(t *testing.T) { - defer os.Unsetenv("BEATS_LOCAL_PATH") - os.Setenv("BEATS_LOCAL_PATH", beatsDir) + defer func() { BeatsLocalPath = "" }() + BeatsLocalPath = beatsDir artifactName := versionPrefix + "-amd64.deb" expectedFilePath := path.Join(distributionsDir, artifactName) @@ -424,8 +424,8 @@ func TestFetchBeatsBinaryFromLocalPath(t *testing.T) { assert.Equal(t, downloadedFilePath, expectedFilePath) }) t.Run("Fetching DEB binary (arm64) from local Beats dir", func(t *testing.T) { - defer os.Unsetenv("BEATS_LOCAL_PATH") - os.Setenv("BEATS_LOCAL_PATH", beatsDir) + defer func() { BeatsLocalPath = "" }() + BeatsLocalPath = beatsDir artifactName := versionPrefix + "-arm64.deb" expectedFilePath := path.Join(distributionsDir, artifactName) @@ -436,8 +436,8 @@ func TestFetchBeatsBinaryFromLocalPath(t *testing.T) { }) t.Run("Fetching TAR binary (amd64) from local Beats dir", func(t *testing.T) { - defer os.Unsetenv("BEATS_LOCAL_PATH") - os.Setenv("BEATS_LOCAL_PATH", beatsDir) + defer func() { BeatsLocalPath = "" }() + BeatsLocalPath = beatsDir artifactName := versionPrefix + "-linux-amd64.tar.gz" expectedFilePath := path.Join(distributionsDir, artifactName) @@ -447,8 +447,8 @@ func TestFetchBeatsBinaryFromLocalPath(t *testing.T) { assert.Equal(t, downloadedFilePath, expectedFilePath) }) t.Run("Fetching TAR binary (x86_64) from local Beats dir", func(t *testing.T) { - defer os.Unsetenv("BEATS_LOCAL_PATH") - os.Setenv("BEATS_LOCAL_PATH", beatsDir) + defer func() { BeatsLocalPath = "" }() + BeatsLocalPath = beatsDir artifactName := versionPrefix + "-linux-x86_64.tar.gz" expectedFilePath := path.Join(distributionsDir, artifactName) @@ -458,8 +458,8 @@ func TestFetchBeatsBinaryFromLocalPath(t *testing.T) { assert.Equal(t, downloadedFilePath, expectedFilePath) }) t.Run("Fetching TAR binary (arm64) from local Beats dir", func(t *testing.T) { - defer os.Unsetenv("BEATS_LOCAL_PATH") - os.Setenv("BEATS_LOCAL_PATH", beatsDir) + defer func() { BeatsLocalPath = "" }() + BeatsLocalPath = beatsDir artifactName := versionPrefix + "-linux-arm64.tar.gz" expectedFilePath := path.Join(distributionsDir, artifactName) @@ -470,8 +470,8 @@ func TestFetchBeatsBinaryFromLocalPath(t *testing.T) { }) t.Run("Fetching Docker binary (amd64) from local Beats dir", func(t *testing.T) { - defer os.Unsetenv("BEATS_LOCAL_PATH") - os.Setenv("BEATS_LOCAL_PATH", beatsDir) + defer func() { BeatsLocalPath = "" }() + BeatsLocalPath = beatsDir artifactName := versionPrefix + "-linux-amd64.docker.tar.gz" expectedFilePath := path.Join(distributionsDir, artifactName) @@ -481,8 +481,8 @@ func TestFetchBeatsBinaryFromLocalPath(t *testing.T) { assert.Equal(t, downloadedFilePath, expectedFilePath) }) t.Run("Fetching Docker binary (arm64) from local Beats dir", func(t *testing.T) { - defer os.Unsetenv("BEATS_LOCAL_PATH") - os.Setenv("BEATS_LOCAL_PATH", beatsDir) + defer func() { BeatsLocalPath = "" }() + BeatsLocalPath = beatsDir artifactName := versionPrefix + "-linux-arm64.docker.tar.gz" expectedFilePath := path.Join(distributionsDir, artifactName) @@ -493,8 +493,8 @@ func TestFetchBeatsBinaryFromLocalPath(t *testing.T) { }) t.Run("Fetching ubi8 Docker binary (amd64) from local Beats dir", func(t *testing.T) { - defer os.Unsetenv("BEATS_LOCAL_PATH") - os.Setenv("BEATS_LOCAL_PATH", beatsDir) + defer func() { BeatsLocalPath = "" }() + BeatsLocalPath = beatsDir artifactName := ubi8VersionPrefix + "-linux-amd64.docker.tar.gz" expectedFilePath := path.Join(distributionsDir, artifactName) @@ -504,8 +504,8 @@ func TestFetchBeatsBinaryFromLocalPath(t *testing.T) { assert.Equal(t, downloadedFilePath, expectedFilePath) }) t.Run("Fetching ubi8 Docker binary (arm64) from local Beats dir", func(t *testing.T) { - defer os.Unsetenv("BEATS_LOCAL_PATH") - os.Setenv("BEATS_LOCAL_PATH", beatsDir) + defer func() { BeatsLocalPath = "" }() + BeatsLocalPath = beatsDir artifactName := ubi8VersionPrefix + "-linux-arm64.docker.tar.gz" expectedFilePath := path.Join(distributionsDir, artifactName) From aa21e74a517b857b09716d2137ed1b5c12c7b3f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Fri, 1 Oct 2021 10:48:42 +0200 Subject: [PATCH 5/7] chore: rename stage --- .ci/e2eFleetServer.groovy | 2 +- .ci/e2eKibana.groovy | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.ci/e2eFleetServer.groovy b/.ci/e2eFleetServer.groovy index 8857a05ddb..af6208fa5a 100644 --- a/.ci/e2eFleetServer.groovy +++ b/.ci/e2eFleetServer.groovy @@ -71,7 +71,7 @@ pipeline { sh(label:'Prepare Agent Drop path', script: 'mkdir -p ${AGENT_DROP_PATH}') } } - stage('Process GitHub Event') { + stage('Build Elastic Agent dependencies') { options { skipDefaultCheckout() } parallel { stage('Build Fleet Server') { diff --git a/.ci/e2eKibana.groovy b/.ci/e2eKibana.groovy index 17184f9968..8677a6ac23 100644 --- a/.ci/e2eKibana.groovy +++ b/.ci/e2eKibana.groovy @@ -63,7 +63,7 @@ pipeline { setEnvVar('DOCKER_TAG', getDockerTagFromPayload()) } } - stage('Process GitHub Event') { + stage('Build Linux Images') { options { skipDefaultCheckout() } parallel { stage('AMD build') { From 30e85305fb10db0dedb6895052679e5cf27468de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Fri, 1 Oct 2021 11:44:17 +0200 Subject: [PATCH 6/7] fix: detect branches --- .ci/e2eFleetServer.groovy | 26 ++++++++++++++++++++------ .ci/e2eKibana.groovy | 6 ++++++ 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/.ci/e2eFleetServer.groovy b/.ci/e2eFleetServer.groovy index af6208fa5a..b8715230d0 100644 --- a/.ci/e2eFleetServer.groovy +++ b/.ci/e2eFleetServer.groovy @@ -153,18 +153,19 @@ def getE2EBaseBranch() { // we need a second API request, as the issue_comment API does not retrieve data about the pull request // See https://docs.github.com/en/developers/webhooks-and-events/webhook-events-and-payloads#issue_comment def prID = getID() + + if (!prID.isInteger()) { + // in the case we are triggering the job for a branch (i.e master, 7.x) we directly use branch name as Docker tag + return getMaintenanceBranch(prID) + } + def token = githubAppToken(secret: "${env.GITHUB_APP_SECRET}") def pullRequest = githubApiCall(token: token, url: "https://api.github.com/repos/${env.ELASTIC_REPO}/pulls/${prID}") def baseRef = pullRequest?.base?.ref //def headSha = pullRequest?.head?.sha - if (!baseRef.endsWith('.x')) { - // use maintenance branches mode (i.e. 7.16 translates to 7.16.x) - return baseRef + '.x' - } - - return baseRef + return getMaintenanceBranch(baseRef) } def getID(){ @@ -174,3 +175,16 @@ def getID(){ return "${params.fleet_server_pr}" } + +def getMaintenanceBranch(String branch){ + if (branch == 'master' || branch == 'main') { + return branch + } + + if (!branch.endsWith('.x')) { + // use maintenance branches mode (i.e. 7.16 translates to 7.16.x) + return branch + '.x' + } + + return branch +} diff --git a/.ci/e2eKibana.groovy b/.ci/e2eKibana.groovy index 8677a6ac23..9324bd1ba7 100644 --- a/.ci/e2eKibana.groovy +++ b/.ci/e2eKibana.groovy @@ -151,6 +151,12 @@ def getDockerTagFromPayload() { // we need a second API request, as the issue_comment API does not retrieve data about the pull request // See https://docs.github.com/en/developers/webhooks-and-events/webhook-events-and-payloads#issue_comment def prID = getID() + + if (!prID.isInteger()) { + // in the case we are triggering the job for a branch (i.e master, 7.x) we directly use branch name as Docker tag + return prID + } + def token = githubAppToken(secret: "${env.GITHUB_APP_SECRET}") def pullRequest = githubApiCall(token: token, url: "https://api.github.com/repos/${env.ELASTIC_REPO}/pulls/${prID}") From 6a784768ecc85142ec4a34333cfe1709ed3b1d8b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Fri, 1 Oct 2021 13:35:35 +0200 Subject: [PATCH 7/7] chore: pass inline env vars to commands Co-authored-by: Victor Martinez --- .ci/e2eFleetServer.groovy | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/.ci/e2eFleetServer.groovy b/.ci/e2eFleetServer.groovy index b8715230d0..c8278d0066 100644 --- a/.ci/e2eFleetServer.groovy +++ b/.ci/e2eFleetServer.groovy @@ -92,9 +92,7 @@ pipeline { gitCheckout(basedir: BEATS_BASE_DIR, branch: 'master', githubNotifyFirstTimeContributor: true, repo: "git@github.com:${env.BEATS_ELASTIC_REPO}.git", credentialsId: env.JOB_GIT_CREDENTIALS) dir("${BEATS_BASE_DIR}/x-pack/elastic-agent") { withGoEnv(){ - withEnv(["DEV=true", "SNAPSHOT=true", "PLATFORMS='+all linux/amd64'"]) { - sh(label: 'Build Fleet Server', script: 'mage package') - } + sh(label: 'Build Fleet Server', script: 'DEV=true SNAPSHOT=true PLATFORMS="+all linux/amd64" mage package') } } dir("${BEATS_BASE_DIR}/x-pack") { @@ -111,9 +109,7 @@ pipeline { steps { dir("${BEATS_BASE_DIR}/x-pack/elastic-agent") { withGoEnv(){ - withEnv(["AGENT_DROP_PATH='${env.AGENT_DROP_PATH}'", "DEV=true", "SNAPSHOT=true", "PLATFORMS='+all linux/amd64'"]) { - sh(label: 'Build Fleet Server', script: 'mage package') - } + sh(label: 'Build Fleet Server', script: "AGENT_DROP_PATH='${env.AGENT_DROP_PATH}' DEV=true SNAPSHOT=true PLATFORMS='+all linux/amd64' mage package") } } }