chore: 7.17 backports

.ci/Jenkinsfile

@@ -24,8 +24,8 @@ pipeline {
     RUN_ID = UUID.randomUUID().toString()
   }
   options {
-    timeout(time: 1, unit: 'HOURS')
-    buildDiscarder(logRotator(numToKeepStr: '20', artifactNumToKeepStr: '20', daysToKeepStr: '30'))
+    timeout(time: 90, unit: 'MINUTES')
+    buildDiscarder(logRotator(numToKeepStr: '200', artifactNumToKeepStr: '30', daysToKeepStr: '30'))
     timestamps()
     ansiColor('xterm')
     disableResume()
@@ -48,6 +48,7 @@ pipeline {
     booleanParam(name: "notifyOnGreenBuilds", defaultValue: false, description: "If it's needed to notify to Slack with green builds.")
     string(name: 'SLACK_CHANNEL', defaultValue: 'observablt-bots', description: 'The Slack channel(s) where errors will be posted. For multiple channels, use a comma-separated list of channels')
     string(name: 'ELASTIC_AGENT_DOWNLOAD_URL', defaultValue: '', description: 'If present, it will override the download URL for the Elastic agent artifact. (I.e. https://snapshots.elastic.co/7.12.0-069dfaa4/downloads/beats/elastic-agent/elastic-agent-7.12.0-SNAPSHOT-linux-x86_64.tar.gz')
+    string(name: 'ELASTIC_AGENT_VERSION', defaultValue: '7.17.1-952e96f1-SNAPSHOT', description: 'SemVer version of the Elastic Agent to be used for the tests. You can use here the tag of your PR to test your changes')
     string(name: 'BEAT_VERSION', defaultValue: '7.17.1-952e96f1-SNAPSHOT', description: 'SemVer version of the Beat to be used for the tests. You can use here the tag of your PR to test your changes')
     string(name: 'ELASTIC_AGENT_STALE_VERSION', defaultValue: '7.16-SNAPSHOT', description: 'SemVer version of the stale stand-alone elastic-agent to be used for Fleet upgrade tests.')
     choice(name: 'LOG_LEVEL', choices: ['DEBUG', 'TRACE', 'INFO'], description: 'Log level to be used')
@@ -59,7 +60,7 @@ pipeline {
     string(name: 'KIND_VERSION', defaultValue: '0.10.0', description: 'SemVer version of Kind to be used.')
     string(name: 'KUBERNETES_VERSION', defaultValue: '1.18.2', description: 'SemVer version of Kubernetes to be used.')
     string(name: 'GITHUB_CHECK_NAME', defaultValue: '', description: 'Name of the GitHub check to be updated. Only if this build is triggered from another parent stream.')
-    string(name: 'GITHUB_CHECK_REPO', defaultValue: '', description: 'Name of the GitHub repo to be updated. Only if this build is triggered from another parent stream.')
+    string(name: 'GITHUB_CHECK_REPO', defaultValue: 'elastic-agent', description: 'Name of the GitHub repo to be updated. Only if this build is triggered from another parent stream.')
     string(name: 'GITHUB_CHECK_SHA1', defaultValue: '', description: 'Git SHA for the Beats upstream project (branch or PR)')
   }
   stages {
@@ -74,6 +75,7 @@ pipeline {
         SLACK_CHANNEL = "${params.SLACK_CHANNEL.trim()}"
         ELASTIC_AGENT_DOWNLOAD_URL = "${params.ELASTIC_AGENT_DOWNLOAD_URL.trim()}"
         BEAT_VERSION = "${params.BEAT_VERSION.trim()}"
+        ELASTIC_AGENT_VERSION = "${params.ELASTIC_AGENT_VERSION.trim()}"
         KIBANA_VERSION = "${params.KIBANA_VERSION.trim()}"
         STACK_VERSION = "${params.STACK_VERSION.trim()}"
         FORCE_SKIP_GIT_CHECKS = "${params.forceSkipGitChecks}"
@@ -84,6 +86,8 @@ pipeline {
         KUBERNETES_VERSION = "${params.KUBERNETES_VERSION.trim()}"
         LOG_LEVEL = "${params.LOG_LEVEL.trim()}"
         TIMEOUT_FACTOR = "${params.TIMEOUT_FACTOR.trim()}"
+        GITHUB_CHECK_REPO = "${params.GITHUB_CHECK_REPO.trim()}"
+        GITHUB_CHECK_SHA1 = "${params.GITHUB_CHECK_SHA1.trim()}"
       }
       stages {
         stage('Checkout') {
@@ -507,7 +511,7 @@ def generateFunctionalTestStep(Map args = [:]){
   def envContext = []
   // envContext.add("PROVIDER=${platformProvider}")
   envContext.add("SUITE=${suite}")
-  envContext.add("ELASTIC_APM_GLOBAL_LABELS=branch_name=${BRANCH_NAME},build_pr=${isPR()},build_id=${env.BUILD_ID},go_arch=${goArch},beat_version=${env.BEAT_VERSION},stack_version=${env.STACK_VERSION}")
+  envContext.add("ELASTIC_APM_GLOBAL_LABELS=branch_name=${BRANCH_NAME},build_pr=${isPR()},build_id=${env.BUILD_ID},go_arch=${goArch},beat_version=${env.BEAT_VERSION},elastic_agent_version=${env.ELASTIC_AGENT_VERSION},stack_version=${env.STACK_VERSION}")
 
   def stackRunner = getNodeIp("${env.WORKSPACE}/${env.BASE_DIR}", 'stack')
 
@@ -520,9 +524,11 @@ def generateFunctionalTestStep(Map args = [:]){
         unstash 'sourceEnvModified'
         withEnv(envContext) {
           // This step will help to send the APM traces to the
-          // APM service defined by the Otel Jenkins plugin.
+          // withOtelEnv is the one that uses the APM service defined by the Otel Jenkins plugin.
+          // withAPMEnv uses Vault to prepare the context.
+          // IMPORTANT: withAPMEnv is now the one in used since withOtelEnv uses a specific Opentelemetry Collector at the moment.
           // TODO: This will need to be integrated into the provisioned VMs
-          withOtelEnv() {
+          withAPMEnv() {
             // Start node, capture ip address
             ansible("${env.WORKSPACE}",
                     runId,

.ci/ansible/tasks/setup_test_script.yml

@@ -11,6 +11,9 @@
     - "export PROVIDER=docker"
     - "export DEVELOPER_MODE=true"
     - "export SKIP_PULL=1"
+    - "export STACK_VERSION={{ lookup('env', 'STACK_VERSION') or lookup('file', '{{workspace}}/.stack-version') }}"
+    - "export KIBANA_VERSION={{ lookup('env', 'KIBANA_VERSION') or lookup('file', '{{ workspace }}/.stack-version') }}"
+    - "export ELASTIC_AGENT_VERSION={{ lookup('env', 'ELASTIC_AGENT_VERSION') or lookup('file', '{{ workspace }}/.stack-version') }}"
   when: "'stack' == nodeLabel"
 
 - name: Extend environment for Fleet testing
@@ -57,6 +60,8 @@
   with_items:
     - "export SUITE={{ lookup('env', 'SUITE') or 'kubernetes-autodiscover' }}"
     - "export PROVIDER={{ lookup('env', 'PROVIDER') or 'docker' }}"
+    - "export KIND_VERSION={{ lookup('env', 'KIND_VERSION') }}"
+    - "export KUBERNETES_VERSION={{ lookup('env', 'KUBERNETES_VERSION') }}"
   when:
     - suite is defined
     - suite == "kubernetes-autodiscover"
@@ -71,6 +76,10 @@
   with_items:
     - "export SUITE={{ lookup('env', 'SUITE') or 'helm' }}"
     - "export PROVIDER={{ lookup('env', 'PROVIDER') or 'docker' }}"
+    - "export HELM_CHART_VERSION={{ lookup('env', 'HELM_CHART_VERSION') }}"
+    - "export HELM_VERSION={{ lookup('env', 'HELM_VERSION') }}"
+    - "export KIND_VERSION={{ lookup('env', 'KIND_VERSION') }}"
+    - "export KUBERNETES_VERSION={{ lookup('env', 'KUBERNETES_VERSION') }}"
   when:
     - suite is defined
     - suite == "helm"
@@ -85,6 +94,9 @@
   with_items:
     - "export STACK_VERSION={{ lookup('file', '{{workspace}}/.stack-version') or '8.0.0-SNAPSHOT' }}"
     - "export BEAT_VERSION={{ lookup('file', '{{workspace}}/.stack-version') or '8.0.0-SNAPSHOT' }}"
+    - "export ELASTIC_AGENT_VERSION={{ lookup('file', '{{workspace}}/.stack-version') or '8.0.0-SNAPSHOT' }}"
+    - "export GITHUB_CHECK_SHA1={{ lookup('env', 'GITHUB_CHECK_SHA1') }}"
+    - "export GITHUB_CHECK_REPO={{ lookup('env', 'GITHUB_CHECK_REPO') }}"
     - "export ELASTIC_APM_GLOBAL_LABELS={{ lookup('env', 'ELASTIC_APM_GLOBAL_LABELS') }}"
     - "export PATH=$PATH:/opt/go/{{golang_version}}/bin:/usr/local/bin"
 
@@ -110,4 +122,5 @@
          FORMAT=junit:${REPORT}.xml \
          STACK_VERSION=${STACK_VERSION} \
          BEAT_VERSION=${BEAT_VERSION} \
+         ELASTIC_AGENT_VERSION=${ELASTIC_AGENT_VERSION} \
          make --no-print-directory -C "${BASE_DIR}/e2e/_suites/${SUITE}" functional-test

.ci/bump-stack-version.sh

@@ -36,6 +36,7 @@ git add $FILE
 echo "Update stack with version ${VERSION} in Jenkinsfile"
 FILE="./.ci/Jenkinsfile"
 ${SED} -E -e "s#(name: 'BEAT_VERSION', defaultValue: ')[0-9]+\.[0-9]+\.[0-9]+(-[a-f0-9]{8})?#\1${VERSION}#g" $FILE
+${SED} -E -e "s#(name: 'ELASTIC_AGENT_VERSION', defaultValue: ')[0-9]+\.[0-9]+\.[0-9]+(-[a-f0-9]{8})?#\1${VERSION}#g" $FILE
 ${SED} -E -e "s#(name: 'STACK_VERSION', defaultValue: ')[0-9]+\.[0-9]+\.[0-9]+(-[a-f0-9]{8})?#\1${VERSION}#g" $FILE
 git add $FILE
 

.ci/scripts/functional-test.sh

@@ -14,6 +14,7 @@ set -euxo pipefail
 #   - TAGS - that's the tags to be tested. Default '' which means all of them.
 #   - STACK_VERSION - that's the version of the stack to be tested. Default is stored in '.stack-version'.
 #   - BEAT_VERSION - that's the version of the Beat to be tested. Default is stored in '.stack-version'.
+#   - ELASTIC_AGENT_VERSION - that's the version of the Elastic Agent to be tested. Default is stored in '.stack-version'.
 #
 
 BASE_VERSION="$(cat $(pwd)/.stack-version)"
@@ -22,6 +23,7 @@ SUITE=${1:-''}
 TAGS=${2:-''}
 STACK_VERSION=${3:-"${BASE_VERSION}"}
 BEAT_VERSION=${4:-"${BASE_VERSION}"}
+ELASTIC_AGENT_VERSION=${5:-"${BASE_VERSION}"}
 GOARCH=${GOARCH:-"amd64"}
 
 ## Install the required dependencies for the given SUITE
@@ -32,4 +34,4 @@ mkdir -p outputs
 
 REPORT="$(pwd)/outputs/TEST-${GOARCH}-${SUITE}"
 
-TAGS="${TAGS}" FORMAT=junit:${REPORT}.xml GOARCH=${GOARCH} STACK_VERSION=${STACK_VERSION} BEAT_VERSION=${BEAT_VERSION} make --no-print-directory -C e2e/_suites/${SUITE} functional-test
+TAGS="${TAGS}" FORMAT=junit:${REPORT}.xml GOARCH=${GOARCH} STACK_VERSION=${STACK_VERSION} BEAT_VERSION=${BEAT_VERSION} ELASTIC_AGENT_VERSION=${ELASTIC_AGENT_VERSION} make --no-print-directory -C e2e/_suites/${SUITE} functional-test

Makefile

@@ -37,7 +37,7 @@ notice:
 		-depsOut ""
 
 .PHONY: unit-test
-unit-test: test-report-setup unit-test-dir-cli unit-test-dir-internal
+unit-test: test-report-setup unit-test-dir-cli unit-test-dir-internal unit-test-dir-pkg
 
 # See https://pkg.go.dev/gotest.tools/gotestsum/#readme-junit-xml-output
 .PHONY: unit-test-suite-%

e2e/README.md

@@ -105,7 +105,8 @@ We are going to enumerate the variables that will affect the product versions us
 > Related to this compatibility matrix too, it's also remarkable that Kibana **Fleet** plugin should not allow to enroll an agent with a version higher than kibana (See https://github.com/elastic/kibana/blob/fed9a4fddcc0087ee9eca6582a2a84e001890f08/x-pack/test/fleet_api_integration/apis/agents/enroll.ts#L99).
 
 #### Fleet
-- `BEAT_VERSION`. Set this environment variable to the proper version of the Elastic Agent to be used in the current execution. Default: See https://github.com/elastic/e2e-testing/blob/70b1d3ddaf39567aeb4c322054b93ad7ce53e825/.ci/Jenkinsfile#L44
+- `BEAT_VERSION`. Set this environment variable to the proper version of the Beats to be used in the current execution. Default: See https://github.com/elastic/e2e-testing/blob/70b1d3ddaf39567aeb4c322054b93ad7ce53e825/.ci/Jenkinsfile#L44
+- `ELASTIC_AGENT_VERSION`. Set this environment variable to the proper version of the Elastic Agent to be used in the current execution. Default: See https://github.com/elastic/e2e-testing/blob/70b1d3ddaf39567aeb4c322054b93ad7ce53e825/.ci/Jenkinsfile#L44
 - `ELASTIC_AGENT_DOWNLOAD_URL`. Set this environment variable if you know the bucket URL for an Elastic Agent artifact generated by the CI, i.e. for a pull request. It will take precedence over the `BEAT_VERSION` variable. Default empty: See https://github.com/elastic/e2e-testing/blob/0446248bae1ff604219735998841a21a7576bfdd/.ci/Jenkinsfile#L35
 - `ELASTIC_AGENT_STALE_VERSION`. Set this environment variable to the proper version of the Elastic Agent to be used in the upgrade tests, representing the version to be upgraded. Default: See https://github.com/elastic/e2e-testing/blob/b8d0cb09d575f90f447fe3331b6df0a185c01c89/.ci/Jenkinsfile#L38
 
@@ -117,6 +118,7 @@ We are going to enumerate the variables that will affect the product versions us
 
 #### Kubernetes autodiscover charts
 - `BEAT_VERSION`. Set this environment variable to the proper version of the Beat to be used in the current execution. Default: See https://github.com/elastic/e2e-testing/blob/70b1d3ddaf39567aeb4c322054b93ad7ce53e825/.ci/Jenkinsfile#L44
+- `ELASTIC_AGENT_VERSION`. Set this environment variable to the proper version of the Elastic Agent to be used in the current execution. Default: See https://github.com/elastic/e2e-testing/blob/70b1d3ddaf39567aeb4c322054b93ad7ce53e825/.ci/Jenkinsfile#L44
 - `KIND_VERSION`. Set this environment variable to the proper version of Kind (Kubernetes in Docker) to be used in the current execution. Default: See https://github.com/elastic/e2e-testing/blob/0446248bae1ff604219735998841a21a7576bfdd/.ci/Jenkinsfile#L45
 - `KUBERNETES_VERSION`. Set this environment variable to the proper version of Kubernetes to be used in the current execution. Default: See https://github.com/elastic/e2e-testing/blob/0446248bae1ff604219735998841a21a7576bfdd/.ci/Jenkinsfile#L46
 
@@ -127,7 +129,8 @@ The following environment variables affect how the tests are run in both the CI
 - `ELASTIC_APM_ENVIRONMENT`: Set this environment variable to `ci` to send APM data to Elastic Cloud. Otherwise, the framework will spin up local APM Server and Kibana instances. For the CI, it will read credentials from Vault. Default value: `local`.
 - `SKIP_PULL`: Set this environment variable to prevent the test suite to pull Docker images and/or external dependencies for all components. Default: `false`
 - `BEATS_LOCAL_PATH`: Set this environment variable to the base path to your local clone of Beats if it's needed to use the binary snapshots produced by your local build instead of the official releases. The snapshots will be fetched from the `${BEATS_LOCAL_PATH}/${THE_BEAT}/build/distributions` local directory. This variable is intended to be used by Beats developers, when testing locally the artifacts generated its own build. Default: empty.
-- `GITHUB_CHECK_SHA1`: Set this environment variable to the git commit in the Beats repo to use the binary snapshots produced by Beats CI instead of the official releases. The snapshots will be downloaded from a bucket in Google Cloud Storage. This variable is used by the Beats repository, when testing the artifacts generated by the packaging job. Default: empty.
+- `GITHUB_CHECK_SHA1`: Set this environment variable to the git commit in the right repository to use the binary snapshots produced by the CI instead of the official releases. The snapshots will be downloaded from a bucket in Google Cloud Storage. This variable is used by the repository, when testing the artifacts generated by the packaging job. Default: empty.
+- `GITHUB_CHECK_REPO`: Set this environment variable to the name of the Github repository where the above git SHA commit lives. Default: elastic-agent.
 - `LOG_LEVEL`: Set this environment variable to `TRACE`, `DEBUG`, `INFO`, `WARN`, `ERROR` or `FATAL` to set the log level in the project. Default: `INFO`.
 - `DEVELOPER_MODE`: Set this environment variable to `true` to activate developer mode, which means not destroying the services provisioned by the test framework. Default: `false`.
 - `KIBANA_VERSION`. Set this environment variable to the proper version of the Kibana instance to be used in the current execution, which should be used for the Docker tag of the kibana instance. It will refer to an image related to a Kibana PR, under the Observability-CI namespace. Default is empty
@@ -154,6 +157,7 @@ git checkout main
 TAGS="fleet_mode_agent" \
     TIMEOUT_FACTOR=3 LOG_LEVEL=TRACE \
     BEAT_VERSION="7.10.1" \
+    ELASTIC_AGENT_VERSION="7.10.1" \
     make -C e2e/_suites/fleet functional-test
 ```
 Or running by feature file:
@@ -163,6 +167,7 @@ git checkout main
 FEATURES="fleet_mode_agent.feature" \
     TIMEOUT_FACTOR=3 LOG_LEVEL=TRACE \
     BEAT_VERSION="7.10.1" \
+    ELASTIC_AGENT_VERSION="7.10.1" \
     make -C e2e/_suites/fleet functional-test
 ```
 
@@ -214,8 +219,8 @@ To do so:
 1. In the input parameters form, keep the Beat version (for Fleet) as is, to use each branch's default version.
 1. In the input parameters form, keep the stack version (for Fleet) as is, to use each branch's default version.
 1. In the input parameters form, set the `GITHUB_CHECK_NAME` to `E2E Tests`. This value will appear as the label for the Github check for the E2E tests.
-1. In the input parameters form, set the `GITHUB_CHECK_REPO` to `beats`.
 1. In the input parameters form, set the `GITHUB_CHECK_SHA1` to the `SHA1` of the last commit in your pull request. This value will allow us to modify the mergeable status of that commit with the Github check. Besides that, it will set the specific directory in the GCP bucket to look up the CI binaries.
+1. In the input parameters form, set the `GITHUB_CHECK_REPO` to `elastic-agent` or `beats`, depending where the aforementioned SHA1 belongs.
 1. Click the `Build` button at the bottom of the parameters form.
 
 ## Noticing the test framework

e2e/_suites/fleet/README.md

@@ -88,6 +88,8 @@ This is an example of the optional configuration:
    export ELASTIC_AGENT_DOWNLOAD_URL="https://snapshots.elastic.co/7.12.0-069dfaa4/downloads/beats/elastic-agent/elastic-agent-7.12.0-SNAPSHOT-linux-x86_64.tar.gz"
    # This environment variable will use the its value as the Docker tag produced by Beats CI (Please look up Google Cloud Storage CI bucket).
    export GITHUB_CHECK_SHA1="78a762c76080aafa34c52386341b590dac24e2df"
+   # This environment variable will use the name of the repository where the commit lives
+   export GITHUB_CHECK_REPO="beats"
    ```
 
 3. Define the proper Docker images to be used in tests (Optional).