openshift · devguyio · Nov 7, 2024 · Nov 7, 2024
diff --git a/ci-operator/config/openshift/hypershift/openshift-hypershift-main.yaml b/ci-operator/config/openshift/hypershift/openshift-hypershift-main.yaml
@@ -62,6 +62,10 @@ releases:
       relative: 1
       stream: ci
       version: "4.17"
+  initial-int:
+    integration:
+      name: "4.18"
+      namespace: ocp
   latest:
     integration:
       include_built_images: true
@@ -127,6 +131,13 @@ tests:
       ENABLE_HYPERSHIFT_CERT_ROTATION_SCALE: "true"
       REQUEST_SERVING_COMPONENT_TEST: "true"
     workflow: hypershift-aws-e2e-external
+- as: e2e-upgrade-tests-aws
+  skip_if_only_changed: (^(\.tekton|docs|examples|enhancements|contrib)/)|(^[A-Z]+\.md$)|((^|/)OWNERS$)
+  steps:
+    cluster_profile: hypershift
+    env:
+      ENABLE_HYPERSHIFT_CERT_ROTATION_SCALE: "true"
+    workflow: hypershift-aws-upgrade-test
 - as: e2e-aws-4-17
   run_if_changed: ^(test/e2e|hypershift-operator)
   steps:

diff --git a/ci-operator/jobs/openshift/hypershift/openshift-hypershift-main-presubmits.yaml b/ci-operator/jobs/openshift/hypershift/openshift-hypershift-main-presubmits.yaml
@@ -1095,6 +1095,79 @@ presubmits:
         secret:
           secretName: result-aggregator
     trigger: (?m)^/test( | .* )e2e-openstack-nfv,?($|\s.*)
+  - agent: kubernetes
+    always_run: false
+    branches:
+    - ^main$
+    - ^main-
+    cluster: build01
+    context: ci/prow/e2e-upgrade-tests-aws
+    decorate: true
+    labels:
+      ci-operator.openshift.io/cloud: hypershift
+      ci-operator.openshift.io/cloud-cluster-profile: hypershift
+      ci.openshift.io/generator: prowgen
+      pj-rehearse.openshift.io/can-be-rehearsed: "true"
+    name: pull-ci-openshift-hypershift-main-e2e-upgrade-tests-aws
+    rerun_command: /test e2e-upgrade-tests-aws
+    skip_if_only_changed: (^(\.tekton|docs|examples|enhancements|contrib)/)|(^[A-Z]+\.md$)|((^|/)OWNERS$)
+    spec:
+      containers:
+      - args:
+        - --gcs-upload-secret=/secrets/gcs/service-account.json
+        - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson
+        - --lease-server-credentials-file=/etc/boskos/credentials
+        - --report-credentials-file=/etc/report/credentials
+        - --secret-dir=/secrets/ci-pull-credentials
+        - --target=e2e-upgrade-tests-aws
+        command:
+        - ci-operator
+        image: ci-operator:latest
+        imagePullPolicy: Always
+        name: ""
+        resources:
+          requests:
+            cpu: 10m
+        volumeMounts:
+        - mountPath: /etc/boskos
+          name: boskos
+          readOnly: true
+        - mountPath: /secrets/ci-pull-credentials
+          name: ci-pull-credentials
+          readOnly: true
+        - mountPath: /secrets/gcs
+          name: gcs-credentials
+          readOnly: true
+        - mountPath: /secrets/manifest-tool
+          name: manifest-tool-local-pusher
+          readOnly: true
+        - mountPath: /etc/pull-secret
+          name: pull-secret
+          readOnly: true
+        - mountPath: /etc/report
+          name: result-aggregator
+          readOnly: true
+      serviceAccountName: ci-operator
+      volumes:
+      - name: boskos
+        secret:
+          items:
+          - key: credentials
+            path: credentials
+          secretName: boskos-credentials
+      - name: ci-pull-credentials
+        secret:
+          secretName: ci-pull-credentials
+      - name: manifest-tool-local-pusher
+        secret:
+          secretName: manifest-tool-local-pusher
+      - name: pull-secret
+        secret:
+          secretName: registry-pull-credentials
+      - name: result-aggregator
+        secret:
+          secretName: result-aggregator
+    trigger: (?m)^/test( | .* )e2e-upgrade-tests-aws,?($|\s.*)
   - agent: kubernetes
     always_run: true
     branches:

diff --git a/ci-operator/step-registry/hypershift/aws/run-e2e/external/upgrade-setup/OWNERS b/ci-operator/step-registry/hypershift/aws/run-e2e/external/upgrade-setup/OWNERS
@@ -0,0 +1 @@
+../OWNERS
diff --git a/.../run-e2e/external/upgrade-setup/hypershift-aws-run-e2e-external-upgrade-setup-commands.sh b/.../run-e2e/external/upgrade-setup/hypershift-aws-run-e2e-external-upgrade-setup-commands.sh
@@ -0,0 +1,51 @@
+#!/bin/bash
+
+set -o nounset
+set -o errexit
+set -o pipefail
+set -o xtrace
+
+function cleanup() {
+  for child in $( jobs -p ); do
+    kill "${child}"
+  done
+  wait
+}
+trap cleanup EXIT
+
+CI_TESTS_RUN="^TestUpgradeHyperShift$"
+
+
+AWS_OBJECT_PARAMS=""
+if grep -q 'e2e.aws-oidc-s3-bucket-name' <<<"$( bin/test-e2e -h 2>&1 )"; then
+  AWS_OBJECT_PARAMS="--e2e.aws-oidc-s3-bucket-name=hypershift-ci-oidc --e2e.aws-kms-key-alias=alias/hypershift-ci"
+fi
+
+AWS_MULTI_ARCH_PARAMS=""
+if [[ "${AWS_MULTI_ARCH:-}" == "true" ]]; then
+  AWS_MULTI_ARCH_PARAMS="--e2e.aws-multi-arch=true"
+fi
+
+# Set --upgrade.run-setup to run the upgrade tests setup
+TEST_SCRIPT_ARGS="-test.parallel=20 \
+  --upgrade.run-setup \
+  --upgrade.setup-artifacts-dir=${SHARED_DIR} \
+  --e2e.aws-credentials-file=/etc/hypershift-pool-aws-credentials/credentials \
+  --e2e.aws-zones=us-east-1a,us-east-1b,us-east-1c \
+  --e2e.pull-secret-file=/etc/ci-pull-credentials/.dockerconfigjson \
+  --e2e.base-domain=ci.hypershift.devcluster.openshift.com \
+  ${AWS_OBJECT_PARAMS:-} \
+  --e2e.additional-tags=\"expirationDate=$(date -d '4 hours' --iso=minutes --utc)\" \
+  --e2e.aws-endpoint-access=PublicAndPrivate \
+  --e2e.external-dns-domain=service.ci.hypershift.devcluster.openshift.com \
+  --e2e.latest-release-image=\"${OCP_IMAGE_LATEST}\" \
+  --e2e.previous-release-image=\"${OCP_IMAGE_PREVIOUS}\" \
+  ${AWS_MULTI_ARCH_PARAMS:-}"
+
+
+export EVENTUALLY_VERBOSE="false"
+
+hack/ci-test-e2e.sh -test.v \
+  -test.run=${CI_TESTS_RUN:-''} \
+  ${TEST_SCRIPT_ARGS} &
+wait $!
diff --git a/...2e/external/upgrade-setup/hypershift-aws-run-e2e-external-upgrade-setup-ref.metadata.json b/...2e/external/upgrade-setup/hypershift-aws-run-e2e-external-upgrade-setup-ref.metadata.json
@@ -0,0 +1,15 @@
+{
+	"path": "hypershift/aws/run-e2e/external/upgrade-setup/hypershift-aws-run-e2e-external-upgrade-setup-ref.yaml",
+	"owners": {
+		"approvers": [
+			"csrwng",
+			"enxebre",
+			"sjenning"
+		],
+		"reviewers": [
+			"csrwng",
+			"enxebre",
+			"sjenning"
+		]
+	}
+}
diff --git a/...aws/run-e2e/external/upgrade-setup/hypershift-aws-run-e2e-external-upgrade-setup-ref.yaml b/...aws/run-e2e/external/upgrade-setup/hypershift-aws-run-e2e-external-upgrade-setup-ref.yaml
@@ -0,0 +1,35 @@
+ref:
+  as: hypershift-aws-run-e2e-external-upgrade-setup
+  cli: latest
+  commands: hypershift-aws-run-e2e-external-upgrade-setup-commands.sh
+  credentials:
+    - mount_path: /etc/hypershift-pool-aws-credentials
+      name: hypershift-pool-aws-credentials
+      namespace: test-credentials
+    - mount_path: /etc/ci-pull-credentials
+      name: ci-pull-credentials
+      namespace: test-credentials
+    - mount_path: /etc/hypershift-kubeconfig
+      name: hypershift-ci-2
+      namespace: test-credentials
+  dependencies:
+    - env: OCP_IMAGE_LATEST
+      name: release:latest
+    - env: OCP_IMAGE_PREVIOUS
+      name: release:initial
+    - env: CI_HYPERSHIFT_OPERATOR
+      name: hypershift-operator
+  env:
+    - default: ${SHARED_DIR}/management_cluster_kubeconfig
+      name: KUBECONFIG
+    - default: ""
+      name: CI_TESTS_RUN
+    - default: "false"
+      name: AWS_MULTI_ARCH
+  from: hypershift-tests
+  grace_period: 30m0s
+  resources:
+    requests:
+      cpu: 100m
+      memory: 100Mi
+  timeout: 2h0m0s
diff --git a/ci-operator/step-registry/hypershift/aws/run-e2e/external/upgrade-test/OWNERS b/ci-operator/step-registry/hypershift/aws/run-e2e/external/upgrade-test/OWNERS
@@ -0,0 +1 @@
+../OWNERS
diff --git a/...ws/run-e2e/external/upgrade-test/hypershift-aws-run-e2e-external-upgrade-test-commands.sh b/...ws/run-e2e/external/upgrade-test/hypershift-aws-run-e2e-external-upgrade-test-commands.sh
@@ -0,0 +1,51 @@
+#!/bin/bash
+
+set -o nounset
+set -o errexit
+set -o pipefail
+set -o xtrace
+
+function cleanup() {
+  for child in $( jobs -p ); do
+    kill "${child}"
+  done
+  wait
+}
+trap cleanup EXIT
+
+CI_TESTS_RUN="^TestUpgradeHyperShift$"
+
+
+AWS_OBJECT_PARAMS=""
+if grep -q 'e2e.aws-oidc-s3-bucket-name' <<<"$( bin/test-e2e -h 2>&1 )"; then
+  AWS_OBJECT_PARAMS="--e2e.aws-oidc-s3-bucket-name=hypershift-ci-oidc --e2e.aws-kms-key-alias=alias/hypershift-ci"
+fi
+
+AWS_MULTI_ARCH_PARAMS=""
+if [[ "${AWS_MULTI_ARCH:-}" == "true" ]]; then
+  AWS_MULTI_ARCH_PARAMS="--e2e.aws-multi-arch=true"
+fi
+
+# Set --upgrade.run-tests to run the upgrade tests
+TEST_SCRIPT_ARGS="-test.parallel=20 \
+  --upgrade.run-tests \
+  --upgrade.setup-artifacts-dir=${SHARED_DIR} \
+  --e2e.aws-credentials-file=/etc/hypershift-pool-aws-credentials/credentials \
+  --e2e.aws-zones=us-east-1a,us-east-1b,us-east-1c \
+  --e2e.pull-secret-file=/etc/ci-pull-credentials/.dockerconfigjson \
+  --e2e.base-domain=ci.hypershift.devcluster.openshift.com \
+  ${AWS_OBJECT_PARAMS:-} \
+  --e2e.additional-tags=\"expirationDate=$(date -d '4 hours' --iso=minutes --utc)\" \
+  --e2e.aws-endpoint-access=PublicAndPrivate \
+  --e2e.external-dns-domain=service.ci.hypershift.devcluster.openshift.com \
+  --e2e.latest-release-image=\"${OCP_IMAGE_LATEST}\" \
+  --e2e.previous-release-image=\"${OCP_IMAGE_PREVIOUS}\" \
+  ${AWS_MULTI_ARCH_PARAMS:-}"
+
+
+export EVENTUALLY_VERBOSE="false"
+
+hack/ci-test-e2e.sh -test.v \
+  -test.run=${CI_TESTS_RUN:-''} \
+  ${TEST_SCRIPT_ARGS} &
+wait $!
diff --git a/...-e2e/external/upgrade-test/hypershift-aws-run-e2e-external-upgrade-test-ref.metadata.json b/...-e2e/external/upgrade-test/hypershift-aws-run-e2e-external-upgrade-test-ref.metadata.json
@@ -0,0 +1,15 @@
+{
+	"path": "hypershift/aws/run-e2e/external/upgrade-test/hypershift-aws-run-e2e-external-upgrade-test-ref.yaml",
+	"owners": {
+		"approvers": [
+			"csrwng",
+			"enxebre",
+			"sjenning"
+		],
+		"reviewers": [
+			"csrwng",
+			"enxebre",
+			"sjenning"
+		]
+	}
+}
diff --git a/...t/aws/run-e2e/external/upgrade-test/hypershift-aws-run-e2e-external-upgrade-test-ref.yaml b/...t/aws/run-e2e/external/upgrade-test/hypershift-aws-run-e2e-external-upgrade-test-ref.yaml
@@ -0,0 +1,35 @@
+ref:
+  as: hypershift-aws-run-e2e-external-upgrade-test
+  cli: latest
+  commands: hypershift-aws-run-e2e-external-upgrade-test-commands.sh
+  credentials:
+    - mount_path: /etc/hypershift-pool-aws-credentials
+      name: hypershift-pool-aws-credentials
+      namespace: test-credentials
+    - mount_path: /etc/ci-pull-credentials
+      name: ci-pull-credentials
+      namespace: test-credentials
+    - mount_path: /etc/hypershift-kubeconfig
+      name: hypershift-ci-2
+      namespace: test-credentials
+  dependencies:
+    - env: OCP_IMAGE_LATEST
+      name: release:latest
+    - env: OCP_IMAGE_PREVIOUS
+      name: release:initial
+    - env: CI_HYPERSHIFT_OPERATOR
+      name: hypershift-operator
+  env:
+    - default: ${SHARED_DIR}/management_cluster_kubeconfig
+      name: KUBECONFIG
+    - default: ""
+      name: CI_TESTS_RUN
+    - default: "false"
+      name: AWS_MULTI_ARCH
+  from: hypershift-tests
+  grace_period: 30m0s
+  resources:
+    requests:
+      cpu: 100m
+      memory: 100Mi
+  timeout: 2h0m0s
diff --git a/ci-operator/step-registry/hypershift/aws/upgrade-test/OWNERS b/ci-operator/step-registry/hypershift/aws/upgrade-test/OWNERS
@@ -0,0 +1 @@
+../OWNERS
diff --git a/...p-registry/hypershift/aws/upgrade-test/hypershift-aws-upgrade-test-workflow.metadata.json b/...p-registry/hypershift/aws/upgrade-test/hypershift-aws-upgrade-test-workflow.metadata.json
@@ -0,0 +1,23 @@
+{
+	"path": "hypershift/aws/upgrade-test/hypershift-aws-upgrade-test-workflow.yaml",
+	"owners": {
+		"approvers": [
+			"csrwng",
+			"enxebre",
+			"sjenning",
+			"imain",
+			"davidvossel",
+			"LiangquanLi930",
+			"Patryk-Stefanski",
+			"bryan-cox"
+		],
+		"reviewers": [
+			"csrwng",
+			"enxebre",
+			"sjenning",
+			"imain",
+			"Patryk-Stefanski",
+			"bryan-cox"
+		]
+	}
+}
diff --git a/...rator/step-registry/hypershift/aws/upgrade-test/hypershift-aws-upgrade-test-workflow.yaml b/...rator/step-registry/hypershift/aws/upgrade-test/hypershift-aws-upgrade-test-workflow.yaml
@@ -0,0 +1,24 @@
+workflow:
+  as: hypershift-aws-upgrade-test
+  documentation: |-
+    The HyperShift upgrade test workflow executes upgrade tests against a 
+    HyperShift management cluster.
+
+    The workflow basically consists of the following steps:
+    - Setup a nested management cluster
+    - Install HyperShift Operator from initial integration stream on the management cluster 
+    - Run upgrade tests setup that prepares the management cluster for the upgrade tests
+    - Upgrade HyperShift Operator to the version under test
+    - Run upgrade tests
+    - Cleanup any resources created during the workflow
+
+    Learn more about HyperShift here: https://github.com/openshift/hypershift
+
+    Track HyperShift's development here: https://issues.redhat.com/projects/HOSTEDCP
+  steps:
+    pre:
+    - chain: hypershift-aws-upgrade-test-setup
+    test:
+    - chain: hypershift-aws-upgrade-test-run
+    post:
+    - chain: hypershift-destroy-nested-management-cluster
diff --git a/ci-operator/step-registry/hypershift/aws/upgrade-test/run/OWNERS b/ci-operator/step-registry/hypershift/aws/upgrade-test/run/OWNERS
@@ -0,0 +1 @@
+../OWNERS