diff --git a/.github/workflows/e2e_aws.yaml b/.github/workflows/e2e_aws.yaml new file mode 100644 index 000000000..8452fb00f --- /dev/null +++ b/.github/workflows/e2e_aws.yaml @@ -0,0 +1,174 @@ +# (C) Copyright Confidential Containers Contributors 2025. +# SPDX-License-Identifier: Apache-2.0 +# +# Run aws e2e tests. +name: (Callable) aws e2e tests + +on: + workflow_call: + inputs: + podvm_image: + required: true + type: string + caa_image: + required: true + type: string + git_ref: + default: 'main' + description: Git ref to checkout the cloud-api-adaptor repository. Defaults to main. + required: false + type: string + oras: + description: Whether the podvm_image is oras published + default: false + required: false + type: boolean + cluster_type: + description: Specify the cluster type. Accepted values are "onprem" or "eks". + default: onprem + required: false + type: string + container_runtime: + default: 'containerd' + description: Name of the container runtime. Either containerd or crio. + required: false + type: string +env: + CLOUD_PROVIDER: aws + DEBIAN_FRONTEND: noninteractive + +jobs: + # Check the org/repository has AWS secrets (AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY). On absence of + # secrets it should skip the execution of the test job. + aws-credentials: + runs-on: ubuntu-22.04 + outputs: + has_secrets: ${{ steps.check_secrets.outputs.has_secrets }} + steps: + - name: Check secrets + id: check_secrets + run: | + if [[ -n "${{ secrets.AWS_ACCESS_KEY_ID }}" && -n "${{ secrets.AWS_SECRET_ACCESS_KEY }}" ]]; then + echo "has_secrets=true" >> "$GITHUB_OUTPUT" + else + echo "has_secrets=false" >> "$GITHUB_OUTPUT" + fi + + test-aws: + needs: aws-credentials + if: needs.aws-credentials.outputs.has_secrets == 'true' + runs-on: ubuntu-22.04 + defaults: + run: + working-directory: src/cloud-api-adaptor + steps: + - name: Checkout Code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ inputs.git_ref }} + + - name: Rebase the code + if: github.event_name == 'pull_request_target' + working-directory: ./ + run: | + ./hack/ci-helper.sh rebase-atop-of-the-latest-target-branch + + - name: Read properties from versions.yaml + run: | + sudo snap install yq + go_version="$(yq '.tools.golang' versions.yaml)" + [ -n "$go_version" ] + echo "GO_VERSION=${go_version}" >> "$GITHUB_ENV" + echo "ORAS_VERSION=$(yq -e '.tools.oras' versions.yaml)" >> "$GITHUB_ENV" + + - name: Setup Golang version ${{ env.GO_VERSION }} + uses: actions/setup-go@v5 + with: + go-version: ${{ env.GO_VERSION }} + + - uses: oras-project/setup-oras@v1 + with: + version: ${{ env.ORAS_VERSION }} + + - name: Extract qcow2 from ${{ inputs.podvm_image }} + if: ${{ !inputs.oras }} + run: | + qcow2=$(echo "${{ inputs.podvm_image }}" | sed -e "s#.*/\(.*\):.*#\1.qcow2#") + ./hack/download-image.sh "${{ inputs.podvm_image }}" . -o "${qcow2}" + echo "PODVM_QCOW2=$(pwd)/${qcow2}" >> "$GITHUB_ENV" + # Clean up docker images to make space + docker system prune -a -f + working-directory: src/cloud-api-adaptor/podvm + + - name: Use oras to get qcow2 from ${{ inputs.podvm_image }} + if: ${{ inputs.oras }} + run: | + oras pull ${{ inputs.podvm_image }} + tar xvJpf podvm.tar.xz + qcow2=$(find ./*.qcow2) + echo "PODVM_QCOW2=$(pwd)/${qcow2}" >> "$GITHUB_ENV" + working-directory: src/cloud-api-adaptor/podvm + + - name: Install kustomize + run: | + command -v kustomize >/dev/null || \ + curl -s "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh" | \ + sudo bash -s /usr/local/bin + + - name: Update kustomization configuration + run: | + cd "install/overlays/aws" + kustomize edit set image "cloud-api-adaptor=${{ inputs.caa_image }}" + # Print for debugging + echo "::group::aws kustomization" + cat kustomization.yaml + echo "::endgroup::" + + - name: Config aws + run: | + cat <>aws.properties + CAA_IMAGE="${{ inputs.caa_image }}" + disablecvm="true" + cluster_type="${{ inputs.cluster_type }}" + ssh_kp_name="caa-e2e-test" + EOF + # For debugging + echo "::group::aws.properties" + cat aws.properties + echo "::endgroup::" + + # Note: aws cli is already installed on github's runner image. + - name: Config aws CLI + run: | + aws configure set aws_access_key_id ${{ secrets.AWS_ACCESS_KEY_ID }} --profile default + aws configure set aws_secret_access_key ${{ secrets.AWS_SECRET_ACCESS_KEY }} --profile default + aws configure set region us-east-1 --profile default + + - name: Create on-prem cluster + if: inputs.cluster_type == 'onprem' + run: | + # Let's use kcli to build a kubeadm cluster for us + echo "::group::Configure libvirt" + ./libvirt/config_libvirt.sh + # Add the kcli install directory to PATH for later steps + echo "${HOME}/.local/bin" >> "$GITHUB_PATH" + echo "::endgroup::" + export CONTAINER_RUNTIME=${{ inputs.container_runtime }} + echo "::group::Create cluster with $CONTAINER_RUNTIME" + ./libvirt/kcli_cluster.sh create + echo "KUBECONFIG=$HOME/.kcli/clusters/peer-pods/auth/kubeconfig" >> "$GITHUB_ENV" + echo "::endgroup::" + + - name: run tests + id: runTests + run: | + export CLOUD_PROVIDER=aws + export DEPLOY_KBS=false + export TEST_PROVISION="yes" + export TEST_TEARDOWN="yes" + export TEST_PROVISION_FILE="$PWD/aws.properties" + export TEST_PODVM_IMAGE="${{ env.PODVM_QCOW2 }}" + export TEST_E2E_TIMEOUT="90m" + + make test-e2e diff --git a/.github/workflows/e2e_run_all.yaml b/.github/workflows/e2e_run_all.yaml index fd06a0a73..fb04d779d 100644 --- a/.github/workflows/e2e_run_all.yaml +++ b/.github/workflows/e2e_run_all.yaml @@ -122,7 +122,7 @@ jobs: outputs: matrix: ${{ steps.matrix.outputs.matrix }} env: - PROVIDERS: "docker libvirt" + PROVIDERS: "aws docker libvirt" steps: - name: Checkout Code uses: actions/checkout@v4 @@ -174,6 +174,28 @@ jobs: id: matrix run: | echo "matrix=$(jq -c . < ./libvirt/e2e_matrix_libvirt.json)" >> "$GITHUB_OUTPUT" + # Run AWS e2e tests + aws: + name: aws + if: | + github.event_name == 'workflow_dispatch' + needs: [podvm, image, prep_install] + strategy: + fail-fast: false + matrix: + os: + - ubuntu + provider: + - generic + arch: + - amd64 + uses: ./.github/workflows/e2e_aws.yaml + with: + caa_image: ${{ inputs.registry }}/cloud-api-adaptor:${{ inputs.caa_image_tag }} + podvm_image: ${{ inputs.registry }}/podvm-${{ matrix.provider }}-${{ matrix.os }}-${{ matrix.arch }}:${{ inputs.podvm_image_tag }} + git_ref: ${{ inputs.git_ref }} + oras: false + secrets: inherit # Run libvirt e2e tests if pull request labeled 'test_e2e_libvirt' libvirt: diff --git a/src/cloud-api-adaptor/go.mod b/src/cloud-api-adaptor/go.mod index 06d110661..6a48cf4af 100644 --- a/src/cloud-api-adaptor/go.mod +++ b/src/cloud-api-adaptor/go.mod @@ -10,10 +10,10 @@ require ( github.com/IBM/go-sdk-core/v5 v5.13.1 github.com/IBM/platform-services-go-sdk v0.36.0 // indirect github.com/IBM/vpc-go-sdk v0.35.0 - github.com/aws/aws-sdk-go-v2 v1.21.0 + github.com/aws/aws-sdk-go-v2 v1.34.0 github.com/aws/aws-sdk-go-v2/config v1.15.11 github.com/aws/aws-sdk-go-v2/credentials v1.12.6 // indirect - github.com/aws/aws-sdk-go-v2/service/ec2 v1.117.0 + github.com/aws/aws-sdk-go-v2/service/ec2 v1.202.0 github.com/containerd/containerd v1.7.16 github.com/containerd/ttrpc v1.2.3 github.com/coreos/go-iptables v0.6.0 @@ -44,8 +44,8 @@ require ( github.com/IBM/ibm-cos-sdk-go v1.9.4 github.com/avast/retry-go/v4 v4.5.1 github.com/aws/aws-sdk-go-v2/service/eks v1.29.5 - github.com/aws/aws-sdk-go-v2/service/iam v1.22.5 - github.com/aws/aws-sdk-go-v2/service/s3 v1.38.5 + github.com/aws/aws-sdk-go-v2/service/iam v1.38.8 + github.com/aws/aws-sdk-go-v2/service/s3 v1.75.0 github.com/confidential-containers/cloud-api-adaptor/src/cloud-providers v0.12.0 github.com/confidential-containers/cloud-api-adaptor/src/peerpod-ctrl v0.12.0 github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f @@ -85,19 +85,19 @@ require ( github.com/Microsoft/go-winio v0.6.1 // indirect github.com/Microsoft/hcsshim v0.12.0-rc.1 // indirect github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect - github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.13 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.8 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.6 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.41 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.29 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.29 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.3.13 // indirect - github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.4 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.14 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.36 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.35 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.4 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.29 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.2 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.5.3 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.10 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.10 // indirect github.com/aws/aws-sdk-go-v2/service/sso v1.11.9 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.16.7 // indirect - github.com/aws/smithy-go v1.17.0 // indirect + github.com/aws/smithy-go v1.22.2 // indirect github.com/containerd/cgroups/v3 v3.0.2 // indirect github.com/containerd/continuity v0.4.2 // indirect github.com/containerd/fifo v1.1.0 // indirect diff --git a/src/cloud-api-adaptor/go.sum b/src/cloud-api-adaptor/go.sum index 05fcd56fe..420741a35 100644 --- a/src/cloud-api-adaptor/go.sum +++ b/src/cloud-api-adaptor/go.sum @@ -82,10 +82,11 @@ github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:W github.com/avast/retry-go/v4 v4.5.1 h1:AxIx0HGi4VZ3I02jr78j5lZ3M6x1E0Ivxa6b0pUUh7o= github.com/avast/retry-go/v4 v4.5.1/go.mod h1:/sipNsvNB3RRuT5iNcb6h73nw3IBmXJ/H3XrCQYSOpc= github.com/aws/aws-sdk-go-v2 v1.16.5/go.mod h1:Wh7MEsmEApyL5hrWzpDkba4gwAPc5/piwLVLFnCxp48= -github.com/aws/aws-sdk-go-v2 v1.21.0 h1:gMT0IW+03wtYJhRqTVYn0wLzwdnK9sRMcxmtfGzRdJc= github.com/aws/aws-sdk-go-v2 v1.21.0/go.mod h1:/RfNgGmRxI+iFOB1OeJUyxiU+9s88k3pfHvDagGEp0M= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.13 h1:OPLEkmhXf6xFPiz0bLeDArZIDx1NNS4oJyG4nv3Gct0= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.13/go.mod h1:gpAbvyDGQFozTEmlTFO8XcQKHzubdq0LzRyJpG6MiXM= +github.com/aws/aws-sdk-go-v2 v1.34.0 h1:9iyL+cjifckRGEVpRKZP3eIxVlL06Qk1Tk13vreaVQU= +github.com/aws/aws-sdk-go-v2 v1.34.0/go.mod h1:JgstGg0JjWU1KpVJjD5H0y0yyAIpSdKEq556EI6yOOM= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.8 h1:zAxi9p3wsZMIaVCdoiQp2uZ9k1LsZvmAnoTBeZPXom0= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.8/go.mod h1:3XkePX5dSaxveLAYY7nsbsZZrKxCyEuE5pM4ziFxyGg= github.com/aws/aws-sdk-go-v2/config v1.15.11 h1:qfec8AtiCqVbwMcx51G1yO2PYVfWfhp2lWkDH65V9HA= github.com/aws/aws-sdk-go-v2/config v1.15.11/go.mod h1:mD5tNFciV7YHNjPpFYqJ6KGpoSfY107oZULvTHIxtbI= github.com/aws/aws-sdk-go-v2/credentials v1.12.6 h1:No1wZFW4bcM/uF6Tzzj6IbaeQJM+xxqXOYmoObm33ws= @@ -93,40 +94,42 @@ github.com/aws/aws-sdk-go-v2/credentials v1.12.6/go.mod h1:mQgnRmBPF2S/M01W4T4Ob github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.6 h1:+NZzDh/RpcQTpo9xMFUgkseIam6PC+YJbdhbQp1NOXI= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.6/go.mod h1:ClLMcuQA/wcHPmOIfNzNI4Y1Q0oDbmEkbYhMFOzHDh8= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.12/go.mod h1:Afj/U8svX6sJ77Q+FPWMzabJ9QjbwP32YlopgKALUpg= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.41 h1:22dGT7PneFMx4+b3pz7lMTRyN8ZKH7M2cW4GP9yUS2g= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.41/go.mod h1:CrObHAuPneJBlfEJ5T3szXOUkLEThaGfvnhTf33buas= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.29 h1:Ej0Rf3GMv50Qh4G4852j2djtoDb7AzQ7MuQeFHa3D70= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.29/go.mod h1:oeNTC7PwJNoM5AznVr23wxhLnuJv0ZDe5v7w0wqIs9M= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.6/go.mod h1:FwpAKI+FBPIELJIdmQzlLtRe8LQSOreMcM2wBsPMvvc= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35 h1:SijA0mgjV8E+8G45ltVHs0fvKpTj8xmZJ3VwhGKtUSI= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35/go.mod h1:SJC1nEVVva1g3pHAIdCp7QsRIkMmLAgoDquQ9Rr8kYw= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.29 h1:6e8a71X+9GfghragVevC5bZqvATtc3mAMgxpSNbgzF0= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.29/go.mod h1:c4jkZiQ+BWpNqq7VtrxjwISrLrt/VvPq3XiopkUIolI= github.com/aws/aws-sdk-go-v2/internal/ini v1.3.13 h1:L/l0WbIpIadRO7i44jZh1/XeXpNDX0sokFppb4ZnXUI= github.com/aws/aws-sdk-go-v2/internal/ini v1.3.13/go.mod h1:hiM/y1XPp3DoEPhoVEYc/CZcS58dP6RKJRDFp99wdX0= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.4 h1:6lJvvkQ9HmbHZ4h/IEwclwv2mrTW8Uq1SOB/kXy0mfw= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.4/go.mod h1:1PrKYwxTM+zjpw9Y41KFtoJCQrJ34Z47Y4VgVbfndjo= -github.com/aws/aws-sdk-go-v2/service/ec2 v1.117.0 h1:Yq39vbwQX+Xw+Ubcsg/ElwO+TWAxAIAdrREtpjGnCHw= -github.com/aws/aws-sdk-go-v2/service/ec2 v1.117.0/go.mod h1:0FhI2Rzcv5BNM3dNnbcCx2qa2naFZoAidJi11cQgzL0= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.29 h1:g9OUETuxA8i/Www5Cby0R3WSTe7ppFTZXHVLNskNS4w= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.29/go.mod h1:CQk+koLR1QeY1+vm7lqNfFii07DEderKq6T3F1L2pyc= +github.com/aws/aws-sdk-go-v2/service/ec2 v1.202.0 h1:/kB9Uf7fgpYNLvwhAW0YiDSg7xQyxB6MbEYoC0yXtjs= +github.com/aws/aws-sdk-go-v2/service/ec2 v1.202.0/go.mod h1:cRD0Fhzj0YD+uAh16NChQAv9/BB0S9x3YK9hLx1jb/k= github.com/aws/aws-sdk-go-v2/service/eks v1.29.5 h1:6eSpTHOsDixcFIvPdiAAVdyCru3k2jIVRPdIQfGzfc8= github.com/aws/aws-sdk-go-v2/service/eks v1.29.5/go.mod h1:TwqefcyPlF31NTF+fH34tJ2VwMMR6c74IbiiUgA6kVY= -github.com/aws/aws-sdk-go-v2/service/iam v1.22.5 h1:qGv+oW4uV1T3kbE9uSYEfdZbo38OqxgRxxfStfDr4BU= -github.com/aws/aws-sdk-go-v2/service/iam v1.22.5/go.mod h1:8lyPrjQczmx72ac9s82zTjf9xLqs7uuFMG9TVEZ07XU= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.14 h1:m0QTSI6pZYJTk5WSKx3fm5cNW/DCicVzULBgU/6IyD0= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.14/go.mod h1:dDilntgHy9WnHXsh7dDtUPgHKEfTJIBUTHM8OWm0f/0= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.36 h1:eev2yZX7esGRjqRbnVk1UxMLw4CyVZDpZXRCcy75oQk= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.36/go.mod h1:lGnOkH9NJATw0XEPcAknFBj3zzNTEGRHtSw+CwC1YTg= +github.com/aws/aws-sdk-go-v2/service/iam v1.38.8 h1:+PjS9gfr15U+MaUafN89dWxhbsvVrJg2D1umkc8R4uA= +github.com/aws/aws-sdk-go-v2/service/iam v1.38.8/go.mod h1:V7xF4f2fgf9GSVxTqeYQz7bNu8AITVsgqP6otlHzjPs= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.2 h1:D4oz8/CzT9bAEYtVhSBmFj2dNOtaHOtMKc2vHBwYizA= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.2/go.mod h1:Za3IHqTQ+yNcRHxu1OFucBh0ACZT4j4VQFF0BqpZcLY= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.5.3 h1:EP1ITDgYVPM2dL1bBBntJ7AW5yTjuWGz9XO+CZwpALU= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.5.3/go.mod h1:5lWNWeAgWenJ/BZ/CP9k9DjLbC0pjnM045WjXRPPi14= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.6/go.mod h1:DxAPjquoEHf3rUHh1b9+47RAaXB8/7cB6jkzCt/GOEI= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.35 h1:CdzPW9kKitgIiLV1+MHobfR5Xg25iYnyzWZhyQuSlDI= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.35/go.mod h1:QGF2Rs33W5MaN9gYdEQOBBFPLwTZkEhRwI33f7KIG0o= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.4 h1:v0jkRigbSD6uOdwcaUQmgEwG1BkPfAPDqaeNt/29ghg= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.4/go.mod h1:LhTyt8J04LL+9cIt7pYJ5lbS/U98ZmXovLOR/4LUsk8= -github.com/aws/aws-sdk-go-v2/service/s3 v1.38.5 h1:A42xdtStObqy7NGvzZKpnyNXvoOmm+FENobZ0/ssHWk= -github.com/aws/aws-sdk-go-v2/service/s3 v1.38.5/go.mod h1:rDGMZA7f4pbmTtPOk5v5UM2lmX6UAbRnMDJeDvnH7AM= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.10 h1:hN4yJBGswmFTOVYqmbz1GBs9ZMtQe8SrYxPwrkrlRv8= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.10/go.mod h1:TsxON4fEZXyrKY+D+3d2gSTyJkGORexIYab9PTf56DA= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.10 h1:fXoWC2gi7tdJYNTPnnlSGzEVwewUchOi8xVq/dkg8Qs= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.10/go.mod h1:cvzBApD5dVazHU8C2rbBQzzzsKc8m5+wNJ9mCRZLKPc= +github.com/aws/aws-sdk-go-v2/service/s3 v1.75.0 h1:UPQJDyqUXICUt60X4PwbiEf+2QQ4VfXUhDk8OEiGtik= +github.com/aws/aws-sdk-go-v2/service/s3 v1.75.0/go.mod h1:hHnELVnIHltd8EOF3YzahVX6F6y2C6dNqpRj1IMkS5I= github.com/aws/aws-sdk-go-v2/service/sso v1.11.9 h1:Gju1UO3E8ceuoYc/AHcdXLuTZ0WGE1PT2BYDwcYhJg8= github.com/aws/aws-sdk-go-v2/service/sso v1.11.9/go.mod h1:UqRD9bBt15P0ofRyDZX6CfsIqPpzeHOhZKWzgSuAzpo= github.com/aws/aws-sdk-go-v2/service/sts v1.16.7 h1:HLzjwQM9975FQWSF3uENDGHT1gFQm/q3QXu2BYIcI08= github.com/aws/aws-sdk-go-v2/service/sts v1.16.7/go.mod h1:lVxTdiiSHY3jb1aeg+BBFtDzZGSUCv6qaNOyEGCJ1AY= github.com/aws/smithy-go v1.11.3/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/aws/smithy-go v1.14.2/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= -github.com/aws/smithy-go v1.17.0 h1:wWJD7LX6PBV6etBUwO0zElG0nWN9rUhp0WdYeHSHAaI= -github.com/aws/smithy-go v1.17.0/go.mod h1:NukqUGpCZIILqqiV0NIjeFh24kd/FAa4beRb6nbIUPE= +github.com/aws/smithy-go v1.22.2 h1:6D9hW43xKFrRx/tXXfAlIZc4JI+yQe6snnWcQyxSyLQ= +github.com/aws/smithy-go v1.22.2/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ= diff --git a/src/cloud-api-adaptor/test/provisioner/aws/provision_common.go b/src/cloud-api-adaptor/test/provisioner/aws/provision_common.go index 15c1ec6e9..5bbbc5fde 100644 --- a/src/cloud-api-adaptor/test/provisioner/aws/provision_common.go +++ b/src/cloud-api-adaptor/test/provisioner/aws/provision_common.go @@ -10,6 +10,7 @@ import ( "os" "os/exec" "path/filepath" + "strconv" "strings" "text/template" "time" @@ -35,6 +36,7 @@ const ( EksCniAddonVersion = "v1.12.5-eksbuild.2" EksVersion = "1.26" AwsCredentialsFile = "aws-cred.env" + ResourcesBaseName = "caa-e2e-test" ) var AWSProps = &AWSProvisioner{} @@ -253,6 +255,15 @@ func (a *AWSProvisioner) DeleteVPC(ctx context.Context, cfg *envconf.Config) err } } + if a.Image.ID != "" || a.Image.EBSSnapshotId != "" { + a.Image.deregisterImage() + } + + if a.Bucket.Key != "" { + log.Infof("Delete key %s from bucket: %s", a.Bucket.Key, a.Bucket.Name) + a.Bucket.deleteKey() + } + return nil } @@ -320,7 +331,8 @@ func (a *AWSProvisioner) UploadPodvm(imagePath string, ctx context.Context, cfg return err } - imageName := strings.Replace(filepath.Base(imagePath), ".qcow2", ".raw", 1) + imageNameSuffix := "-" + strconv.FormatInt(time.Now().Unix(), 10) + imageName := strings.Replace(filepath.Base(imagePath), ".qcow2", imageNameSuffix, 1) log.Infof("Register image with name: %s", imageName) err = a.Image.registerImage(imageName) if err != nil { @@ -338,7 +350,7 @@ func NewVpc(client *ec2.Client, properties map[string]string) *Vpc { } return &Vpc{ - BaseName: "caa-e2e-test", + BaseName: ResourcesBaseName, CidrBlock: cidrBlock, Client: client, ID: properties["aws_vpc_id"], @@ -353,18 +365,8 @@ func NewVpc(client *ec2.Client, properties map[string]string) *Vpc { // createVpc creates the VPC func (v *Vpc) createVpc() error { vpc, err := v.Client.CreateVpc(context.TODO(), &ec2.CreateVpcInput{ - CidrBlock: aws.String(v.CidrBlock), - TagSpecifications: []ec2types.TagSpecification{ - { - ResourceType: ec2types.ResourceTypeVpc, - Tags: []ec2types.Tag{ - { - Key: aws.String("Name"), - Value: aws.String(v.BaseName + "-vpc"), - }, - }, - }, - }, + CidrBlock: aws.String(v.CidrBlock), + TagSpecifications: defaultTagSpecifications(v.BaseName+"-vpc", ec2types.ResourceTypeVpc), }) if err != nil { return err @@ -378,19 +380,9 @@ func (v *Vpc) createVpc() error { func (v *Vpc) createSubnet() error { subnet, err := v.Client.CreateSubnet(context.TODO(), &ec2.CreateSubnetInput{ - VpcId: aws.String(v.ID), - CidrBlock: aws.String("10.0.0.0/25"), - TagSpecifications: []ec2types.TagSpecification{ - { - ResourceType: ec2types.ResourceTypeSubnet, - Tags: []ec2types.Tag{ - { - Key: aws.String("Name"), - Value: aws.String(v.BaseName + "-subnet"), - }, - }, - }, - }, + VpcId: aws.String(v.ID), + CidrBlock: aws.String("10.0.0.0/25"), + TagSpecifications: defaultTagSpecifications(v.BaseName+"-subnet", ec2types.ResourceTypeSubnet), }) if err != nil { @@ -437,20 +429,10 @@ func (v *Vpc) createSecondarySubnet() error { subnet, err := v.Client.CreateSubnet(context.TODO(), &ec2.CreateSubnetInput{ - AvailabilityZone: aws.String(secondarySubnetAz), - VpcId: aws.String(v.ID), - CidrBlock: aws.String("10.0.0.128/25"), - TagSpecifications: []ec2types.TagSpecification{ - { - ResourceType: ec2types.ResourceTypeSubnet, - Tags: []ec2types.Tag{ - { - Key: aws.String("Name"), - Value: aws.String(v.BaseName + "-subnet-2"), - }, - }, - }, - }, + AvailabilityZone: aws.String(secondarySubnetAz), + VpcId: aws.String(v.ID), + CidrBlock: aws.String("10.0.0.128/25"), + TagSpecifications: defaultTagSpecifications(v.BaseName+"-subnet-2", ec2types.ResourceTypeSubnet), }) if err != nil { @@ -477,17 +459,7 @@ func (v *Vpc) setupVpcNetworking() error { if igwOutput, err = v.Client.CreateInternetGateway(context.TODO(), &ec2.CreateInternetGatewayInput{ - TagSpecifications: []ec2types.TagSpecification{ - { - ResourceType: ec2types.ResourceTypeInternetGateway, - Tags: []ec2types.Tag{ - { - Key: aws.String("Name"), - Value: aws.String(v.BaseName + "-igw"), - }, - }, - }, - }, + TagSpecifications: defaultTagSpecifications(v.BaseName+"-igw", ec2types.ResourceTypeInternetGateway), }); err != nil { return err } @@ -503,18 +475,8 @@ func (v *Vpc) setupVpcNetworking() error { if rtOutput, err = v.Client.CreateRouteTable(context.TODO(), &ec2.CreateRouteTableInput{ - VpcId: aws.String(v.ID), - TagSpecifications: []ec2types.TagSpecification{ - { - ResourceType: ec2types.ResourceTypeRouteTable, - Tags: []ec2types.Tag{ - { - Key: aws.String("Name"), - Value: aws.String(v.BaseName + "-rtb"), - }, - }, - }, - }, + VpcId: aws.String(v.ID), + TagSpecifications: defaultTagSpecifications(v.BaseName+"-rtb", ec2types.ResourceTypeRouteTable), }); err != nil { return err } @@ -544,9 +506,10 @@ func (v *Vpc) setupVpcNetworking() error { func (v *Vpc) setupSecurityGroup() error { if sgOutput, err := v.Client.CreateSecurityGroup(context.TODO(), &ec2.CreateSecurityGroupInput{ - Description: aws.String("cloud-api-adaptor e2e tests"), - GroupName: aws.String(v.BaseName + "-sg"), - VpcId: aws.String(v.ID), + Description: aws.String("cloud-api-adaptor e2e tests"), + GroupName: aws.String(v.BaseName + "-sg"), + VpcId: aws.String(v.ID), + TagSpecifications: defaultTagSpecifications(v.BaseName+"-sg", ec2types.ResourceTypeSecurityGroup), }); err != nil { return err } else { @@ -873,6 +836,7 @@ func (i *AMIImage) importEBSSnapshot(bucket *S3Bucket) error { S3Key: aws.String(bucket.Key), }, }, + TagSpecifications: defaultTagSpecifications(ResourcesBaseName+"-snap", ec2types.ResourceTypeImportSnapshotTask), }) if err != nil { return err @@ -897,6 +861,19 @@ func (i *AMIImage) importEBSSnapshot(bucket *S3Bucket) error { taskDetail := describeTasks.ImportSnapshotTasks[0].SnapshotTaskDetail i.EBSSnapshotId = *taskDetail.SnapshotId + // Let's warn but ignore any tagging error + if _, err = i.Client.CreateTags(context.TODO(), &ec2.CreateTagsInput{ + Resources: []string{i.EBSSnapshotId}, + Tags: []ec2types.Tag{ + { + Key: aws.String("Name"), + Value: aws.String(ResourcesBaseName + "-snap"), + }, + }, + }); err != nil { + log.Warnf("Failed to tag EBS snapshot %s: %v", i.EBSSnapshotId, err) + } + return nil } @@ -921,6 +898,7 @@ func (i *AMIImage) registerImage(imageName string) error { EnaSupport: aws.Bool(true), RootDeviceName: aws.String(i.RootDeviceName), VirtualizationType: aws.String("hvm"), + TagSpecifications: defaultTagSpecifications(ResourcesBaseName+"-img", ec2types.ResourceTypeImage), }) if err != nil { return err @@ -931,6 +909,34 @@ func (i *AMIImage) registerImage(imageName string) error { return nil } +// deregisterImage Deregisters an AMI image. The associated EBS snapshot is deleted too. +func (i *AMIImage) deregisterImage() error { + var err error + + if i.ID != "" { + log.Infof("Deregister AMI ID: %s", i.ID) + _, err = i.Client.DeregisterImage(context.TODO(), &ec2.DeregisterImageInput{ + ImageId: aws.String(i.ID), + }) + if err != nil { + log.Errorf("Failed to deregister AMI: %s", err) + } + } + + // Removing the EBS snapshot + if i.EBSSnapshotId != "" { + log.Infof("Delete Snapshot ID: %s", i.EBSSnapshotId) + i.Client.DeleteSnapshot(context.TODO(), &ec2.DeleteSnapshotInput{ + SnapshotId: aws.String(i.EBSSnapshotId), + }) + if err != nil { + log.Errorf("Failed to delete snapshot: %s", err) + } + } + + return err +} + // uploadLargeFileWithCli Uploads large files (>5GB) using the AWS CLI func (b *S3Bucket) uploadLargeFileWithCli(filepath string) error { file, err := os.Open(filepath) @@ -963,6 +969,17 @@ func (b *S3Bucket) uploadLargeFileWithCli(filepath string) error { return nil } +func (b *S3Bucket) deleteKey() error { + if _, err := b.Client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{ + Bucket: aws.String(b.Name), + Key: aws.String(b.Key), + }); err != nil { + return err + } + + return nil +} + // ConvertQcow2ToRaw Converts an qcow2 image to raw. Requires `qemu-img` installed. func ConvertQcow2ToRaw(qcow2 string, raw string) error { cmd := exec.Command("qemu-img", "convert", "-O", "raw", qcow2, raw) diff --git a/src/cloud-api-adaptor/test/provisioner/aws/utils.go b/src/cloud-api-adaptor/test/provisioner/aws/utils.go new file mode 100644 index 000000000..d0f3e3c23 --- /dev/null +++ b/src/cloud-api-adaptor/test/provisioner/aws/utils.go @@ -0,0 +1,24 @@ +// (C) Copyright Confidential Containers Contributors +// SPDX-License-Identifier: Apache-2.0 + +package aws + +import ( + "github.com/aws/aws-sdk-go-v2/aws" + ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types" +) + +// defaultTagSpecifications returns a tag specifications array with only one element and one "Name" tag. +func defaultTagSpecifications(name string, resourceType ec2types.ResourceType) []ec2types.TagSpecification { + return []ec2types.TagSpecification{ + { + ResourceType: resourceType, + Tags: []ec2types.Tag{ + { + Key: aws.String("Name"), + Value: aws.String(name), + }, + }, + }, + } +}