E2E tests #504
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: E2E tests | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| api_env: | |
| type: choice | |
| default: "Production" | |
| options: | |
| - Production | |
| - Staging | |
| - Development | |
| - Custom | |
| description: "What API ENV to use as backend." | |
| api_url: | |
| type: string | |
| default: "" | |
| description: "(Optional) Full URL of the API service to use when API Env is 'Custom'" | |
| organization_id: | |
| type: string | |
| default: "" | |
| description: "(Optional) Organization ID to use when API Env is 'Custom'" | |
| api_key_id: | |
| type: string | |
| default: "" | |
| description: "(Optional) API Key ID to use when API Env is 'Custom'" | |
| api_key_secret: | |
| type: string | |
| default: "" | |
| description: "(Optional) API Key secret to use when API Env is 'Custom'" | |
| aws_region: | |
| type: string | |
| default: "" | |
| description: "(Optional) AWS region to run tests in when API Env is 'Custom'" | |
| azure_region: | |
| type: string | |
| default: "" | |
| description: "(Optional) Azure region to run tests in when API Env is 'Custom'" | |
| gcp_region: | |
| type: string | |
| default: "" | |
| description: "(Optional) GCP region to run tests in when API Env is 'Custom'" | |
| run_upgrade_test: | |
| type: boolean | |
| default: false | |
| description: "If checked, additionally create service using latest stable version, then upgrade to the current version" | |
| schedule: | |
| - cron: "0/30 * * * *" | |
| defaults: | |
| run: | |
| shell: bash | |
| jobs: | |
| # Generate a random token to tag the tests with | |
| token: | |
| outputs: | |
| token: ${{ steps.generate.outputs.token }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Generate | |
| id: generate | |
| run: | | |
| # Number of seconds since January first 2025. | |
| token=$(( $(date +%s) - $(date -d "Jan 1, 2025 00:00:00" +%s) )) | |
| echo "token=${token}" >> $GITHUB_OUTPUT | |
| # Find the most recent release of terraform CLI | |
| find-tf-releases: | |
| outputs: | |
| releases: ${{ steps.find.outputs.releases }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - uses: ./.github/actions/find-tf-releases | |
| id: find | |
| with: | |
| count: '3' | |
| list-examples: | |
| outputs: | |
| examples: ${{ steps.list.outputs.examples }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - uses: ./.github/actions/list-examples | |
| id: list | |
| # Run e2e tests | |
| e2e: | |
| outputs: | |
| status: ${{ steps.status.outputs.status }} | |
| needs: [ "token", "find-tf-releases", "list-examples" ] | |
| runs-on: ubuntu-latest | |
| continue-on-error: true | |
| permissions: | |
| id-token: write | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 10 | |
| matrix: | |
| tf_release: ${{ fromJSON(needs.find-tf-releases.outputs.releases) }} | |
| test: ${{ fromJSON(needs.list-examples.outputs.examples) }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Generate test name | |
| id: name | |
| run: | | |
| echo "test_name=[e2e]-${{ matrix.test.name }}-${{ matrix.tf_release }}-${{ matrix.test.cloud }}-${{ needs.token.outputs.token }}" >> $GITHUB_OUTPUT | |
| - name: Get API details for env | |
| id: credentials | |
| env: | |
| api_env: "${{ inputs.api_env }}" | |
| api_env_production: "${{ secrets.API_ENV_PRODUCTION }}" | |
| api_env_staging: "${{ secrets.API_ENV_STAGING }}" | |
| api_env_development: "${{ secrets.API_ENV_DEVELOPMENT }}" | |
| api_url: "${{ inputs.api_url }}" | |
| organization_id: "${{ inputs.organization_id }}" | |
| api_key_id: "${{ inputs.api_key_id }}" | |
| api_key_secret: "${{ inputs.api_key_secret }}" | |
| aws_region: "${{ inputs.aws_region }}" | |
| azure_region: "${{ inputs.azure_region }}" | |
| gcp_region: "${{ inputs.gcp_region }}" | |
| run: | | |
| bash ./.github/scripts/set_api_env.sh ${{ matrix.test.cloud }} | |
| - name: e2e | |
| id: e2e | |
| uses: ./.github/actions/e2e | |
| with: | |
| api_url: ${{ steps.credentials.outputs.api_url }} | |
| organization_id: ${{ steps.credentials.outputs.organization_id }} | |
| token_key: ${{ steps.credentials.outputs.api_key_id }} | |
| token_secret: ${{ steps.credentials.outputs.api_key_secret }} | |
| service_name: ${{steps.name.outputs.test_name}} | |
| test_name: ${{ matrix.test.name }} | |
| tf_release: ${{ matrix.tf_release }} | |
| cloud_provider: ${{ matrix.test.cloud }} | |
| upgrade_test: "false" | |
| skip_build: "false" | |
| region: ${{ steps.credentials.outputs.region }} | |
| aws_role_arn: ${{ secrets.AWS_ASSUME_ROLE_ARN }} | |
| - name: cleanup | |
| if: ${{ always() && matrix.test.cloud == 'aws' && matrix.test.name == 'private_endpoint' }} | |
| uses: ./.github/actions/cleanup-aws | |
| with: | |
| service_name: ${{steps.name.outputs.test_name}} | |
| aws_region: ${{ steps.credentials.outputs.region }} | |
| aws_role_arn: ${{ secrets.AWS_ASSUME_ROLE_ARN }} | |
| - name: Mark error | |
| id: status | |
| if: failure() | |
| run: | | |
| echo "status=failure" >> $GITHUB_OUTPUT | |
| exit 1 | |
| upgrade: | |
| outputs: | |
| status: ${{ steps.status.outputs.status }} | |
| needs: [ "token", "find-tf-releases", "list-examples" ] | |
| runs-on: ubuntu-latest | |
| continue-on-error: true | |
| permissions: | |
| id-token: write | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 10 | |
| matrix: | |
| tf_release: ${{ fromJSON(needs.find-tf-releases.outputs.releases) }} | |
| test: ${{ fromJSON(needs.list-examples.outputs.examples) }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Generate test name | |
| id: name | |
| run: | | |
| echo "test_name=[upg]-${{ matrix.test.name }}-${{ matrix.tf_release }}-${{ matrix.test.cloud }}-${{ needs.token.outputs.token }}" >> $GITHUB_OUTPUT | |
| - name: Get API details for env | |
| id: credentials | |
| env: | |
| api_env: "${{ inputs.api_env }}" | |
| api_env_production: "${{ secrets.API_ENV_PRODUCTION }}" | |
| api_env_staging: "${{ secrets.API_ENV_STAGING }}" | |
| api_env_development: "${{ secrets.API_ENV_DEVELOPMENT }}" | |
| api_url: "${{ inputs.api_url }}" | |
| organization_id: "${{ inputs.organization_id }}" | |
| api_key_id: "${{ inputs.api_key_id }}" | |
| api_key_secret: "${{ inputs.api_key_secret }}" | |
| aws_region: "${{ inputs.aws_region }}" | |
| azure_region: "${{ inputs.azure_region }}" | |
| gcp_region: "${{ inputs.gcp_region }}" | |
| run: | | |
| bash ./.github/scripts/set_api_env.sh ${{ matrix.test.cloud }} | |
| - name: upgrade | |
| if: ${{ inputs.run_upgrade_test }} | |
| id: upgrade | |
| uses: ./.github/actions/e2e | |
| with: | |
| api_url: ${{ steps.credentials.outputs.api_url }} | |
| organization_id: ${{ steps.credentials.outputs.organization_id }} | |
| token_key: ${{ steps.credentials.outputs.api_key_id }} | |
| token_secret: ${{ steps.credentials.outputs.api_key_secret }} | |
| service_name: ${{steps.name.outputs.test_name}} | |
| test_name: ${{ matrix.test.name }} | |
| tf_release: ${{ matrix.tf_release }} | |
| cloud_provider: ${{ matrix.test.cloud }} | |
| upgrade_test: "true" | |
| skip_build: "false" | |
| region: ${{ steps.credentials.outputs.region }} | |
| aws_role_arn: ${{ secrets.AWS_ASSUME_ROLE_ARN }} | |
| - name: cleanup | |
| if: ${{ always() && matrix.test.cloud == 'aws' && matrix.test.name == 'private_endpoint' }} | |
| uses: ./.github/actions/cleanup-aws | |
| with: | |
| service_name: ${{steps.name.outputs.test_name}} | |
| aws_region: ${{ steps.credentials.outputs.region }} | |
| aws_role_arn: ${{ secrets.AWS_ASSUME_ROLE_ARN }} | |
| - name: Mark error | |
| id: status | |
| if: failure() | |
| run: | | |
| echo "status=failure" >> $GITHUB_OUTPUT | |
| exit 1 | |
| report: | |
| runs-on: ubuntu-latest | |
| needs: [ "e2e", "upgrade" ] | |
| if: ${{ needs.e2e.outputs.status == 'failure' }} | |
| steps: | |
| - name: Report Failure on slack | |
| if: ${{ needs.e2e.outputs.status == 'failure' }} | |
| uses: ravsamhq/notify-slack-action@v2 | |
| with: | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| status: 'failure' | |
| notification_title: "E2E tests failed for {branch}" | |
| footer: "{run_url}" | |
| env: | |
| SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK }} | |
| - name: Report error on github UI | |
| run: | | |
| if [ "${{ needs.e2e.outputs.status }}" == 'failure' ] | |
| then | |
| exit 1 | |
| fi | |
| # Delete any leftover service that might have failed deleting | |
| cleanup: | |
| runs-on: ubuntu-latest | |
| needs: [ "e2e", "upgrade", "token" ] | |
| if: always() | |
| continue-on-error: true | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Get API details for env | |
| id: credentials | |
| env: | |
| api_env: "${{ inputs.api_env }}" | |
| api_env_production: "${{ secrets.API_ENV_PRODUCTION }}" | |
| api_env_staging: "${{ secrets.API_ENV_STAGING }}" | |
| api_env_development: "${{ secrets.API_ENV_DEVELOPMENT }}" | |
| api_url: "${{ inputs.api_url }}" | |
| organization_id: "${{ inputs.organization_id }}" | |
| api_key_id: "${{ inputs.api_key_id }}" | |
| api_key_secret: "${{ inputs.api_key_secret }}" | |
| aws_region: "${{ inputs.aws_region }}" | |
| azure_region: "${{ inputs.azure_region }}" | |
| gcp_region: "${{ inputs.gcp_region }}" | |
| run: | | |
| bash ./.github/scripts/set_api_env.sh | |
| - name: cleanup | |
| uses: ./.github/actions/cleanup-clickhouse | |
| with: | |
| api_url: ${{ steps.credentials.outputs.api_url }} | |
| organization_id: ${{ steps.credentials.outputs.organization_id }} | |
| token_key: ${{ steps.credentials.outputs.api_key_id }} | |
| token_secret: ${{ steps.credentials.outputs.api_key_secret }} | |
| token: ${{needs.token.outputs.token}} |