🧪 Tests (pull_request) #6913
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: 🧪 Tests | |
| run-name: 🧪 Tests (${{ github.event_name }}) | |
| on: | |
| workflow_dispatch: | |
| workflow_call: | |
| pull_request: | |
| branches: [main] | |
| push: | |
| branches: [main] | |
| concurrency: | |
| group: ${{ github.head_ref || github.run_id }} | |
| cancel-in-progress: true | |
| jobs: | |
| check_changes: | |
| name: Check changed files | |
| runs-on: ubuntu-22.04 | |
| permissions: | |
| pull-requests: read | |
| outputs: | |
| pr_changes: ${{ steps.pr_changes.outputs.changes }} | |
| push_changes: ${{ steps.push_changes.outputs.changes }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: dorny/paths-filter@v2 | |
| if: github.event_name == 'pull_request' | |
| id: pr_changes | |
| with: | |
| filters: | | |
| dcpy: | |
| - dcpy/** | |
| - pyproject.toml | |
| - admin/run_environment/** | |
| template: | |
| - .github/workflows/test.yml | |
| - .github/workflows/build.yml | |
| - .github/workflows/template_test.yml | |
| - products/template/** | |
| - dcpy/** | |
| - admin/run_environment/** | |
| qa: | |
| - apps/qa/** | |
| - dcpy/** | |
| - admin/run_environment/** | |
| cbbr: | |
| - products/cbbr/** | |
| - admin/run_environment/** | |
| checkbook: | |
| - products/checkbook/** | |
| - admin/run_environment/** | |
| zap: | |
| - products/zap-opendata/** | |
| - admin/run_environment/** | |
| docker: | |
| - admin/run_environment/** | |
| docker_base: | |
| - admin/run_environment/docker/base/** | |
| - uses: fvankrieken/paths-filter@master | |
| if: github.event_name == 'pull_request' | |
| id: push_changes | |
| with: | |
| filters: | | |
| docker: | |
| - admin/run_environment/** | |
| docker_base: | |
| - admin/run_environment/docker/base/** | |
| treat-pr-as-push: true | |
| base_image: | |
| name: > | |
| ${{ | |
| ( | |
| contains(needs.check_changes.outputs.pr_changes, 'docker_base') | |
| && contains(needs.check_changes.outputs.push_changes, 'docker_base') | |
| ) | |
| && format( | |
| 'Build base:{0} docker image', | |
| contains(needs.check_changes.outputs.pr_changes, 'docker_base') && (format('dev-{0}', github.head_ref || github.ref_name)) || 'latest' | |
| ) || format( | |
| 'Use existing base:{0} docker image', | |
| contains(needs.check_changes.outputs.pr_changes, 'docker_base') && (format('dev-{0}', github.head_ref || github.ref_name)) || 'latest' | |
| ) | |
| }} | |
| needs: check_changes | |
| runs-on: ubuntu-22.04 | |
| env: | |
| branch_tag: ${{ format('dev-{0}', github.head_ref || github.ref_name) }} | |
| outputs: | |
| base_tag: ${{ steps.tag.outputs.base_tag }} | |
| tag: ${{ steps.tag.outputs.tag }} | |
| steps: | |
| - name: set image tag | |
| id: tag | |
| run: | | |
| base_tag="${{ contains(needs.check_changes.outputs.pr_changes, 'docker_base') && env.branch_tag || 'latest'}}" | |
| echo "base_tag=$base_tag" >> $GITHUB_OUTPUT | |
| echo "base_tag=$base_tag" >> $GITHUB_ENV | |
| tag="${{ contains(needs.check_changes.outputs.pr_changes, 'docker') && env.branch_tag || 'latest'}}" | |
| echo "tag=$tag" >> $GITHUB_OUTPUT | |
| - uses: actions/checkout@v4 | |
| if: > | |
| contains(needs.check_changes.outputs.pr_changes, 'docker_base') | |
| && contains(needs.check_changes.outputs.push_changes, 'docker_base') | |
| - name: Load Secrets | |
| if: > | |
| contains(needs.check_changes.outputs.pr_changes, 'docker_base') | |
| && contains(needs.check_changes.outputs.push_changes, 'docker_base') | |
| uses: 1password/load-secrets-action@v1 | |
| with: | |
| export-env: true | |
| env: | |
| OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }} | |
| DOCKER_USERNAME: "op://Data Engineering/Dockerhub/username" | |
| DOCKER_PASSWORD: "op://Data Engineering/Dockerhub/password" | |
| - name: publish images | |
| if: > | |
| contains(needs.check_changes.outputs.pr_changes, 'docker_base') | |
| && contains(needs.check_changes.outputs.push_changes, 'docker_base') | |
| run: ./admin/ops/docker_build_and_publish.sh base "${{ env.base_tag }}" | |
| build_docker_images: | |
| name: > | |
| ${{ | |
| ( | |
| contains(needs.check_changes.outputs.pr_changes, 'docker') | |
| && contains(needs.check_changes.outputs.push_changes, 'docker') | |
| ) | |
| && format('Build {0}:{1} docker image', matrix.image, needs.base_image.outputs.tag) | |
| || format('Use existing {0}:{1} docker image', matrix.image, needs.base_image.outputs.tag) | |
| }} | |
| needs: | |
| - check_changes | |
| - base_image | |
| runs-on: ubuntu-22.04 | |
| strategy: | |
| matrix: | |
| image: | |
| - build-base | |
| - build-geosupport | |
| - dev | |
| steps: | |
| - uses: actions/checkout@v4 | |
| if: > | |
| contains(needs.check_changes.outputs.pr_changes, 'docker') | |
| && contains(needs.check_changes.outputs.push_changes, 'docker') | |
| - name: Load Secrets | |
| if: > | |
| contains(needs.check_changes.outputs.pr_changes, 'docker') | |
| && contains(needs.check_changes.outputs.push_changes, 'docker') | |
| uses: 1password/load-secrets-action@v1 | |
| with: | |
| export-env: true | |
| env: | |
| OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }} | |
| DOCKER_USERNAME: "op://Data Engineering/Dockerhub/username" | |
| DOCKER_PASSWORD: "op://Data Engineering/Dockerhub/password" | |
| - name: publish images | |
| if: contains(needs.check_changes.outputs.pr_changes, 'docker') && contains(needs.check_changes.outputs.push_changes, 'docker') | |
| run: ./admin/ops/docker_build_and_publish.sh "${{ matrix.image }}" "${{ needs.base_image.outputs.tag }}" "${{ needs.base_image.outputs.base_tag }}" | |
| test: | |
| uses: ./.github/workflows/test_helper.yml | |
| needs: | |
| - check_changes | |
| - base_image | |
| - build_docker_images | |
| with: | |
| image_tag: ${{ needs.base_image.outputs.tag }} | |
| path_filter: ${{ needs.check_changes.outputs.pr_changes }} | |
| secrets: inherit | |
| build: | |
| needs: build_docker_images | |
| # only run build as part of tests in the case of the PRs generated by python_compile_requirements.yml | |
| if: ${{ github.head_ref == 'compile_python_reqs' }} | |
| uses: ./.github/workflows/build.yml | |
| secrets: inherit | |
| with: | |
| dataset_name: all | |
| build_note: check newly compiled python requirements | |
| recipe_file: recipe | |
| dev_image: true |