test.yml

name: 🧪 Tests
run-name: 🧪 Tests (${{ github.event_name }})

on:
  workflow_dispatch:
  workflow_call:
  pull_request:
    branches: [main]
  push:
    branches: [main]

concurrency:
  group: ${{ github.head_ref || github.run_id }}
  cancel-in-progress: true

jobs:
  check_changes:
    name: Check changed files
    runs-on: ubuntu-22.04
    permissions:
      pull-requests: read
    outputs:
      pr_changes: ${{ steps.pr_changes.outputs.changes }}
      push_changes: ${{ steps.push_changes.outputs.changes }}
    steps:
      - uses: actions/checkout@v4
      - uses: dorny/paths-filter@v2
        if: github.event_name == 'pull_request'
        id: pr_changes
        with:
          filters: |
            dcpy:
              - dcpy/**
              - pyproject.toml
              - admin/run_environment/**
            template:
              - .github/workflows/test.yml
              - .github/workflows/build.yml
              - .github/workflows/template_test.yml
              - products/template/**
              - dcpy/**
              - admin/run_environment/**
            qa:
              - apps/qa/**
              - dcpy/**
              - admin/run_environment/**
            cbbr:
              - products/cbbr/**
              - admin/run_environment/**
            checkbook:
              - products/checkbook/**
              - admin/run_environment/**
            zap:
              - products/zap-opendata/**
              - admin/run_environment/**
            docker:
              - admin/run_environment/**
            docker_base:
              - admin/run_environment/docker/base/**
      - uses: fvankrieken/paths-filter@master
        if: github.event_name == 'pull_request'
        id: push_changes
        with:
          filters: |
            docker:
              - admin/run_environment/**
            docker_base:
              - admin/run_environment/docker/base/**
          treat-pr-as-push: true

  base_image:
    name: >
      ${{
        (
          contains(needs.check_changes.outputs.pr_changes, 'docker_base')
          && contains(needs.check_changes.outputs.push_changes, 'docker_base')
        )
        && format(
          'Build base:{0} docker image', 
          contains(needs.check_changes.outputs.pr_changes, 'docker_base') && (format('dev-{0}', github.head_ref || github.ref_name)) || 'latest'
        ) || format(
          'Use existing base:{0} docker image', 
          contains(needs.check_changes.outputs.pr_changes, 'docker_base') && (format('dev-{0}', github.head_ref || github.ref_name)) || 'latest'
        )
      }}
    needs: check_changes
    runs-on: ubuntu-22.04
    env:
      branch_tag: ${{ format('dev-{0}', github.head_ref || github.ref_name) }}
    outputs:
      base_tag: ${{ steps.tag.outputs.base_tag }}
      tag: ${{ steps.tag.outputs.tag }}
    steps:
      - name: set image tag
        id: tag
        run: |
          base_tag="${{ contains(needs.check_changes.outputs.pr_changes, 'docker_base') && env.branch_tag || 'latest'}}"
          echo "base_tag=$base_tag" >> $GITHUB_OUTPUT
          echo "base_tag=$base_tag" >> $GITHUB_ENV
          tag="${{ contains(needs.check_changes.outputs.pr_changes, 'docker') && env.branch_tag || 'latest'}}"
          echo "tag=$tag" >> $GITHUB_OUTPUT
      - uses: actions/checkout@v4
        if: >
          contains(needs.check_changes.outputs.pr_changes, 'docker_base')
          && contains(needs.check_changes.outputs.push_changes, 'docker_base')
      - name: Load Secrets
        if: >
          contains(needs.check_changes.outputs.pr_changes, 'docker_base')
          && contains(needs.check_changes.outputs.push_changes, 'docker_base')
        uses: 1password/load-secrets-action@v1
        with:
          export-env: true
        env:
          OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}
          DOCKER_USERNAME: "op://Data Engineering/Dockerhub/username"
          DOCKER_PASSWORD: "op://Data Engineering/Dockerhub/password"
      - name: publish images
        if: >
          contains(needs.check_changes.outputs.pr_changes, 'docker_base')
          && contains(needs.check_changes.outputs.push_changes, 'docker_base')
        run: ./admin/ops/docker_build_and_publish.sh base "${{ env.base_tag }}"

  build_docker_images:
    name: >
      ${{
        (
          contains(needs.check_changes.outputs.pr_changes, 'docker')
          && contains(needs.check_changes.outputs.push_changes, 'docker')
        )
        && format('Build {0}:{1} docker image', matrix.image, needs.base_image.outputs.tag)
        || format('Use existing {0}:{1} docker image', matrix.image, needs.base_image.outputs.tag)
      }}
    needs: 
      - check_changes
      - base_image
    runs-on: ubuntu-22.04
    strategy: 
      matrix: 
        image:
          - build-base
          - build-geosupport
          - dev
    steps:
      - uses: actions/checkout@v4
        if: >
          contains(needs.check_changes.outputs.pr_changes, 'docker')
          && contains(needs.check_changes.outputs.push_changes, 'docker')
      - name: Load Secrets
        if: >
          contains(needs.check_changes.outputs.pr_changes, 'docker')
          && contains(needs.check_changes.outputs.push_changes, 'docker')
        uses: 1password/load-secrets-action@v1
        with:
          export-env: true
        env:
          OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}
          DOCKER_USERNAME: "op://Data Engineering/Dockerhub/username"
          DOCKER_PASSWORD: "op://Data Engineering/Dockerhub/password"
      - name: publish images
        if: contains(needs.check_changes.outputs.pr_changes, 'docker') && contains(needs.check_changes.outputs.push_changes, 'docker')
        run: ./admin/ops/docker_build_and_publish.sh "${{ matrix.image }}" "${{ needs.base_image.outputs.tag }}" "${{ needs.base_image.outputs.base_tag }}"

  test:
    uses: ./.github/workflows/test_helper.yml
    needs: 
      - check_changes
      - base_image
      - build_docker_images
    with:
      image_tag: ${{ needs.base_image.outputs.tag }}
      path_filter: ${{ needs.check_changes.outputs.pr_changes }}
    secrets: inherit

  build:
    needs: build_docker_images
    # only run build as part of tests in the case of the PRs generated by python_compile_requirements.yml
    if: ${{ github.head_ref == 'compile_python_reqs' }}
    uses: ./.github/workflows/build.yml
    secrets: inherit
    with:
      dataset_name: all
      build_note: check newly compiled python requirements
      recipe_file: recipe
      dev_image: true