test.yml

name: Test

on:
  workflow_dispatch:
    inputs:
      os:
        type: choice
        description: OS to test with
        default: 'ubuntu-latest'
        options:
        - all
        - ubuntu-latest
        - macos-14
        - windows-latest
      python_version:
        type: choice
        description: Python version to test with
        default: '3.12'
        options:
        - 'all'
        - '3.9'
        - '3.10'
        - '3.11'
        - '3.12'
        - 'pypy3.9'
        - 'pypy3.10'
      test_specification:
        type: string
        description: Specification for the tests to run
        default: 'tests/test_import_hook/'
      fail_fast:
        type: boolean
        default: true
        description: Fail fast
  pull_request:
  merge_group:

concurrency:
  group: ${{ github.workflow }}-${{ github.ref_name }}-${{ github.event.pull_request.number }}
  cancel-in-progress: true

permissions:
  contents: read
  checks: write

jobs:
  generate-matrix:
    name: Generate Matrix
    runs-on: ubuntu-latest
    outputs:
      platform: ${{ steps.generate-matrix.outputs.platform }}
      fail-fast: ${{ steps.generate-matrix.outputs.fail-fast }}
    env:
      OS_MATRIX: |
        - ubuntu-latest
        - macos-14
        - windows-latest
      PYTHON_VERSION: |
        - '3.9'
        - '3.10'
        - '3.11'
        - '3.12'
        - 'pypy3.9'
        - 'pypy3.10'
    steps:
      - uses: actions/setup-node@v4
        with:
          node-version: 16
      - run: npm install js-yaml
      - name: Generate matrix
        id: generate-matrix
        uses: actions/github-script@v7
        with:
          script: |
            const yaml = require("js-yaml");

            const ALL_OS = yaml.load(process.env.OS_MATRIX);
            const ALL_PYTHON_VERSIONS = yaml.load(process.env.PYTHON_VERSION);

            let platforms = [];
            let fail_fast = false;

            let all_platforms = [];
            ALL_OS.forEach(os => {
              ALL_PYTHON_VERSIONS.forEach(python_version => {
                if (os === "macos-14") {
                  if (python_version.startsWith("pypy")) {
                    return;  // PyPy is only built for x64
                  } else if (parseInt(python_version.slice(2)) < 11) {
                    return;  // macOS ARM runners only have Python 3.11+
                  }
                }
                all_platforms.push({
                  "os": os,
                  "python-version": python_version
                })
              })
            });

            core.info(`job triggered by: ${context.eventName}`);
            if (context.eventName == 'workflow_dispatch') {
              const INPUT_OS = "${{ github.event.inputs.os }}";
              const INPUT_PYTHON_VERSION = "${{ github.event.inputs.python_version }}";
              const INPUT_FAIL_FAST = "${{ github.event.inputs.fail_fast }}";

              platforms = all_platforms.filter(platform => (
                  (INPUT_OS == "all" || INPUT_OS.includes(platform.os))
                  && (INPUT_PYTHON_VERSION == "all" || INPUT_PYTHON_VERSION.includes(platform['python-version']))
              ));
              fail_fast = INPUT_FAIL_FAST;

            } else if (context.eventName == 'merge_group') {
              platforms = all_platforms;
              fail_fast = false;

            } else if (context.eventName == 'pull_request') {
              const { data: { labels: labels } } = await github.rest.pulls.get({
                owner: context.repo.owner,
                repo: context.repo.repo,
                pull_number: context.payload.pull_request.number
              });
              const label_names = labels.map(label => label.name);

              if (label_names.includes("CI-test-all")) {
                platforms = all_platforms;
              } else {
                // assumes versions are listed in ascending order
                const latest_pypy_index = ALL_PYTHON_VERSIONS.findLastIndex(version => version.startsWith('pypy'));
                const latest_cpython_index = ALL_PYTHON_VERSIONS.findLastIndex(version => /^\d/.test(version));
                const python_versions = [ALL_PYTHON_VERSIONS[latest_pypy_index], ALL_PYTHON_VERSIONS[latest_cpython_index]];
                platforms = all_platforms.filter(platform => (
                  platform.os == "ubuntu-latest" && python_versions.includes(platform["python-version"])
                ));
              }
              fail_fast = !label_names.includes("CI-no-fail-fast");
            }

            core.info(`platforms = ${JSON.stringify(platforms)}`);
            core.setOutput("platform", platforms);

            core.info(`fail fast = ${fail_fast}`);
            core.setOutput("fail-fast", fail_fast);

  test:
    name: Test
    needs: [generate-matrix]
    strategy:
      fail-fast: ${{ needs.generate-matrix.outputs.fail-fast != 'false' }}
      matrix:
        platform: ${{ fromJson(needs.generate-matrix.outputs.platform) }}
    runs-on: ${{ matrix.platform.os }}
    steps:
      - uses: actions/checkout@v4
        with:
          submodules: recursive
      - uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.platform.python-version }}
          cache: "pip"
      - uses: dtolnay/rust-toolchain@stable
        id: rustup
      - name: Install aarch64-apple-darwin Rust target
        if: startsWith(matrix.platform.os, 'macos')
        run: rustup target add aarch64-apple-darwin
      - name: Setup Xcode env
        if: startsWith(matrix.platform.os, 'macos')
        shell: bash
        run: |
          set -ex
          sudo xcode-select -s /Applications/Xcode.app
          bindir="$(xcode-select --print-path)/Toolchains/XcodeDefault.xctoolchain/usr/bin"
          echo "CC=${bindir}/clang" >> "${GITHUB_ENV}"
          echo "CXX=${bindir}/clang++" >> "${GITHUB_ENV}"
          echo "SDKROOT=$(xcrun --sdk macosx --show-sdk-path)" >> "${GITHUB_ENV}"
      # To save disk space
      - name: Disable debuginfo on Windows
        if: startsWith(matrix.platform.os, 'windows')
        run: echo "RUSTFLAGS="-C debuginfo=0"" >> $GITHUB_ENV
      - name: Install test requirements
        run: cd tests && pip install --disable-pip-version-check -r requirements.txt
      - name: Run tests
        shell: bash
        run: |
          EXTRA_ARGS=""
          if [ "${{ needs.generate-matrix.outputs.fail-fast }}" != "false" ]; then
            EXTRA_ARGS="$EXTRA_ARGS --max-failures 4"
          fi

          python tests/runner.py \
            --workspace ./test_workspace \
            --name "${{ matrix.platform.os }}_${{ matrix.platform.python-version }}" \
            ${EXTRA_ARGS} \
            "${{ github.event.inputs.test_specification || 'tests/test_import_hook' }}"
      - name: Upload HTML test report
        uses: actions/upload-artifact@v4
        if: failure()
        with:
          name: ${{ matrix.platform.os }}-${{ matrix.platform.python-version }}-test-report.html
          path: './test_workspace/report.html'
      - name: Publish Test Report
        uses: mikepenz/action-junit-report@v4
        if: always()
        with:
          report_paths: './test_workspace/reports/*.xml'
          test_files_prefix: "${{ matrix.platform.os }}_${{ matrix.platform.python-version }}"
          check_annotations: false
          job_summary: true


  conclusion:
    needs:
      - test
    if: always()
    runs-on: ubuntu-latest
    steps:
      - name: Result
        run: |
          jq -C <<< "${needs}"
          # Check if all needs were successful or skipped.
          "$(jq -r 'all(.result as $result | (["success", "skipped"] | contains([$result])))' <<< "${needs}")"
        env:
          needs: ${{ toJson(needs) }}