Run parallel tests for ARM platform (#50198)

Since we have `ubintu22.04-arm` public runners available, we should
be able to run a set of parallel tests for ARM - this would make our
ARM images move out of experimental phase.

We are not running "all" tests - only those that make sense to run
on ARM for "production" quality.

MySQL tests have some timeouts and run longer on ARM, that's why we
do not run MySQL tests on ARM as well.

We can also get rid of the "ARM collection" test that was introduced
to test if just **collection** of tests when we remove packages that
shoud not be installed on ARM works. But if we are running actual
pytest tests on ARM, we do not need to run those tests any more.

Author: potiuk

.github/workflows/additional-ci-image-checks.yml

@@ -20,12 +20,12 @@ name: Additional CI image checks
 on:  # yamllint disable-line rule:truthy
   workflow_call:
     inputs:
-      amd-runners:
-        description: "The array of labels (in json form) determining AMD public runners."
+      runners:
+        description: "The array of labels (in json form) determining runners."
         required: true
         type: string
-      arm-runners:
-        description: "The array of labels (in json form) determining ARM public runners."
+      platform:
+        description: "Platform for the build - 'linux/amd64' or 'linux/arm64'"
         required: true
         type: string
       python-versions:
@@ -99,12 +99,11 @@ jobs:
       # from forks. This is to prevent malicious PRs from creating images in the "apache/airflow" repo.
       packages: write
     with:
-      amd-runners: ${{ inputs.amd-runners }}
-      arm-runners: ${{ inputs.arm-runners }}
+      runners: ${{ inputs.runners }}
       cache-type: "Early"
       include-prod-images: "false"
       push-latest-images: "false"
-      platform: "linux/amd64"
+      platform: ${{ inputs.platform }}
       python-versions: ${{ inputs.python-versions }}
       branch: ${{ inputs.branch }}
       constraints-branch: ${{ inputs.constraints-branch }}
@@ -120,7 +119,7 @@ jobs:
   check-that-image-builds-quickly:
     timeout-minutes: 11
     name: Check that image builds quickly
-    runs-on: ${{ fromJSON(inputs.amd-runners) }}
+    runs-on: ${{ fromJSON(inputs.runners) }}
     env:
       UPGRADE_TO_NEWER_DEPENDENCIES: false
       PYTHON_MAJOR_MINOR_VERSION: ${{ inputs.default-python-version }}
@@ -129,6 +128,7 @@ jobs:
       GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       GITHUB_USERNAME: ${{ github.actor }}
       VERBOSE: "true"
+      PLATFORM: ${{ inputs.platform }}
     if: inputs.branch == 'main'
     steps:
       - name: "Cleanup repo"
@@ -150,29 +150,4 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: echo "$GITHUB_TOKEN" | docker login ghcr.io -u "$actor" --password-stdin
       - name: "Check that image builds quickly"
-        run: breeze shell --max-time 600 --platform "linux/amd64"
-
-# This is only a check if ARM images are successfully building when committer runs PR from
-# Apache repository. This is needed in case you want to fix failing cache job in "canary" run
-# There is no point in running this one in "canary" run, because the above step is doing the
-# same build anyway.
-#  build-ci-arm-images:
-#    name: Build CI ARM images
-#    uses: ./.github/workflows/ci-image-build.yml
-#    permissions:
-#      contents: read
-#      packages: write
-#    with:
-#      amd-runners: ${{ inputs.amd-runners }}
-#      arm-runners: ${{ inputs.arm-runners }}
-#      platform: "linux/arm64"
-#      push-image: "false"
-#      upload-image-artifact: "true"
-#      upload-mount-cache-artifact: ${{ inputs.canary-run }}
-#      python-versions: ${{ inputs.python-versions }}
-#      branch: ${{ inputs.branch }}
-#      constraints-branch: ${{ inputs.constraints-branch }}
-#      use-uv: ${{ inputs.use-uv }}
-#      upgrade-to-newer-dependencies: ${{ inputs.upgrade-to-newer-dependencies }}
-#      docker-cache: ${{ inputs.docker-cache }}
-#      disable-airflow-repo-cache: ${{ inputs.disable-airflow-repo-cache }}
+        run: breeze shell --max-time 600 --platform "${PLATFORM}"

.github/workflows/additional-prod-image-tests.yml

@@ -20,8 +20,12 @@ name: Additional PROD image tests
 on:  # yamllint disable-line rule:truthy
   workflow_call:
     inputs:
-      amd-runners:
-        description: "The array of labels (in json form) determining AMD public runners."
+      runners:
+        description: "The array of labels (in json form) determining runners."
+        required: true
+        type: string
+      platform:
+        description: "Platform for the build - 'linux/amd64' or 'linux/arm64'"
         required: true
         type: string
       default-branch:
@@ -63,7 +67,8 @@ jobs:
     name: PROD image extra checks (main)
     uses: ./.github/workflows/prod-image-extra-checks.yml
     with:
-      amd-runners: ${{ inputs.amd-runners }}
+      runners: ${{ inputs.runners }}
+      platform: ${{ inputs.platform }}
       python-versions: "[ '${{ inputs.default-python-version }}' ]"
       default-python-version: ${{ inputs.default-python-version }}
       branch: ${{ inputs.default-branch }}
@@ -78,7 +83,8 @@ jobs:
     name: PROD image extra checks (release)
     uses: ./.github/workflows/prod-image-extra-checks.yml
     with:
-      amd-runners: ${{ inputs.amd-runners }}
+      runners: ${{ inputs.runners }}
+      platform: ${{ inputs.platform }}
       python-versions: "[ '${{ inputs.default-python-version }}' ]"
       default-python-version: ${{ inputs.default-python-version }}
       branch: ${{ inputs.default-branch }}
@@ -92,7 +98,7 @@ jobs:
   test-examples-of-prod-image-building:
     timeout-minutes: 60
     name: "Test examples of PROD image building"
-    runs-on: ${{ fromJSON(inputs.amd-runners) }}
+    runs-on: ${{ fromJSON(inputs.runners) }}
     env:
       GITHUB_REPOSITORY: ${{ github.repository }}
       GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -112,7 +118,7 @@ jobs:
       - name: "Prepare breeze & PROD image: ${{ inputs.default-python-version }}"
         uses: ./.github/actions/prepare_breeze_and_image
         with:
-          platform: "linux/amd64"
+          platform: ${{ inputs.platform }}
           image-type: "prod"
           python: ${{ inputs.default-python-version }}
           use-uv: ${{ inputs.use-uv }}
@@ -130,7 +136,7 @@ jobs:
   test-docker-compose-quick-start:
     timeout-minutes: 60
     name: "Docker Compose quick start with PROD image verifying"
-    runs-on: ${{ fromJSON(inputs.amd-runners) }}
+    runs-on: ${{ fromJSON(inputs.runners) }}
     env:
       PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
       GITHUB_REPOSITORY: ${{ github.repository }}
@@ -149,7 +155,7 @@ jobs:
       - name: "Prepare breeze & PROD image: ${{ env.PYTHON_MAJOR_MINOR_VERSION }}"
         uses: ./.github/actions/prepare_breeze_and_image
         with:
-          platform: "linux/amd64"
+          platform: ${{ inputs.platform }}
           image-type: "prod"
           python: ${{ env.PYTHON_MAJOR_MINOR_VERSION }}
           use-uv: ${{ inputs.use-uv }}

.github/workflows/airflow-distributions-tests.yml

@@ -21,8 +21,12 @@ on:  # yamllint disable-line rule:truthy
   workflow_call:
     inputs:
       # Static inputs defined to choose which distribution to test to run
-      amd-runners:
-        description: "The array of labels (in json form) determining public AMD runners."
+      runners:
+        description: "The array of labels (in json form) determining runners."
+        required: true
+        type: string
+      platform:
+        description: "Platform for the build - 'linux/amd64' or 'linux/arm64'"
         required: true
         type: string
       distribution-name:
@@ -59,7 +63,7 @@ jobs:
   distributions-tests:
     timeout-minutes: 80
     name: ${{ inputs.distribution-name }}:P${{ matrix.python-version }} tests
-    runs-on: ${{ fromJSON(inputs.amd-runners) }}
+    runs-on: ${{ fromJSON(inputs.runners) }}
     strategy:
       fail-fast: false
       matrix:
@@ -82,7 +86,7 @@ jobs:
       - name: "Prepare breeze & CI image: ${{ matrix.python-version }}"
         uses: ./.github/actions/prepare_breeze_and_image
         with:
-          platform: "linux/amd64"
+          platform: ${{ inputs.platform }}
           python: ${{ matrix.python-version }}
           use-uv: ${{ inputs.use-uv }}
       - name: "Cleanup dist files"

.github/workflows/basic-tests.yml

@@ -20,8 +20,8 @@ name: Basic tests
 on:  # yamllint disable-line rule:truthy
   workflow_call:
     inputs:
-      amd-runners:
-        description: "The array of labels (in json form) determining public runners."
+      runners:
+        description: "The array of labels (in json form) determining runners."
         required: true
         type: string
       run-ui-tests:
@@ -66,7 +66,7 @@ jobs:
   run-breeze-tests:
     timeout-minutes: 10
     name: Breeze unit tests
-    runs-on: ${{ fromJSON(inputs.amd-runners) }}
+    runs-on: ${{ fromJSON(inputs.runners) }}
     steps:
       - name: "Cleanup repo"
         shell: bash
@@ -87,7 +87,7 @@ jobs:
   tests-ui:
     timeout-minutes: 15
     name: React UI tests
-    runs-on: ${{ fromJSON(inputs.amd-runners) }}
+    runs-on: ${{ fromJSON(inputs.runners) }}
     if: inputs.run-ui-tests == 'true'
     steps:
       - name: "Cleanup repo"
@@ -155,7 +155,7 @@ jobs:
   install-pre-commit:
     timeout-minutes: 5
     name: "Install pre-commit for cache"
-    runs-on: ${{ fromJSON(inputs.amd-runners) }}
+    runs-on: ${{ fromJSON(inputs.runners) }}
     env:
       PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
     steps:
@@ -183,7 +183,7 @@ jobs:
   static-checks-basic-checks-only:
     timeout-minutes: 30
     name: "Static checks: basic checks only"
-    runs-on: ${{ fromJSON(inputs.amd-runners) }}
+    runs-on: ${{ fromJSON(inputs.runners) }}
     needs: install-pre-commit
     if: inputs.basic-checks-only == 'true'
     steps:
@@ -236,7 +236,7 @@ jobs:
   upgrade-check:
     timeout-minutes: 45
     name: "Upgrade checks"
-    runs-on: ${{ fromJSON(inputs.amd-runners) }}
+    runs-on: ${{ fromJSON(inputs.runners) }}
     needs: install-pre-commit
     env:
       PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
@@ -306,7 +306,7 @@ jobs:
   test-airflow-release-commands:
     timeout-minutes: 80
     name: "Test Airflow release commands"
-    runs-on: ${{ fromJSON(inputs.amd-runners) }}
+    runs-on: ${{ fromJSON(inputs.runners) }}
     env:
       PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
       GITHUB_REPOSITORY: ${{ github.repository }}