Add Python 3.12 support (#36755)

Finally after a number of dependency upgrades we seem to be able to upgrade to Python 3.12 (pending universal_pathlib 0.2.0 conversion) Several providers are excluded from being installed and wait for Python 3.12, but it should not block Airlfow's general 3.12 support. Co-authored-by: dirrao <[email protected]> (cherry picked from commit 76dee0b)
apache · Mar 11, 2024 · e5d6a37 · e5d6a37
1 parent c6f3439
commit e5d6a37
Show file tree

Hide file tree

Showing 119 changed files with 1,324 additions and 914 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -129,6 +129,7 @@ jobs:
       is-arm-runner: ${{ steps.selective-checks.outputs.is-arm-runner }}
       is-vm-runner: ${{ steps.selective-checks.outputs.is-vm-runner }}
       is-k8s-runner: ${{ steps.selective-checks.outputs.is-k8s-runner }}
+      latest-versions-only: ${{ steps.selective-checks.outputs.latest-versions-only }}
       chicken-egg-providers: ${{ steps.selective-checks.outputs.chicken-egg-providers }}
       has-migrations: ${{ steps.selective-checks.outputs.has-migrations }}
       source-head-repo: ${{ steps.source-run-info.outputs.source-head-repo }}
@@ -604,7 +605,9 @@ jobs:
     env:
       RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
       PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}"
-    if: needs.build-info.outputs.canary-run == 'true'
+    if: >
+      needs.build-info.outputs.canary-run == 'true' &&
+      needs.build-info.outputs.latest-versions-only != 'true'
     steps:
       - name: Cleanup repo
         run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
@@ -858,7 +861,9 @@ jobs:
     env:
       RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
       PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}"
-    if: needs.build-info.outputs.skip-provider-tests != 'true'
+    if: >
+      needs.build-info.outputs.skip-provider-tests != 'true' &&
+      needs.build-info.outputs.latest-versions-only != 'true'
     steps:
       - name: Cleanup repo
         run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
@@ -1085,7 +1090,9 @@ jobs:
       JOB_ID: "helm-tests"
       USE_XDIST: "true"
     if: >
-      needs.build-info.outputs.needs-helm-tests == 'true' && needs.build-info.outputs.default-branch == 'main'
+      needs.build-info.outputs.needs-helm-tests == 'true' &&
+      needs.build-info.outputs.default-branch == 'main' &&
+      needs.build-info.outputs.latest-versions-only != 'true'
     steps:
       - name: Cleanup repo
         run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
@@ -1385,7 +1392,9 @@ jobs:
       JOB_ID: >
         pendulum-2-${{needs.build-info.outputs.default-python-version}}-
         ${{needs.build-info.outputs.default-postgres-version}}
-    if: needs.build-info.outputs.run-tests == 'true'
+    if: >
+      needs.build-info.outputs.run-tests == 'true' &&
+      needs.build-info.outputs.latest-versions-only != 'true'
     steps:
       - name: Cleanup repo
         shell: bash
@@ -1796,7 +1805,9 @@ jobs:
       JOB_ID: "no-db-pendulum-2-${{needs.build-info.outputs.default-python-version}}"
       ENABLE_COVERAGE: "${{needs.build-info.outputs.run-coverage}}"
       DOWNGRADE_PENDULUM: "true"
-    if: needs.build-info.outputs.run-tests == 'true'
+    if: >
+      needs.build-info.outputs.run-tests == 'true' &&
+      needs.build-info.outputs.latest-versions-only != 'true'
     steps:
       - name: Cleanup repo
         shell: bash
@@ -2298,13 +2309,15 @@ jobs:
       - name: >
           Build CI ARM images ${{ needs.build-info.outputs.image-tag }}
           ${{needs.build-info.outputs.all-python-versions-list-as-string}}:${{env.IMAGE_TAG}}
-        run: >
-          breeze ci-image build --run-in-parallel --builder airflow_cache --platform "linux/arm64"
+        run: |
+          # Do not run parallel builds here as they often fail due to github token expiry issues similar to
+          # those described in https://github.com/moby/buildkit/issues/2367
+          for python in ${{needs.build-info.outputs.python-versions-list-as-string}}; do
+            breeze ci-image build --builder airflow_cache --platform "linux/arm64" --python ${python}
+          done
         env:
           UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
           DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }}
-          PYTHON_VERSIONS: ${{needs.build-info.outputs.all-python-versions-list-as-string}}
-          DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}}
           COMMIT_SHA: ${{ github.sha }}
       - name: "Stop ARM instance"
         run: ./scripts/ci/images/ci_stop_arm_instance.sh

diff --git a/Dockerfile b/Dockerfile
@@ -1528,7 +1528,7 @@ ARG USE_CONSTRAINTS_FOR_CONTEXT_PACKAGES="false"
 
 # By changing the epoch we can force reinstalling Airflow and pip all dependencies
 # It can also be overwritten manually by setting the AIRFLOW_CI_BUILD_EPOCH environment variable.
-ARG AIRFLOW_CI_BUILD_EPOCH="10"
+ARG AIRFLOW_CI_BUILD_EPOCH="11"
 ENV AIRFLOW_CI_BUILD_EPOCH=${AIRFLOW_CI_BUILD_EPOCH}
 
 

diff --git a/Dockerfile.ci b/Dockerfile.ci
@@ -1108,7 +1108,7 @@ ARG PYTHON_BASE_IMAGE
 ARG AIRFLOW_IMAGE_REPOSITORY="https://github.com/apache/airflow"
 
 # By increasing this number we can do force build of all dependencies
-ARG DEPENDENCIES_EPOCH_NUMBER="10"
+ARG DEPENDENCIES_EPOCH_NUMBER="11"
 
 # Make sure noninteractive debian install is used and language variables set
 ENV PYTHON_BASE_IMAGE=${PYTHON_BASE_IMAGE} \

diff --git a/README.md b/README.md
@@ -100,7 +100,7 @@ Apache Airflow is tested with:
 
 |             | Main version (dev)           | Stable version (2.8.2) |
 |-------------|------------------------------|------------------------|
-| Python      | 3.8, 3.9, 3.10, 3.11         | 3.8, 3.9, 3.10, 3.11   |
+| Python      | 3.8, 3.9, 3.10, 3.11, 3.12   | 3.8, 3.9, 3.10, 3.11   |
 | Platform    | AMD64/ARM64(\*)              | AMD64/ARM64(\*)        |
 | Kubernetes  | 1.25, 1.26, 1.27, 1.28, 1.29 | 1.25, 1.26, 1.27, 1.28 |
 | PostgreSQL  | 12, 13, 14, 15, 16           | 12, 13, 14, 15, 16     |

diff --git a/airflow/__init__.py b/airflow/__init__.py
@@ -39,7 +39,12 @@
 # configuration is therefore initted early here, simply by importing it.
 from airflow import configuration, settings
 
-__all__ = ["__version__", "DAG", "Dataset", "XComArg"]
+__all__ = [
+    "__version__",
+    "DAG",
+    "Dataset",
+    "XComArg",
+]
 
 # Make `airflow` a namespace package, supporting installing
 # airflow.providers.* in different locations (i.e. one in site, and one in user

diff --git a/airflow/providers/apache/cassandra/provider.yaml b/airflow/providers/apache/cassandra/provider.yaml
@@ -46,6 +46,26 @@ dependencies:
   - apache-airflow>=2.6.0
   - cassandra-driver>=3.13.0
 
+# Cassandra provider is not yet compatible with Python 3.12
+# The main issue is that python cassandra driver by default uses asyncore which has been deprecated since
+# Python 3.6 and removed in Python 3.12 (https://docs.python.org/3.11/library/asyncore.html)
+#
+# Currently the "wheel" package 3.29.0 distributed for manylinux platform is build without libev support (
+# which could be a viable asyncore replacement), and cassandra driver works in Python 3.12 if it is built
+# with libev support (using sdist, having gcc, libev4 and libev-dev installed). But it would be too much to
+# expect our users to build the driver from sources to use it with Python 3.12, so we are waiting for the
+# next release of cassandra-driver which will have libev support in the wheel package.
+# The issue is tracked here https://datastax-oss.atlassian.net/browse/PYTHON-1378
+#
+# Another option to get cassandra drive back is to have asyncio support in the driver instead of asyncore:
+# The issue is tracked here: https://datastax-oss.atlassian.net/browse/PYTHON-1375 and is scheduled
+# to be fixed in cassandra-driver 3.30.0.
+#
+# All Cassandra tests are automatically skipped if cassandra package is not present, so once you remove the
+# exclusion, they will start running for Python 3.12.
+#
+excluded-python-versions: ['3.12']
+
 integrations:
   - integration-name: Apache Cassandra
     external-doc-url: https://cassandra.apache.org/

diff --git a/contributing-docs/07_local_virtualenv.rst b/contributing-docs/07_local_virtualenv.rst
@@ -37,7 +37,7 @@ Required Software Packages
 Use system-level package managers like yum, apt-get for Linux, or
 Homebrew for macOS to install required software packages:
 
-* Python (One of: 3.8, 3.9, 3.10, 3.11)
+* Python (One of: 3.8, 3.9, 3.10, 3.11, 3.12)
 * MySQL 5.7+
 * libxml
 * helm (only for helm chart tests)
@@ -186,6 +186,8 @@ This is what it shows currently:
 +-------------+---------+----------+---------------------------------------------------------------+
 | airflow-311 | virtual | devel    | Environment with Python 3.11                                  |
 +-------------+---------+----------+---------------------------------------------------------------+
+| airflow-312 | virtual | devel    | Environment with Python 3.12                                  |
++-------------+---------+----------+---------------------------------------------------------------+
 
 The default env (if you have not used one explicitly) is ``default`` and it is a Python 3.8
 virtualenv for maximum compatibility with ``devel`` extra installed - this devel extra contains the minimum set

diff --git a/dev/README_RELEASE_AIRFLOW.md b/dev/README_RELEASE_AIRFLOW.md
@@ -822,7 +822,7 @@ the older branches, you should set the "skip" field to true.
 ## Verify production images
 
 ```shell script
-for PYTHON in 3.8 3.9 3.10 3.11
+for PYTHON in 3.8 3.9 3.10 3.11 3.12
 do
     docker pull apache/airflow:${VERSION}-python${PYTHON}
     breeze prod-image verify --image-name apache/airflow:${VERSION}-python${PYTHON}

diff --git a/dev/breeze/README.md b/dev/breeze/README.md
@@ -66,6 +66,6 @@ PLEASE DO NOT MODIFY THE HASH BELOW! IT IS AUTOMATICALLY UPDATED BY PRE-COMMIT.
 
 ---------------------------------------------------------------------------------------------------------
 
-Package config hash: e2db123fd25e40b515520fb9f6ed32a601fe85e6a22b9845343bc5c81e5785b472527ed3bf6d07521c512d2222f99877a1ce1911ac504a3a6af7b7866aa674cc
+Package config hash: 0c6255210f3c20a29aa24405412399cf3f6ff897658f13a0fb7628b4888d6bfe99f49bdb2aa68d9045e14179d4be33a02543c12e394395931024522431bf5dec
 
 ---------------------------------------------------------------------------------------------------------
diff --git a/dev/breeze/doc/ci/02_images.md b/dev/breeze/doc/ci/02_images.md
@@ -574,7 +574,7 @@ percent-encoded when you access them via UI (/ = %2F)
 
 - \<BRANCH\> might be either "main" or "v2-\*-test"
 - \<X.Y\> - Python version (Major + Minor).Should be one of \["3.8",
-  "3.9", "3.10", "3.11"\].
+  "3.9", "3.10", "3.11", "3.12" \].
 - \<COMMIT_SHA\> - full-length SHA of commit either from the tip of the
   branch (for pushes/schedule) or commit from the tip of the branch used
   for the PR.

diff --git a/dev/breeze/doc/ci/04_selective_checks.md b/dev/breeze/doc/ci/04_selective_checks.md
@@ -264,24 +264,33 @@ Also, for most of the jobs, committer builds by default use "Self-hosted" runner
 builds use "Public" runners. For committers, this can be overridden by setting the
 `use public runners` label in the PR.
 
-If you are testing ci workflow changes and want to limit the number of matrix combinations generated by
+If you are testing CI workflow changes and want to limit the number of matrix combinations generated by
 the jobs - you can set `default versions only` label in the PR. This will limit the number of versions
 used in the matrix to the default ones (default Python version and default Kubernetes version).
 
+If you are testing CI workflow changes and want to limit the number of matrix combinations generated by
+the jobs - you can also set `latest versions only` label in the PR. This will limit the number of versions
+used in the matrix to the latest ones (latest Python version and latest Kubernetes version).
+
+You can also disable cache if you want to make sure your tests will run with image that does not have
+left-over package installed from the past cached image - by setting `disable image cache` label in the PR.
+
 ## PR labels
 
 As mentioned below, you can influence the outputs of selected checks by setting labels to the PR. Here is
 am overview of possible labels and their meaning:
 
-| Label                         | Affected outputs                         | Meaning                                                                                                |
-|-------------------------------|------------------------------------------|--------------------------------------------------------------------------------------------------------|
-| canary                        | is-canary-run                            | If set, the PR run from apache/airflow repo behaves as `canary` run (can only be run by maintainer).   |
-| debug ci resources            | debug-ci-resources                       | If set, then debugging resources is enabled during parallel tests and you can see them in the output.  |
-| default versions only         | python-versions-*, kubernetes-versions-* | If set, the number of Python and Kubernetes versions used by the build will be limitd to default ones. |
-| full tests needed             | full-tests-needed                        | Run complete set of tests, including all Python all DB versions, and all test types.                   |
-| non committer build           | is-committer-build                       | If set then even for non-committer builds, the scripts used for images are used from target branch.    |
-| upgrade to newer dependencies | upgrade-to-newer-dependencies            | If set then dependencies in the CI image build are upgraded to the newer ones.                         |
-| use public runners            | runs-on                                  | Force using public runners even for Committer runs.                                                    |
+| Label                         | Affected outputs                         | Meaning                                                                                                     |
+|-------------------------------|------------------------------------------|-------------------------------------------------------------------------------------------------------------|
+| canary                        | is-canary-run                            | If set, the PR run from apache/airflow repo behaves as `canary` run (can only be run by maintainer).        |
+| debug ci resources            | debug-ci-resources                       | If set, then debugging resources is enabled during parallel tests and you can see them in the output.       |
+| default versions only         | python-versions-*, kubernetes-versions-* | If set, the number of Python and Kubernetes versions used by the build will be limited to the default ones. |
+| disable image cache           |  | If set, the number of Python and Kubernetes versions used by the build will be limited to the latest ones.  |
+| latest versions only          | python-versions-*, kubernetes-versions-* | If set, the number of Python and Kubernetes versions used by the build will be limited to the latest ones.  |
+| full tests needed             | full-tests-needed                        | Run complete set of tests, including all Python all DB versions, and all test types.                        |
+| non committer build           | is-committer-build                       | If set then even for non-committer builds, the scripts used for images are used from target branch.         |
+| upgrade to newer dependencies | upgrade-to-newer-dependencies            | If set then dependencies in the CI image build are upgraded to the newer ones.                              |
+| use public runners            | runs-on                                  | Force using public runners even for Committer runs.                                                         |
 
 
 -----

diff --git a/dev/breeze/doc/ci/07_debugging.md b/dev/breeze/doc/ci/07_debugging.md
@@ -54,10 +54,11 @@ maintainer.
   rather than using `Build images` workflow and use the same breeze
   version for building image and testing
 - When you want to test changes to workflows and CI scripts you can set
-  `default versions only` label to the PR. This will make the PR run
-  using the default versions of Python and Kubernetes. This is useful
-  when you want to test changes to the CI scripts and workflows and you
-  want to use far less resources than the full tests.
+  `default versions only` label to the PR or `latest versions only`.
+  This will make the PR run using the default (or latest) versions of
+  Python and Kubernetes. This is useful when you want to test changes to
+  the CI scripts and workflows and you want to use far less resources
+  than the full tests.
 - When you want to test changes to `build-images.yml` workflow you
   should push your branch as `main` branch in your local fork. This will
   run changed `build-images.yml` workflow as it will be in `main` branch
@@ -67,9 +68,11 @@ maintainer.
   in `build-images.yml` workflow, but the workflow is run using the `main`
   version. Setting `non committer build` label will make your PR run using
   the main version of the scripts and the workflow
-- When you want to test how changes in your workflow affect `canary` run,
-  as maintainer, you should push your PR to `apache` repository not to your
-  fork and set `canary` label to the PR
+- When you are a committer want to test how changes in your workflow affect
+  `canary` run, as maintainer, you should push your PR to `apache` repository
+  not to your fork and set `canary` label to the PR
+- When you are a committer and want to test if the tests are passing if the
+  image is freshly built without cache, you can set `disable image cache` label.
 
 -----