From a417cce6bbd80966d5a4a5c17f57f3a137876dc9 Mon Sep 17 00:00:00 2001 From: Jens Scheffler Date: Fri, 4 Oct 2024 15:55:07 +0200 Subject: [PATCH] Drop python3.8 support core and providers - v2.10 backport Add newsfragment (cherry picked from commit f0c14e7e434445a8653ec5074532ad253669f5e8) (cherry picked from commit 1638974d5e3def3c350a24163609725c6336f45c) Integrated changes from PR #43310 migrate last 3 commits from original PR Fix pytests of breeze Fix constraints problem with cloudant in python 3.9 (cherry picked from commit 4375185eb0f7a8afd2305b0661f73733f5641a3d) Fix static checks for providers and core Fix selective checks (cherry picked from commit 1b55c8e6ccb014002566fa8d856e416b247d6d48) Drop Python 3.8 support in provider packages (cherry picked from commit 5de5ca73794a80d4a6f998d55d793baa319275fb) (cherry picked from commit 526948f01d73da8f177977ede3e076a36b226498) Found two more traces with Python 3.8 to be updated Fix k8s requirements merge artifact --- .readthedocs.yml | 2 +- Dockerfile | 2 +- Dockerfile.ci | 4 +- INSTALL | 12 +- README.md | 4 +- airflow/cli/commands/connection_command.py | 2 +- airflow/compat/functools.py | 33 --- airflow/configuration.py | 4 +- airflow/io/__init__.py | 2 +- airflow/models/taskinstance.py | 2 +- airflow/operators/python.py | 2 +- .../MANAGING_PROVIDERS_LIFECYCLE.rst | 2 +- .../amazon/aws/transfers/sql_to_s3.py | 9 +- airflow/providers/amazon/aws/utils/mixins.py | 3 +- airflow/providers/cloudant/provider.yaml | 11 +- .../kubernetes/kubernetes_helper_functions.py | 2 +- airflow/providers/common/io/xcom/backend.py | 21 +- airflow/providers/openlineage/conf.py | 2 +- airflow/serialization/serialized_objects.py | 2 +- airflow/serialization/serializers/timezone.py | 16 +- airflow/utils/log/secrets_masker.py | 3 +- airflow/utils/platform.py | 3 +- airflow/www/forms.py | 2 +- airflow/www/views.py | 5 +- clients/python/README.md | 2 +- clients/python/pyproject.toml | 5 +- clients/python/test_python_client.py | 2 +- constraints/README.md | 6 +- .../03_contributors_quick_start.rst | 18 +- contributing-docs/05_pull_requests.rst | 2 +- contributing-docs/07_local_virtualenv.rst | 14 +- contributing-docs/08_static_code_checks.rst | 2 +- .../12_airflow_dependencies_and_extras.rst | 8 +- .../testing/docker_compose_tests.rst | 8 +- contributing-docs/testing/k8s_tests.rst | 84 +++--- contributing-docs/testing/unit_tests.rst | 12 +- dev/README_RELEASE_AIRFLOW.md | 8 +- dev/README_RELEASE_PROVIDER_PACKAGES.md | 2 +- dev/breeze/README.md | 2 +- dev/breeze/doc/10_advanced_breeze_topics.rst | 5 +- dev/breeze/doc/ci/02_images.md | 1 - dev/breeze/doc/ci/04_selective_checks.md | 1 - dev/breeze/doc/images/output-commands.svg | 4 +- .../doc/images/output_ci-image_build.svg | 6 +- .../doc/images/output_ci-image_build.txt | 2 +- .../doc/images/output_ci-image_load.svg | 4 +- .../doc/images/output_ci-image_load.txt | 2 +- .../doc/images/output_ci-image_pull.svg | 60 ++--- .../doc/images/output_ci-image_pull.txt | 2 +- .../doc/images/output_ci-image_save.svg | 36 ++- .../doc/images/output_ci-image_save.txt | 2 +- .../doc/images/output_ci-image_verify.svg | 6 +- .../doc/images/output_ci-image_verify.txt | 2 +- .../doc/images/output_k8s_build-k8s-image.svg | 6 +- .../doc/images/output_k8s_build-k8s-image.txt | 2 +- .../images/output_k8s_configure-cluster.svg | 60 ++--- .../images/output_k8s_configure-cluster.txt | 2 +- .../doc/images/output_k8s_create-cluster.svg | 6 +- .../doc/images/output_k8s_create-cluster.txt | 2 +- .../doc/images/output_k8s_delete-cluster.svg | 34 ++- .../doc/images/output_k8s_delete-cluster.txt | 2 +- .../doc/images/output_k8s_deploy-airflow.svg | 6 +- .../doc/images/output_k8s_deploy-airflow.txt | 2 +- dev/breeze/doc/images/output_k8s_k9s.svg | 32 +-- dev/breeze/doc/images/output_k8s_k9s.txt | 2 +- dev/breeze/doc/images/output_k8s_logs.svg | 34 ++- dev/breeze/doc/images/output_k8s_logs.txt | 2 +- .../images/output_k8s_run-complete-tests.svg | 70 +++-- .../images/output_k8s_run-complete-tests.txt | 2 +- dev/breeze/doc/images/output_k8s_shell.svg | 42 ++- dev/breeze/doc/images/output_k8s_shell.txt | 2 +- dev/breeze/doc/images/output_k8s_status.svg | 4 +- dev/breeze/doc/images/output_k8s_status.txt | 2 +- dev/breeze/doc/images/output_k8s_tests.svg | 70 +++-- dev/breeze/doc/images/output_k8s_tests.txt | 2 +- .../images/output_k8s_upload-k8s-image.svg | 58 ++-- .../images/output_k8s_upload-k8s-image.txt | 2 +- .../doc/images/output_prod-image_build.svg | 6 +- .../doc/images/output_prod-image_build.txt | 2 +- .../doc/images/output_prod-image_load.svg | 4 +- .../doc/images/output_prod-image_load.txt | 2 +- .../doc/images/output_prod-image_pull.svg | 60 ++--- .../doc/images/output_prod-image_pull.txt | 2 +- .../doc/images/output_prod-image_save.svg | 36 ++- .../doc/images/output_prod-image_save.txt | 2 +- .../doc/images/output_prod-image_verify.svg | 6 +- .../doc/images/output_prod-image_verify.txt | 2 +- ...elease-management_generate-constraints.svg | 6 +- ...elease-management_generate-constraints.txt | 2 +- ...e-management_install-provider-packages.svg | 128 +++++---- ...e-management_install-provider-packages.txt | 2 +- ...release-management_release-prod-images.svg | 2 +- ...release-management_release-prod-images.txt | 2 +- ...se-management_verify-provider-packages.svg | 134 +++++----- ...se-management_verify-provider-packages.txt | 2 +- ...put_sbom_export-dependency-information.svg | 4 +- ...put_sbom_export-dependency-information.txt | 2 +- dev/breeze/doc/images/output_setup_config.svg | 4 +- dev/breeze/doc/images/output_setup_config.txt | 2 +- dev/breeze/doc/images/output_shell.svg | 4 +- dev/breeze/doc/images/output_shell.txt | 2 +- .../doc/images/output_start-airflow.svg | 4 +- .../doc/images/output_start-airflow.txt | 2 +- .../output_testing_core-integration-tests.svg | 4 +- .../output_testing_core-integration-tests.txt | 2 +- .../doc/images/output_testing_core-tests.svg | 4 +- .../doc/images/output_testing_core-tests.txt | 2 +- .../output_testing_docker-compose-tests.svg | 4 +- .../output_testing_docker-compose-tests.txt | 2 +- ...ut_testing_providers-integration-tests.svg | 4 +- ...ut_testing_providers-integration-tests.txt | 2 +- .../images/output_testing_providers-tests.svg | 4 +- .../images/output_testing_providers-tests.txt | 2 +- ...output_testing_python-api-client-tests.svg | 4 +- ...output_testing_python-api-client-tests.txt | 2 +- .../images/output_testing_system-tests.svg | 4 +- .../images/output_testing_system-tests.txt | 2 +- dev/breeze/pyproject.toml | 3 +- .../src/airflow_breeze/global_constants.py | 4 +- .../params/common_build_params.py | 2 +- dev/breeze/src/airflow_breeze/utils/cdxgen.py | 3 - .../airflow_breeze/utils/selective_checks.py | 11 +- dev/breeze/tests/test_cache.py | 5 +- dev/breeze/tests/test_packages.py | 2 +- dev/breeze/tests/test_selective_checks.py | 248 ++++++++++-------- dev/breeze/tests/test_shell_params.py | 8 +- dev/check_files.py | 2 +- dev/refresh_images.sh | 4 +- dev/retag_docker_images.py | 2 +- docker_tests/constants.py | 2 +- docker_tests/docker_utils.py | 4 +- .../executors/general.rst | 6 +- .../installing-from-pypi.rst | 2 +- .../modules_management.rst | 26 +- docs/apache-airflow/extra-packages-ref.rst | 2 +- .../installation/installing-from-pypi.rst | 24 +- .../installation/prerequisites.rst | 2 +- .../installation/supported-versions.rst | 4 +- docs/apache-airflow/start.rst | 4 +- docs/docker-stack/README.md | 8 +- docs/docker-stack/build-arg-ref.rst | 2 +- docs/docker-stack/build.rst | 20 +- docs/docker-stack/changelog.rst | 3 + .../customizing/add-build-essential-custom.sh | 2 +- .../customizing/custom-sources.sh | 2 +- .../github-different-repository.sh | 2 +- .../customizing/github-main.sh | 2 +- .../customizing/github-v2-2-test.sh | 2 +- .../customizing/pypi-dev-runtime-deps.sh | 2 +- .../customizing/pypi-extras-and-deps.sh | 2 +- .../customizing/pypi-selected-version.sh | 2 +- .../restricted/restricted_environments.sh | 10 +- docs/docker-stack/entrypoint.rst | 14 +- docs/docker-stack/index.rst | 8 +- generated/PYPI_README.md | 2 +- generated/provider_dependencies.json | 8 +- hatch_build.py | 2 +- .../test_kubernetes_pod_operator.py | 2 +- newsfragments/42739.significant.rst | 1 + pyproject.toml | 3 +- scripts/ci/docker-compose/devcontainer.env | 4 +- scripts/ci/docker-compose/devcontainer.yml | 2 +- .../ci/pre_commit/common_precommit_utils.py | 2 +- scripts/ci/pre_commit/mypy.py | 2 +- scripts/ci/pre_commit/mypy_folder.py | 2 +- scripts/docker/entrypoint_ci.sh | 2 +- .../install_airflow_and_providers.py | 2 +- .../in_container/run_generate_constraints.py | 4 +- .../providers/cloudant/hooks/test_cloudant.py | 13 +- .../google/cloud/operators/test_mlengine.py | 2 +- .../google/cloud/triggers/test_mlengine.py | 2 +- .../example_kubernetes_decorator.py | 4 +- 172 files changed, 895 insertions(+), 985 deletions(-) delete mode 100644 airflow/compat/functools.py create mode 100644 newsfragments/42739.significant.rst diff --git a/.readthedocs.yml b/.readthedocs.yml index aa16e3a8e3d57..4aa28ee78b036 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -20,7 +20,7 @@ formats: [] sphinx: configuration: docs/rtd-deprecation/conf.py python: - version: "3.8" + version: "3.9" install: - method: pip path: . diff --git a/Dockerfile b/Dockerfile index a96ddc40db759..143efb41cb53a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -47,7 +47,7 @@ ARG AIRFLOW_USER_HOME_DIR=/home/airflow # latest released version here ARG AIRFLOW_VERSION="2.10.4" -ARG PYTHON_BASE_IMAGE="python:3.8-slim-bookworm" +ARG PYTHON_BASE_IMAGE="python:3.9-slim-bookworm" # You can swap comments between those two args to test pip from the main version diff --git a/Dockerfile.ci b/Dockerfile.ci index ca526f53f8a87..1163c2c9a80a4 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -16,7 +16,7 @@ # # WARNING: THIS DOCKERFILE IS NOT INTENDED FOR PRODUCTION USE OR DEPLOYMENT. # -ARG PYTHON_BASE_IMAGE="python:3.8-slim-bookworm" +ARG PYTHON_BASE_IMAGE="python:3.9-slim-bookworm" ############################################################################################## # This is the script image where we keep all inlined bash scripts needed in other segments @@ -702,7 +702,7 @@ chmod 1777 /tmp AIRFLOW_SOURCES=$(cd "${IN_CONTAINER_DIR}/../.." || exit 1; pwd) -PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION:=3.8} +PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION:=3.9} export AIRFLOW_HOME=${AIRFLOW_HOME:=${HOME}} diff --git a/INSTALL b/INSTALL index 78506f9a571fc..7a303fad95ac6 100644 --- a/INSTALL +++ b/INSTALL @@ -141,9 +141,7 @@ This is what it shows currently: ┏━━━━━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ ┃ Name ┃ Type ┃ Description ┃ ┡━━━━━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩ -│ default │ virtual │ Default environment with Python 3.8 for maximum compatibility │ -├─────────────┼─────────┼───────────────────────────────────────────────────────────────┤ -│ airflow-38 │ virtual │ Environment with Python 3.8. No devel installed. │ +│ default │ virtual │ Default environment with Python 3.9 for maximum compatibility │ ├─────────────┼─────────┼───────────────────────────────────────────────────────────────┤ │ airflow-39 │ virtual │ Environment with Python 3.9. No devel installed. │ ├─────────────┼─────────┼───────────────────────────────────────────────────────────────┤ @@ -154,7 +152,7 @@ This is what it shows currently: │ airflow-312 │ virtual │ Environment with Python 3.12. No devel installed │ └─────────────┴─────────┴───────────────────────────────────────────────────────────────┘ -The default env (if you have not used one explicitly) is `default` and it is a Python 3.8 +The default env (if you have not used one explicitly) is `default` and it is a Python 3.9 virtualenv for maximum compatibility with `devel` extra installed - this devel extra contains the minimum set of dependencies and tools that should be used during unit testing of core Airflow and running all `airflow` CLI commands - without support for providers or databases. @@ -228,15 +226,15 @@ to avoid "works-for-me" syndrome, where you use different versions of dependenci that are used in main CI tests and by other contributors. There are different constraint files for different Python versions. For example, this command will install -all basic devel requirements and requirements of Google provider as last successfully tested for Python 3.8: +all basic devel requirements and requirements of Google provider as last successfully tested for Python 3.9: pip install -e ".[devel,google]"" \ - --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-main/constraints-3.8.txt" + --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-main/constraints-3.9.txt" Using the 'constraints-no-providers' constraint files, you can upgrade Airflow without paying attention to the provider's dependencies. This allows you to keep installed provider dependencies and install the latest supported ones using pure Airflow core. pip install -e ".[devel]" \ - --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-main/constraints-no-providers-3.8.txt" + --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-main/constraints-no-providers-3.9.txt" Airflow extras ============== diff --git a/README.md b/README.md index 8da91a71f9ad8..0ae00c9542555 100644 --- a/README.md +++ b/README.md @@ -99,7 +99,7 @@ Apache Airflow is tested with: | | Main version (dev) | Stable version (2.10.4) | |-------------|------------------------------|------------------------------| -| Python | 3.9, 3.10, 3.11, 3.12 | 3.8, 3.9, 3.10, 3.11, 3.12 | +| Python | 3.9, 3.10, 3.11, 3.12 | 3.9, 3.10, 3.11, 3.12 | | Platform | AMD64/ARM64(\*) | AMD64/ARM64(\*) | | Kubernetes | 1.26, 1.27, 1.28, 1.29, 1.30 | 1.26, 1.27, 1.28, 1.29, 1.30 | | PostgreSQL | 12, 13, 14, 15, 16 | 12, 13, 14, 15, 16 | @@ -311,7 +311,7 @@ They are based on the official release schedule of Python and Kubernetes, nicely 1. We drop support for Python and Kubernetes versions when they reach EOL. Except for Kubernetes, a version stays supported by Airflow if two major cloud providers still provide support for it. We drop support for those EOL versions in main right after EOL date, and it is effectively removed when we release - the first new MINOR (Or MAJOR if there is no new MINOR version) of Airflow. For example, for Python 3.8 it + the first new MINOR (Or MAJOR if there is no new MINOR version) of Airflow. For example, for Python 3.9 it means that we will drop support in main right after 27.06.2023, and the first MAJOR or MINOR version of Airflow released after will not have it. diff --git a/airflow/cli/commands/connection_command.py b/airflow/cli/commands/connection_command.py index f68830c490bc8..aace3f9c9aede 100644 --- a/airflow/cli/commands/connection_command.py +++ b/airflow/cli/commands/connection_command.py @@ -21,6 +21,7 @@ import json import os import warnings +from functools import cache from pathlib import Path from typing import Any from urllib.parse import urlsplit, urlunsplit @@ -30,7 +31,6 @@ from airflow.cli.simple_table import AirflowConsole from airflow.cli.utils import is_stdout, print_export_output -from airflow.compat.functools import cache from airflow.configuration import conf from airflow.exceptions import AirflowNotFoundException from airflow.hooks.base import BaseHook diff --git a/airflow/compat/functools.py b/airflow/compat/functools.py deleted file mode 100644 index 7b521f9f9da2e..0000000000000 --- a/airflow/compat/functools.py +++ /dev/null @@ -1,33 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -from __future__ import annotations - -import sys - -if sys.version_info >= (3, 9): - from functools import cache -else: - from functools import lru_cache - - cache = lru_cache(maxsize=None) - -# We need to keep it around, in case it was used in the code of old providers, but since we are -# Python 3.8+ we can directly import the functools one -from functools import cached_property # type: ignore - -__all__ = ["cache", "cached_property"] diff --git a/airflow/configuration.py b/airflow/configuration.py index afb4b5f3808b6..cf77e10f5a042 100644 --- a/airflow/configuration.py +++ b/airflow/configuration.py @@ -1885,9 +1885,7 @@ def load_providers_configuration(self): ) self._default_values = create_default_config_parser(self.configuration_description) # sensitive_config_values needs to be refreshed here. This is a cached_property, so we can delete - # the cached values, and it will be refreshed on next access. This has been an implementation - # detail in Python 3.8 but as of Python 3.9 it is documented behaviour. - # See https://docs.python.org/3/library/functools.html#functools.cached_property + # the cached values, and it will be refreshed on next access. try: del self.sensitive_config_values except AttributeError: diff --git a/airflow/io/__init__.py b/airflow/io/__init__.py index 9996a77717ae0..49f2711c3c6cd 100644 --- a/airflow/io/__init__.py +++ b/airflow/io/__init__.py @@ -18,6 +18,7 @@ import inspect import logging +from functools import cache from typing import ( TYPE_CHECKING, Callable, @@ -26,7 +27,6 @@ from fsspec.implementations.local import LocalFileSystem -from airflow.compat.functools import cache from airflow.providers_manager import ProvidersManager from airflow.stats import Stats from airflow.utils.module_loading import import_string diff --git a/airflow/models/taskinstance.py b/airflow/models/taskinstance.py index f84f898f3384c..d2bd885d2ebca 100644 --- a/airflow/models/taskinstance.py +++ b/airflow/models/taskinstance.py @@ -32,6 +32,7 @@ from contextlib import nullcontext from datetime import timedelta from enum import Enum +from functools import cache from typing import TYPE_CHECKING, Any, Callable, Collection, Generator, Iterable, Mapping, Tuple from urllib.parse import quote @@ -70,7 +71,6 @@ from airflow import settings from airflow.api_internal.internal_api_call import InternalApiConfig, internal_api_call -from airflow.compat.functools import cache from airflow.configuration import conf from airflow.datasets import Dataset, DatasetAlias from airflow.datasets.manager import dataset_manager diff --git a/airflow/operators/python.py b/airflow/operators/python.py index fdfe575fb927f..0e1358162b111 100644 --- a/airflow/operators/python.py +++ b/airflow/operators/python.py @@ -30,13 +30,13 @@ import warnings from abc import ABCMeta, abstractmethod from collections.abc import Container +from functools import cache from pathlib import Path from tempfile import TemporaryDirectory from typing import TYPE_CHECKING, Any, Callable, Collection, Iterable, Mapping, NamedTuple, Sequence, cast import lazy_object_proxy -from airflow.compat.functools import cache from airflow.exceptions import ( AirflowConfigException, AirflowException, diff --git a/airflow/providers/MANAGING_PROVIDERS_LIFECYCLE.rst b/airflow/providers/MANAGING_PROVIDERS_LIFECYCLE.rst index 48980f2153cd0..3d3e95c28b172 100644 --- a/airflow/providers/MANAGING_PROVIDERS_LIFECYCLE.rst +++ b/airflow/providers/MANAGING_PROVIDERS_LIFECYCLE.rst @@ -454,7 +454,7 @@ If you have pre-commit installed, pre-commit will be run automatically on commit manually after commit, you can run it via ``breeze static-checks --last-commit`` some of the tests might fail because suspension of the provider might cause changes in the dependencies, so if you see errors about missing dependencies imports, non-usable classes etc., you will need to build the CI image locally -via ``breeze build-image --python 3.8 --upgrade-to-newer-dependencies`` after the first pre-commit run +via ``breeze build-image --python 3.9 --upgrade-to-newer-dependencies`` after the first pre-commit run and then run the static checks again. If you want to be absolutely sure to run all static checks you can always do this via diff --git a/airflow/providers/amazon/aws/transfers/sql_to_s3.py b/airflow/providers/amazon/aws/transfers/sql_to_s3.py index 65e40797a59b1..19bc7f016b186 100644 --- a/airflow/providers/amazon/aws/transfers/sql_to_s3.py +++ b/airflow/providers/amazon/aws/transfers/sql_to_s3.py @@ -223,12 +223,9 @@ def _partition_dataframe(self, df: pd.DataFrame) -> Iterable[tuple[str, pd.DataF for group_label in (grouped_df := df.groupby(**self.groupby_kwargs)).groups: yield ( cast(str, group_label), - cast( - "pd.DataFrame", - grouped_df.get_group(group_label) - .drop(random_column_name, axis=1, errors="ignore") - .reset_index(drop=True), - ), + grouped_df.get_group(group_label) + .drop(random_column_name, axis=1, errors="ignore") + .reset_index(drop=True), ) def _get_hook(self) -> DbApiHook: diff --git a/airflow/providers/amazon/aws/utils/mixins.py b/airflow/providers/amazon/aws/utils/mixins.py index b9c7e30c23dad..fc191d5b74027 100644 --- a/airflow/providers/amazon/aws/utils/mixins.py +++ b/airflow/providers/amazon/aws/utils/mixins.py @@ -28,13 +28,12 @@ from __future__ import annotations import warnings -from functools import cached_property +from functools import cache, cached_property from typing import Any, Generic, NamedTuple, TypeVar from deprecated import deprecated from typing_extensions import final -from airflow.compat.functools import cache from airflow.exceptions import AirflowProviderDeprecationWarning from airflow.providers.amazon.aws.hooks.base_aws import AwsGenericHook diff --git a/airflow/providers/cloudant/provider.yaml b/airflow/providers/cloudant/provider.yaml index 3b6dc2727c8ec..e02612fbd9724 100644 --- a/airflow/providers/cloudant/provider.yaml +++ b/airflow/providers/cloudant/provider.yaml @@ -44,8 +44,15 @@ versions: - 1.0.0 dependencies: - - apache-airflow>=2.7.0 - - cloudant>=2.13.0 + - apache-airflow>=2.8.0 + - ibmcloudant>=0.9.1 + +excluded-python-versions: + # ibmcloudant transitively brings in urllib3 2.x, but the snowflake provider has a dependency that pins + # urllib3 to 1.x on Python 3.9; thus we exclude those Python versions from taking the update + # to ibmcloudant. + # See #21004, #41555, and https://github.com/snowflakedb/snowflake-connector-python/issues/2016 + - "3.9" integrations: - integration-name: IBM Cloudant diff --git a/airflow/providers/cncf/kubernetes/kubernetes_helper_functions.py b/airflow/providers/cncf/kubernetes/kubernetes_helper_functions.py index d337caf14c113..c603f8a178b37 100644 --- a/airflow/providers/cncf/kubernetes/kubernetes_helper_functions.py +++ b/airflow/providers/cncf/kubernetes/kubernetes_helper_functions.py @@ -19,6 +19,7 @@ import logging import secrets import string +from functools import cache from typing import TYPE_CHECKING import pendulum @@ -26,7 +27,6 @@ from kubernetes.client.rest import ApiException from slugify import slugify -from airflow.compat.functools import cache from airflow.configuration import conf from airflow.exceptions import AirflowProviderDeprecationWarning diff --git a/airflow/providers/common/io/xcom/backend.py b/airflow/providers/common/io/xcom/backend.py index af55baa4c0628..256b503181e0e 100644 --- a/airflow/providers/common/io/xcom/backend.py +++ b/airflow/providers/common/io/xcom/backend.py @@ -19,12 +19,13 @@ import contextlib import json import uuid +from functools import cache +from pathlib import Path from typing import TYPE_CHECKING, Any, TypeVar from urllib.parse import urlsplit import fsspec.utils -from airflow.compat.functools import cache from airflow.configuration import conf from airflow.io.path import ObjectStoragePath from airflow.models.xcom import BaseXCom @@ -40,22 +41,6 @@ SECTION = "common.io" -def _is_relative_to(o: ObjectStoragePath, other: ObjectStoragePath) -> bool: - """ - Return whether or not this path is relative to the other path. - - This is a port of the pathlib.Path.is_relative_to method. It is not available in python 3.8. - """ - if hasattr(o, "is_relative_to"): - return o.is_relative_to(other) - - try: - o.relative_to(other) - return True - except ValueError: - return False - - def _get_compression_suffix(compression: str) -> str: """ Return the compression suffix for the given compression. @@ -111,7 +96,7 @@ def _get_full_path(data: str) -> ObjectStoragePath: raise TypeError(f"Not a valid url: {data}") from None if url.scheme: - if not _is_relative_to(ObjectStoragePath(data), p): + if not Path.is_relative_to(ObjectStoragePath(data), p): raise ValueError(f"Invalid key: {data}") return p / data.replace(str(p), "", 1).lstrip("/") diff --git a/airflow/providers/openlineage/conf.py b/airflow/providers/openlineage/conf.py index 562b673ed5ed5..bcb71c7886277 100644 --- a/airflow/providers/openlineage/conf.py +++ b/airflow/providers/openlineage/conf.py @@ -35,7 +35,7 @@ def decorator(func): cache = decorator else: - from airflow.compat.functools import cache + from functools import cache from airflow.configuration import conf _CONFIG_SECTION = "openlineage" diff --git a/airflow/serialization/serialized_objects.py b/airflow/serialization/serialized_objects.py index 9eb3332a6b6a6..797b98f43c5ee 100644 --- a/airflow/serialization/serialized_objects.py +++ b/airflow/serialization/serialized_objects.py @@ -25,6 +25,7 @@ import logging import warnings import weakref +from functools import cache from inspect import signature from textwrap import dedent from typing import TYPE_CHECKING, Any, Collection, Iterable, Mapping, NamedTuple, Union, cast @@ -36,7 +37,6 @@ from airflow import macros from airflow.callbacks.callback_requests import DagCallbackRequest, SlaCallbackRequest, TaskCallbackRequest -from airflow.compat.functools import cache from airflow.configuration import conf from airflow.datasets import ( BaseDataset, diff --git a/airflow/serialization/serializers/timezone.py b/airflow/serialization/serializers/timezone.py index a1f40e67c6972..3d2a29ea08bc4 100644 --- a/airflow/serialization/serializers/timezone.py +++ b/airflow/serialization/serializers/timezone.py @@ -18,7 +18,6 @@ from __future__ import annotations import datetime -import sys from typing import TYPE_CHECKING, Any, cast from airflow.utils.module_loading import qualname @@ -30,15 +29,9 @@ serializers = [ "pendulum.tz.timezone.FixedTimezone", "pendulum.tz.timezone.Timezone", + "zoneinfo.ZoneInfo", ] -PY39 = sys.version_info >= (3, 9) - -if PY39: - serializers.append("zoneinfo.ZoneInfo") -else: - serializers.append("backports.zoneinfo.ZoneInfo") - deserializers = serializers __version__ = 1 @@ -83,11 +76,8 @@ def deserialize(classname: str, version: int, data: object) -> Any: if version > __version__: raise TypeError(f"serialized {version} of {classname} > {__version__}") - if "zoneinfo.ZoneInfo" in classname: - try: - from zoneinfo import ZoneInfo - except ImportError: - from backports.zoneinfo import ZoneInfo + if classname == "backports.zoneinfo.ZoneInfo" and isinstance(data, str): + from zoneinfo import ZoneInfo return ZoneInfo(data) diff --git a/airflow/utils/log/secrets_masker.py b/airflow/utils/log/secrets_masker.py index 13c93d992fffa..4f9604aced7f4 100644 --- a/airflow/utils/log/secrets_masker.py +++ b/airflow/utils/log/secrets_masker.py @@ -22,7 +22,7 @@ import logging import sys from enum import Enum -from functools import cached_property +from functools import cache, cached_property from typing import ( TYPE_CHECKING, Any, @@ -42,7 +42,6 @@ import re2 from airflow import settings -from airflow.compat.functools import cache if TYPE_CHECKING: from kubernetes.client import V1EnvVar diff --git a/airflow/utils/platform.py b/airflow/utils/platform.py index 7945e2b945382..74f56a0ab037d 100644 --- a/airflow/utils/platform.py +++ b/airflow/utils/platform.py @@ -24,8 +24,7 @@ import pkgutil import platform import sys - -from airflow.compat.functools import cache +from functools import cache IS_WINDOWS = platform.system() == "Windows" diff --git a/airflow/www/forms.py b/airflow/www/forms.py index 5b746ab633c06..a3b1d5262db20 100644 --- a/airflow/www/forms.py +++ b/airflow/www/forms.py @@ -20,6 +20,7 @@ import datetime import json import operator +from functools import cache from typing import Iterator import pendulum @@ -36,7 +37,6 @@ from wtforms.fields import Field, IntegerField, PasswordField, SelectField, StringField, TextAreaField from wtforms.validators import InputRequired, Optional -from airflow.compat.functools import cache from airflow.configuration import conf from airflow.providers_manager import ProvidersManager from airflow.utils.types import DagRunType diff --git a/airflow/www/views.py b/airflow/www/views.py index fdaee7a79f668..1ffc4f464b983 100644 --- a/airflow/www/views.py +++ b/airflow/www/views.py @@ -32,7 +32,7 @@ import warnings from bisect import insort_left from collections import defaultdict -from functools import cached_property +from functools import cache, cached_property from json import JSONDecodeError from pathlib import Path from typing import TYPE_CHECKING, Any, Collection, Iterator, Mapping, MutableMapping, Sequence @@ -88,7 +88,6 @@ set_state, ) from airflow.auth.managers.models.resource_details import AccessView, DagAccessEntity, DagDetails -from airflow.compat.functools import cache from airflow.configuration import AIRFLOW_CONFIG, conf from airflow.datasets import Dataset, DatasetAlias from airflow.exceptions import ( @@ -177,7 +176,7 @@ def sanitize_args(args: dict[str, Any]) -> dict[str, Any]: return {key: value for key, value in args.items() if not key.startswith("_")} -# Following the release of https://github.com/python/cpython/issues/102153 in Python 3.8.17 and 3.9.17 on +# Following the release of https://github.com/python/cpython/issues/102153 in Python 3.9.17 on # June 6, 2023, we are adding extra sanitization of the urls passed to get_safe_url method to make it works # the same way regardless if the user uses latest Python patchlevel versions or not. This also follows # a recommended solution by the Python core team. diff --git a/clients/python/README.md b/clients/python/README.md index f0964d555276f..65ca8b45cfee9 100644 --- a/clients/python/README.md +++ b/clients/python/README.md @@ -248,7 +248,7 @@ For more information, please visit [https://airflow.apache.org](https://airflow. ## Requirements. -Python >=3.8 +Python >=3.9 ## Installation & Usage diff --git a/clients/python/pyproject.toml b/clients/python/pyproject.toml index a3f1cdb5cb5fa..9010c3b54d974 100644 --- a/clients/python/pyproject.toml +++ b/clients/python/pyproject.toml @@ -25,7 +25,7 @@ dynamic = ["version"] description = "Apache Airflow API (Stable)" readme = "README.md" license-files.globs = ["LICENSE", "NOTICE"] -requires-python = "~=3.8" +requires-python = "~=3.9" authors = [ { name = "Apache Software Foundation", email = "dev@airflow.apache.org" }, ] @@ -42,7 +42,6 @@ classifiers = [ "Intended Audience :: Developers", "Intended Audience :: System Administrators", "License :: OSI Approved :: Apache Software License", - "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", @@ -74,7 +73,7 @@ run-coverage = "pytest test" run = "run-coverage --no-cov" [[tool.hatch.envs.test.matrix]] -python = ["3.8", "3.9", "3.10", "3.11"] +python = ["3.9", "3.10", "3.11"] [tool.hatch.version] path = "./version.txt" diff --git a/clients/python/test_python_client.py b/clients/python/test_python_client.py index d36f6d1bcc03c..b4b287afb891f 100644 --- a/clients/python/test_python_client.py +++ b/clients/python/test_python_client.py @@ -17,7 +17,7 @@ # # PEP 723 compliant inline script metadata (not yet widely supported) # /// script -# requires-python = ">=3.8" +# requires-python = ">=3.9" # dependencies = [ # "apache-airflow-client", # "rich", diff --git a/constraints/README.md b/constraints/README.md index 791450d1bd7c9..9d02755dc5cd6 100644 --- a/constraints/README.md +++ b/constraints/README.md @@ -29,12 +29,12 @@ This allows you to iterate on dependencies without having to run `--upgrade-to-n Typical workflow in this case is: * download and copy the constraint file to the folder (for example via -[The GitHub Raw Link](https://raw.githubusercontent.com/apache/airflow/constraints-main/constraints-3.8.txt) +[The GitHub Raw Link](https://raw.githubusercontent.com/apache/airflow/constraints-main/constraints-3.9.txt) * modify the constraint file in "constraints" folder * build the image using this command ```bash -breeze ci-image build --python 3.8 --airflow-constraints-location constraints/constraints-3.8txt +breeze ci-image build --python 3.9 --airflow-constraints-location constraints/constraints-3.9.txt ``` You can continue iterating and updating the constraint file (and rebuilding the image) @@ -46,7 +46,7 @@ pip freeze | sort | \ grep -v "apache_airflow" | \ grep -v "apache-airflow==" | \ grep -v "@" | \ - grep -v "/opt/airflow" > /opt/airflow/constraints/constraints-3.8.txt + grep -v "/opt/airflow" > /opt/airflow/constraints/constraints-3.9.txt ``` If you are working with others on updating the dependencies, you can also commit the constraint diff --git a/contributing-docs/03_contributors_quick_start.rst b/contributing-docs/03_contributors_quick_start.rst index 9c7bdbe985178..2fbb6ea201d23 100644 --- a/contributing-docs/03_contributors_quick_start.rst +++ b/contributing-docs/03_contributors_quick_start.rst @@ -256,7 +256,7 @@ Setting up Breeze .. code-block:: bash - breeze --python 3.8 --backend postgres + breeze --python 3.9 --backend postgres .. note:: If you encounter an error like "docker.credentials.errors.InitializationError: @@ -313,7 +313,7 @@ Using Breeze ------------ 1. Starting breeze environment using ``breeze start-airflow`` starts Breeze environment with last configuration run( - In this case python and backend will be picked up from last execution ``breeze --python 3.8 --backend postgres``) + In this case python and backend will be picked up from last execution ``breeze --python 3.9 --backend postgres``) It also automatically starts webserver, backend and scheduler. It drops you in tmux with scheduler in bottom left and webserver in bottom right. Use ``[Ctrl + B] and Arrow keys`` to navigate. @@ -324,9 +324,9 @@ Using Breeze Use CI image. Branch name: main - Docker image: ghcr.io/apache/airflow/main/ci/python3.8:latest + Docker image: ghcr.io/apache/airflow/main/ci/python3.9:latest Airflow source version: 2.4.0.dev0 - Python version: 3.8 + Python version: 3.9 Backend: mysql 5.7 @@ -363,7 +363,7 @@ Using Breeze .. code-block:: bash - breeze --python 3.8 --backend postgres + breeze --python 3.9 --backend postgres 2. Open tmux @@ -630,7 +630,7 @@ All Tests are inside ./tests directory. root@63528318c8b1:/opt/airflow# pytest tests/utils/test_dates.py ============================================================= test session starts ============================================================== - platform linux -- Python 3.8.16, pytest-7.2.1, pluggy-1.0.0 -- /usr/local/bin/python + platform linux -- Python 3.9.16, pytest-7.2.1, pluggy-1.0.0 -- /usr/local/bin/python cachedir: .pytest_cache rootdir: /opt/airflow, configfile: pytest.ini plugins: timeouts-1.2.1, capture-warnings-0.0.4, cov-4.0.0, requests-mock-1.10.0, rerunfailures-11.1.1, anyio-3.6.2, instafail-0.4.2, time-machine-2.9.0, asyncio-0.20.3, httpx-0.21.3, xdist-3.2.0 @@ -657,7 +657,7 @@ All Tests are inside ./tests directory. .. code-block:: bash - breeze --backend postgres --postgres-version 15 --python 3.8 --db-reset testing tests --test-type All + breeze --backend postgres --postgres-version 15 --python 3.9 --db-reset testing tests --test-type All - Running specific type of test @@ -667,7 +667,7 @@ All Tests are inside ./tests directory. .. code-block:: bash - breeze --backend postgres --postgres-version 15 --python 3.8 --db-reset testing tests --test-type Core + breeze --backend postgres --postgres-version 15 --python 3.9 --db-reset testing tests --test-type Core - Running Integration test for specific test type @@ -676,7 +676,7 @@ All Tests are inside ./tests directory. .. code-block:: bash - breeze --backend postgres --postgres-version 15 --python 3.8 --db-reset testing tests --test-type All --integration mongo + breeze --backend postgres --postgres-version 15 --python 3.9 --db-reset testing tests --test-type All --integration mongo - For more information on Testing visit : |09_testing.rst| diff --git a/contributing-docs/05_pull_requests.rst b/contributing-docs/05_pull_requests.rst index ea9300f9c643f..1e14167943497 100644 --- a/contributing-docs/05_pull_requests.rst +++ b/contributing-docs/05_pull_requests.rst @@ -92,7 +92,7 @@ these guidelines: you can push your code to PR and see results of the tests in the CI. - You can use any supported python version to run the tests, but the best is to check - if it works for the oldest supported version (Python 3.8 currently). In rare cases + if it works for the oldest supported version (Python 3.9 currently). In rare cases tests might fail with the oldest version when you use features that are available in newer Python versions. For that purpose we have ``airflow.compat`` package where we keep back-ported useful features from newer versions. diff --git a/contributing-docs/07_local_virtualenv.rst b/contributing-docs/07_local_virtualenv.rst index db9eef29cff78..d878129873217 100644 --- a/contributing-docs/07_local_virtualenv.rst +++ b/contributing-docs/07_local_virtualenv.rst @@ -37,7 +37,7 @@ Required Software Packages Use system-level package managers like yum, apt-get for Linux, or Homebrew for macOS to install required software packages: -* Python (One of: 3.8, 3.9, 3.10, 3.11, 3.12) +* Python (One of: 3.9, 3.10, 3.11, 3.12) * MySQL 5.7+ * libxml * helm (only for helm chart tests) @@ -187,9 +187,7 @@ This is what it shows currently: +-------------+---------+---------------------------------------------------------------+ | Name | Type | Description | +=============+=========+===============================================================+ -| default | virtual | Default environment with Python 3.8 for maximum compatibility | -+-------------+---------+---------------------------------------------------------------+ -| airflow-38 | virtual | Environment with Python 3.8. No devel installed. | +| default | virtual | Default environment with Python 3.9 for maximum compatibility | +-------------+---------+---------------------------------------------------------------+ | airflow-39 | virtual | Environment with Python 3.9. No devel installed. | +-------------+---------+---------------------------------------------------------------+ @@ -200,7 +198,7 @@ This is what it shows currently: | airflow-312 | virtual | Environment with Python 3.12. No devel installed | +-------------+---------+---------------------------------------------------------------+ -The default env (if you have not used one explicitly) is ``default`` and it is a Python 3.8 +The default env (if you have not used one explicitly) is ``default`` and it is a Python 3.9 virtualenv for maximum compatibility. You can install devel set of dependencies with it by running: @@ -381,12 +379,12 @@ to avoid "works-for-me" syndrome, where you use different version of dependencie that are used in main, CI tests and by other contributors. There are different constraint files for different python versions. For example this command will install -all basic devel requirements and requirements of google provider as last successfully tested for Python 3.8: +all basic devel requirements and requirements of google provider as last successfully tested for Python 3.9: .. code:: bash pip install -e ".[devel,google]" \ - --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-main/constraints-source-providers-3.8.txt" + --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-main/constraints-source-providers-3.9.txt" Make sure to use latest main for such installation, those constraints are "development constraints" and they are refreshed several times a day to make sure they are up to date with the latest changes in the main branch. @@ -403,7 +401,7 @@ and install to latest supported ones by pure airflow core. .. code:: bash pip install -e ".[devel]" \ - --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-main/constraints-no-providers-3.8.txt" + --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-main/constraints-no-providers-3.9.txt" These are examples of the development options available with the local virtualenv in your IDE: diff --git a/contributing-docs/08_static_code_checks.rst b/contributing-docs/08_static_code_checks.rst index 27768d23060b4..0880e2f1a7d30 100644 --- a/contributing-docs/08_static_code_checks.rst +++ b/contributing-docs/08_static_code_checks.rst @@ -40,7 +40,7 @@ use. So, you can be sure your modifications will also work for CI if they pass pre-commit hooks. We have integrated the fantastic `pre-commit `__ framework -in our development workflow. To install and use it, you need at least Python 3.8 locally. +in our development workflow. To install and use it, you need at least Python 3.9 locally. Installing pre-commit hooks --------------------------- diff --git a/contributing-docs/12_airflow_dependencies_and_extras.rst b/contributing-docs/12_airflow_dependencies_and_extras.rst index 50854799ca8a4..b72dbc53cc77c 100644 --- a/contributing-docs/12_airflow_dependencies_and_extras.rst +++ b/contributing-docs/12_airflow_dependencies_and_extras.rst @@ -86,7 +86,7 @@ from the PyPI package: .. code-block:: bash pip install "apache-airflow[google,amazon,async]==2.2.5" \ - --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.2.5/constraints-3.8.txt" + --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.2.5/constraints-3.9.txt" The last one can be used to install Airflow in "minimal" mode - i.e when bare Airflow is installed without extras. @@ -98,7 +98,7 @@ requirements). .. code-block:: bash pip install -e ".[devel]" \ - --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-main/constraints-source-providers-3.8.txt" + --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-main/constraints-source-providers-3.9.txt" This also works with extras - for example: @@ -106,7 +106,7 @@ This also works with extras - for example: .. code-block:: bash pip install ".[ssh]" \ - --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-main/constraints-source-providers-3.8.txt" + --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-main/constraints-source-providers-3.9.txt" There are different set of fixed constraint files for different python major/minor versions and you should @@ -118,7 +118,7 @@ using ``constraints-no-providers`` constraint files as well. .. code-block:: bash pip install . --upgrade \ - --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-main/constraints-no-providers-3.8.txt" + --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-main/constraints-no-providers-3.9.txt" The ``constraints-.txt`` and ``constraints-no-providers-.txt`` diff --git a/contributing-docs/testing/docker_compose_tests.rst b/contributing-docs/testing/docker_compose_tests.rst index 94864b4137de8..921a3cafb193b 100644 --- a/contributing-docs/testing/docker_compose_tests.rst +++ b/contributing-docs/testing/docker_compose_tests.rst @@ -48,7 +48,7 @@ Running complete test with breeze: .. code-block:: bash - breeze prod-image build --python 3.8 + breeze prod-image build --python 3.9 breeze testing docker-compose-tests In case the test fails, it will dump the logs from the running containers to the console and it @@ -65,8 +65,8 @@ to see the output of the test as it happens (it can be also set via The test can be also run manually with ``pytest docker_tests/test_docker_compose_quick_start.py`` command, provided that you have a local airflow venv with ``dev`` extra set and the ``DOCKER_IMAGE`` environment variable is set to the image you want to test. The variable defaults -to ``ghcr.io/apache/airflow/main/prod/python3.8:latest`` which is built by default -when you run ``breeze prod-image build --python 3.8``. also the switches ``--skip-docker-compose-deletion`` +to ``ghcr.io/apache/airflow/main/prod/python3.9:latest`` which is built by default +when you run ``breeze prod-image build --python 3.9``. also the switches ``--skip-docker-compose-deletion`` and ``--wait-for-containers-timeout`` can only be passed via environment variables. If you want to debug the deployment using ``docker compose`` commands after ``SKIP_DOCKER_COMPOSE_DELETION`` @@ -87,7 +87,7 @@ the prod image build command above. .. code-block:: bash - export AIRFLOW_IMAGE_NAME=ghcr.io/apache/airflow/main/prod/python3.8:latest + export AIRFLOW_IMAGE_NAME=ghcr.io/apache/airflow/main/prod/python3.9:latest and follow the instructions in the `Running Airflow in Docker `_ diff --git a/contributing-docs/testing/k8s_tests.rst b/contributing-docs/testing/k8s_tests.rst index a4a6f67da0e2c..a9ba3151fe9bb 100644 --- a/contributing-docs/testing/k8s_tests.rst +++ b/contributing-docs/testing/k8s_tests.rst @@ -47,7 +47,7 @@ per each combination of Python and Kubernetes version. This is used during CI wh tests against those different clusters - even in parallel. The cluster name follows the pattern ``airflow-python-X.Y-vA.B.C`` where X.Y is a major/minor Python version -and A.B.C is Kubernetes version. Example cluster name: ``airflow-python-3.8-v1.24.0`` +and A.B.C is Kubernetes version. Example cluster name: ``airflow-python-3.9-v1.24.0`` Most of the commands can be executed in parallel for multiple images/clusters by adding ``--run-in-parallel`` to create clusters or deploy airflow. Similarly checking for status, dumping logs and deleting clusters @@ -215,7 +215,7 @@ Should result in KinD creating the K8S cluster. .. code-block:: text - Config created in /Users/jarek/IdeaProjects/airflow/.build/.k8s-clusters/airflow-python-3.8-v1.24.2/.kindconfig.yaml: + Config created in /Users/jarek/IdeaProjects/airflow/.build/.k8s-clusters/airflow-python-3.9-v1.24.2/.kindconfig.yaml: # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file @@ -251,7 +251,7 @@ Should result in KinD creating the K8S cluster. - Creating cluster "airflow-python-3.8-v1.24.2" ... + Creating cluster "airflow-python-3.9-v1.24.2" ... ✓ Ensuring node image (kindest/node:v1.24.2) 🖼 ✓ Preparing nodes 📦 📦 ✓ Writing configuration 📜 @@ -259,10 +259,10 @@ Should result in KinD creating the K8S cluster. ✓ Installing CNI 🔌 ✓ Installing StorageClass 💾 ✓ Joining worker nodes 🚜 - Set kubectl context to "kind-airflow-python-3.8-v1.24.2" + Set kubectl context to "kind-airflow-python-3.9-v1.24.2" You can now use your cluster with: - kubectl cluster-info --context kind-airflow-python-3.8-v1.24.2 + kubectl cluster-info --context kind-airflow-python-3.9-v1.24.2 Not sure what to do next? 😅 Check out https://kind.sigs.k8s.io/docs/user/quick-start/ @@ -270,9 +270,9 @@ Should result in KinD creating the K8S cluster. Connecting to localhost:18150. Num try: 1 Error when connecting to localhost:18150 : ('Connection aborted.', RemoteDisconnected('Remote end closed connection without response')) - Airflow webserver is not available at port 18150. Run `breeze k8s deploy-airflow --python 3.8 --kubernetes-version v1.24.2` to (re)deploy airflow + Airflow webserver is not available at port 18150. Run `breeze k8s deploy-airflow --python 3.9 --kubernetes-version v1.24.2` to (re)deploy airflow - KinD cluster airflow-python-3.8-v1.24.2 created! + KinD cluster airflow-python-3.9-v1.24.2 created! NEXT STEP: You might now configure your cluster by: @@ -286,20 +286,20 @@ Should result in KinD creating the K8S cluster. .. code-block:: text - Configuring airflow-python-3.8-v1.24.2 to be ready for Airflow deployment - Deleting K8S namespaces for kind-airflow-python-3.8-v1.24.2 + Configuring airflow-python-3.9-v1.24.2 to be ready for Airflow deployment + Deleting K8S namespaces for kind-airflow-python-3.9-v1.24.2 Error from server (NotFound): namespaces "airflow" not found Error from server (NotFound): namespaces "test-namespace" not found Creating namespaces namespace/airflow created namespace/test-namespace created - Created K8S namespaces for cluster kind-airflow-python-3.8-v1.24.2 + Created K8S namespaces for cluster kind-airflow-python-3.9-v1.24.2 - Deploying test resources for cluster kind-airflow-python-3.8-v1.24.2 + Deploying test resources for cluster kind-airflow-python-3.9-v1.24.2 persistentvolume/test-volume created persistentvolumeclaim/test-volume created service/airflow-webserver-node-port created - Deployed test resources for cluster kind-airflow-python-3.8-v1.24.2 + Deployed test resources for cluster kind-airflow-python-3.9-v1.24.2 NEXT STEP: You might now build your k8s image by: @@ -317,45 +317,45 @@ Should show the status of current KinD cluster. .. code-block:: text ======================================================================================================================== - Cluster: airflow-python-3.8-v1.24.2 + Cluster: airflow-python-3.9-v1.24.2 - * KUBECONFIG=/Users/jarek/IdeaProjects/airflow/.build/.k8s-clusters/airflow-python-3.8-v1.24.2/.kubeconfig - * KINDCONFIG=/Users/jarek/IdeaProjects/airflow/.build/.k8s-clusters/airflow-python-3.8-v1.24.2/.kindconfig.yaml + * KUBECONFIG=/Users/jarek/IdeaProjects/airflow/.build/.k8s-clusters/airflow-python-3.9-v1.24.2/.kubeconfig + * KINDCONFIG=/Users/jarek/IdeaProjects/airflow/.build/.k8s-clusters/airflow-python-3.9-v1.24.2/.kindconfig.yaml - Cluster info: airflow-python-3.8-v1.24.2 + Cluster info: airflow-python-3.9-v1.24.2 Kubernetes control plane is running at https://127.0.0.1:48366 CoreDNS is running at https://127.0.0.1:48366/api/v1/namespaces/kube-system/services/kube-dns:dns/proxy To further debug and diagnose cluster problems, use 'kubectl cluster-info dump'. - Storage class for airflow-python-3.8-v1.24.2 + Storage class for airflow-python-3.9-v1.24.2 NAME PROVISIONER RECLAIMPOLICY VOLUMEBINDINGMODE ALLOWVOLUMEEXPANSION AGE standard (default) rancher.io/local-path Delete WaitForFirstConsumer false 83s - Running pods for airflow-python-3.8-v1.24.2 + Running pods for airflow-python-3.9-v1.24.2 NAME READY STATUS RESTARTS AGE coredns-6d4b75cb6d-rwp9d 1/1 Running 0 71s coredns-6d4b75cb6d-vqnrc 1/1 Running 0 71s - etcd-airflow-python-3.8-v1.24.2-control-plane 1/1 Running 0 84s + etcd-airflow-python-3.9-v1.24.2-control-plane 1/1 Running 0 84s kindnet-ckc8l 1/1 Running 0 69s kindnet-qqt8k 1/1 Running 0 71s - kube-apiserver-airflow-python-3.8-v1.24.2-control-plane 1/1 Running 0 84s - kube-controller-manager-airflow-python-3.8-v1.24.2-control-plane 1/1 Running 0 84s + kube-apiserver-airflow-python-3.9-v1.24.2-control-plane 1/1 Running 0 84s + kube-controller-manager-airflow-python-3.9-v1.24.2-control-plane 1/1 Running 0 84s kube-proxy-6g7hn 1/1 Running 0 69s kube-proxy-dwfvp 1/1 Running 0 71s - kube-scheduler-airflow-python-3.8-v1.24.2-control-plane 1/1 Running 0 84s + kube-scheduler-airflow-python-3.9-v1.24.2-control-plane 1/1 Running 0 84s KinD Cluster API server URL: http://localhost:48366 Connecting to localhost:18150. Num try: 1 Error when connecting to localhost:18150 : ('Connection aborted.', RemoteDisconnected('Remote end closed connection without response')) - Airflow webserver is not available at port 18150. Run `breeze k8s deploy-airflow --python 3.8 --kubernetes-version v1.24.2` to (re)deploy airflow + Airflow webserver is not available at port 18150. Run `breeze k8s deploy-airflow --python 3.9 --kubernetes-version v1.24.2` to (re)deploy airflow - Cluster healthy: airflow-python-3.8-v1.24.2 + Cluster healthy: airflow-python-3.9-v1.24.2 5. Build the image base on PROD Airflow image. You need to build the PROD image first (the command will guide you if you did not) either by running the build separately or passing ``--rebuild-base-image`` @@ -373,15 +373,15 @@ Should show the status of current KinD cluster. .. code-block:: text - Building the K8S image for Python 3.8 using airflow base image: ghcr.io/apache/airflow/main/prod/python3.8:latest + Building the K8S image for Python 3.9 using airflow base image: ghcr.io/apache/airflow/main/prod/python3.9:latest [+] Building 0.1s (8/8) FINISHED => [internal] load build definition from Dockerfile 0.0s => => transferring dockerfile: 301B 0.0s => [internal] load .dockerignore 0.0s => => transferring context: 35B 0.0s - => [internal] load metadata for ghcr.io/apache/airflow/main/prod/python3.8:latest 0.0s - => [1/3] FROM ghcr.io/apache/airflow/main/prod/python3.8:latest 0.0s + => [internal] load metadata for ghcr.io/apache/airflow/main/prod/python3.9:latest 0.0s + => [1/3] FROM ghcr.io/apache/airflow/main/prod/python3.9:latest 0.0s => [internal] load build context 0.0s => => transferring context: 3.00kB 0.0s => CACHED [2/3] COPY airflow/example_dags/ /opt/airflow/dags/ 0.0s @@ -389,7 +389,7 @@ Should show the status of current KinD cluster. => exporting to image 0.0s => => exporting layers 0.0s => => writing image sha256:c0bdd363c549c3b0731b8e8ce34153d081f239ee2b582355b7b3ffd5394c40bb 0.0s - => => naming to ghcr.io/apache/airflow/main/prod/python3.8-kubernetes:latest + => => naming to ghcr.io/apache/airflow/main/prod/python3.9-kubernetes:latest NEXT STEP: You might now upload your k8s image by: @@ -409,9 +409,9 @@ Should show the status of current KinD cluster. Good version of kubectl installed: 1.25.0 in /Users/jarek/IdeaProjects/airflow/.build/.k8s-env/bin Good version of helm installed: 3.9.2 in /Users/jarek/IdeaProjects/airflow/.build/.k8s-env/bin Stable repo is already added - Uploading Airflow image ghcr.io/apache/airflow/main/prod/python3.8-kubernetes to cluster airflow-python-3.8-v1.24.2 - Image: "ghcr.io/apache/airflow/main/prod/python3.8-kubernetes" with ID "sha256:fb6195f7c2c2ad97788a563a3fe9420bf3576c85575378d642cd7985aff97412" not yet present on node "airflow-python-3.8-v1.24.2-worker", loading... - Image: "ghcr.io/apache/airflow/main/prod/python3.8-kubernetes" with ID "sha256:fb6195f7c2c2ad97788a563a3fe9420bf3576c85575378d642cd7985aff97412" not yet present on node "airflow-python-3.8-v1.24.2-control-plane", loading... + Uploading Airflow image ghcr.io/apache/airflow/main/prod/python3.9-kubernetes to cluster airflow-python-3.9-v1.24.2 + Image: "ghcr.io/apache/airflow/main/prod/python3.9-kubernetes" with ID "sha256:fb6195f7c2c2ad97788a563a3fe9420bf3576c85575378d642cd7985aff97412" not yet present on node "airflow-python-3.9-v1.24.2-worker", loading... + Image: "ghcr.io/apache/airflow/main/prod/python3.9-kubernetes" with ID "sha256:fb6195f7c2c2ad97788a563a3fe9420bf3576c85575378d642cd7985aff97412" not yet present on node "airflow-python-3.9-v1.24.2-control-plane", loading... NEXT STEP: You might now deploy airflow by: @@ -426,8 +426,8 @@ Should show the status of current KinD cluster. .. code-block:: text - Deploying Airflow for cluster airflow-python-3.8-v1.24.2 - Deploying kind-airflow-python-3.8-v1.24.2 with airflow Helm Chart. + Deploying Airflow for cluster airflow-python-3.9-v1.24.2 + Deploying kind-airflow-python-3.9-v1.24.2 with airflow Helm Chart. Copied chart sources to /private/var/folders/v3/gvj4_mw152q556w2rrh7m46w0000gn/T/chart_edu__kir/chart Deploying Airflow from /private/var/folders/v3/gvj4_mw152q556w2rrh7m46w0000gn/T/chart_edu__kir/chart NAME: airflow @@ -469,12 +469,12 @@ Should show the status of current KinD cluster. Information on how to set a static webserver secret key can be found here: https://airflow.apache.org/docs/helm-chart/stable/production-guide.html#webserver-secret-key - Deployed kind-airflow-python-3.8-v1.24.2 with airflow Helm Chart. + Deployed kind-airflow-python-3.9-v1.24.2 with airflow Helm Chart. - Airflow for Python 3.8 and K8S version v1.24.2 has been successfully deployed. + Airflow for Python 3.9 and K8S version v1.24.2 has been successfully deployed. - The KinD cluster name: airflow-python-3.8-v1.24.2 - The kubectl cluster name: kind-airflow-python-3.8-v1.24.2. + The KinD cluster name: airflow-python-3.9-v1.24.2 + The kubectl cluster name: kind-airflow-python-3.9-v1.24.2. KinD Cluster API server URL: http://localhost:48366 @@ -508,7 +508,7 @@ The virtualenv required will be created automatically when the scripts are run. .. code-block:: text - Running tests with kind-airflow-python-3.8-v1.24.2 cluster. + Running tests with kind-airflow-python-3.9-v1.24.2 cluster. Command to run: pytest kubernetes_tests ========================================================================================= test session starts ========================================================================================== platform darwin -- Python 3.9.9, pytest-6.2.5, py-1.11.0, pluggy-1.0.0 -- /Users/jarek/IdeaProjects/airflow/.build/.k8s-env/bin/python @@ -537,7 +537,7 @@ Once you enter the environment, you receive this information: Entering interactive k8s shell. - (kind-airflow-python-3.8-v1.24.2:KubernetesExecutor)> + (kind-airflow-python-3.9-v1.24.2:KubernetesExecutor)> In a separate terminal you can open the k9s CLI: @@ -647,9 +647,9 @@ Kind has also useful commands to inspect your running cluster: .. code-block:: text - Deleting KinD cluster airflow-python-3.8-v1.24.2! - Deleting cluster "airflow-python-3.8-v1.24.2" ... - KinD cluster airflow-python-3.8-v1.24.2 deleted! + Deleting KinD cluster airflow-python-3.9-v1.24.2! + Deleting cluster "airflow-python-3.9-v1.24.2" ... + KinD cluster airflow-python-3.9-v1.24.2 deleted! Running complete k8s tests diff --git a/contributing-docs/testing/unit_tests.rst b/contributing-docs/testing/unit_tests.rst index e40b5c82d3b0b..f4aa31eaa9ba8 100644 --- a/contributing-docs/testing/unit_tests.rst +++ b/contributing-docs/testing/unit_tests.rst @@ -209,7 +209,7 @@ rerun in Breeze as you will (``-n auto`` will parallelize tests using ``pytest-x .. code-block:: bash - breeze shell --backend none --python 3.8 + breeze shell --backend none --python 3.9 > pytest tests --skip-db-tests -n auto @@ -251,7 +251,7 @@ You can also run DB tests with ``breeze`` dockerized environment. You can choose ``--backend`` flag. The default is ``sqlite`` but you can also use others such as ``postgres`` or ``mysql``. You can also select backend version and Python version to use. You can specify the ``test-type`` to run - breeze will list the test types you can run with ``--help`` and provide auto-complete for them. Example -below runs the ``Core`` tests with ``postgres`` backend and ``3.8`` Python version: +below runs the ``Core`` tests with ``postgres`` backend and ``3.9`` Python version: We have a dedicated, opinionated ``breeze testing db-tests`` command as well that runs DB tests (it is also used in CI to run the DB tests, where you do not have to specify extra flags for @@ -286,7 +286,7 @@ either by package/module/test or by test type - whatever ``pytest`` supports. .. code-block:: bash - breeze shell --backend postgres --python 3.8 + breeze shell --backend postgres --python 3.9 > pytest tests --run-db-tests-only As explained before, you cannot run DB tests in parallel using ``pytest-xdist`` plugin, but ``breeze`` has @@ -296,7 +296,7 @@ you use ``breeze testing db-tests`` command): .. code-block:: bash - breeze testing tests --run-db-tests-only --backend postgres --python 3.8 --run-in-parallel + breeze testing tests --run-db-tests-only --backend postgres --python 3.9 --run-in-parallel Examples of marking test as DB test ................................... @@ -1122,7 +1122,7 @@ directly to the container. .. code-block:: bash - breeze ci-image build --python 3.8 + breeze ci-image build --python 3.9 2. Enter breeze environment by selecting the appropriate airflow version and choosing ``providers-and-tests`` option for ``--mount-sources`` flag. @@ -1224,7 +1224,7 @@ Herr id how to reproduce it. .. code-block:: bash - breeze ci-image build --python 3.8 + breeze ci-image build --python 3.9 2. Build providers from latest sources: diff --git a/dev/README_RELEASE_AIRFLOW.md b/dev/README_RELEASE_AIRFLOW.md index 2d63edf30119f..c1602ea16521b 100644 --- a/dev/README_RELEASE_AIRFLOW.md +++ b/dev/README_RELEASE_AIRFLOW.md @@ -675,7 +675,7 @@ Optionally it can be followed with constraints ```shell script pip install apache-airflow==rc \ - --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-/constraints-3.8.txt"` + --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-/constraints-3.9.txt"` ``` Note that the constraints contain python version that you are installing it with. @@ -687,7 +687,7 @@ There is also an easy way of installation with Breeze if you have the latest sou Running the following command will use tmux inside breeze, create `admin` user and run Webserver & Scheduler: ```shell script -breeze start-airflow --use-airflow-version 2.7.0rc1 --python 3.8 --backend postgres +breeze start-airflow --use-airflow-version 2.7.0rc1 --python 3.9 --backend postgres ``` You can also choose different executors and extras to install when you are installing airflow this way. For @@ -695,7 +695,7 @@ example in order to run Airflow with CeleryExecutor and install celery, google a Airflow 2.7.0, you need to have celery provider installed to run Airflow with CeleryExecutor) you can run: ```shell script -breeze start-airflow --use-airflow-version 2.7.0rc1 --python 3.8 --backend postgres \ +breeze start-airflow --use-airflow-version 2.7.0rc1 --python 3.9 --backend postgres \ --executor CeleryExecutor --airflow-extras "celery,google,amazon" ``` @@ -831,7 +831,7 @@ the older branches, you should set the "skip" field to true. ## Verify production images ```shell script -for PYTHON in 3.8 3.9 3.10 3.11 3.12 +for PYTHON in 3.9 3.10 3.11 3.12 do docker pull apache/airflow:${VERSION}-python${PYTHON} breeze prod-image verify --image-name apache/airflow:${VERSION}-python${PYTHON} diff --git a/dev/README_RELEASE_PROVIDER_PACKAGES.md b/dev/README_RELEASE_PROVIDER_PACKAGES.md index 25aa8062c7722..d546076af82d1 100644 --- a/dev/README_RELEASE_PROVIDER_PACKAGES.md +++ b/dev/README_RELEASE_PROVIDER_PACKAGES.md @@ -1016,7 +1016,7 @@ pip install apache-airflow-providers-==rc ### Installing with Breeze ```shell -breeze start-airflow --use-airflow-version 2.2.4 --python 3.8 --backend postgres \ +breeze start-airflow --use-airflow-version 2.2.4 --python 3.9 --backend postgres \ --load-example-dags --load-default-connections ``` diff --git a/dev/breeze/README.md b/dev/breeze/README.md index b2f2df0e9f9b7..5c6ed5b7a6b52 100644 --- a/dev/breeze/README.md +++ b/dev/breeze/README.md @@ -128,6 +128,6 @@ PLEASE DO NOT MODIFY THE HASH BELOW! IT IS AUTOMATICALLY UPDATED BY PRE-COMMIT. --------------------------------------------------------------------------------------------------------- -Package config hash: f2fa293aecdc1deadd4f08785b9ea77cd597e50f01ad79c092219b7f06eed68f3b0509d0ee7626a2fe25f23875e42a838899b274b5311690546e34e6c660caeb +Package config hash: 3c844eb78b1bc83eac3568fb88ab23097450185a5971cbada0954a20b88c2387b40f0461e6f27c11aa96ec51c6606a4291c977d5620c429da1d3038755250ecd --------------------------------------------------------------------------------------------------------- diff --git a/dev/breeze/doc/10_advanced_breeze_topics.rst b/dev/breeze/doc/10_advanced_breeze_topics.rst index 9bbf113cb7e62..d1920d72b09f4 100644 --- a/dev/breeze/doc/10_advanced_breeze_topics.rst +++ b/dev/breeze/doc/10_advanced_breeze_topics.rst @@ -32,7 +32,10 @@ if you never tested modules and if you do not know how to bypass version check o For testing, you can create your own virtual environment, or use the one that ``uv`` or ``pipx`` created for you if you already installed breeze following the recommended installation. -Or you can change your directory to +For local virtualenv, you can use ``pyenv`` or any other virtualenv wrapper. For example with ``pyenv``, +you can use ``pyenv virtualenv 3.9.6 airflow-breeze`` to create virtualenv called ``airflow-breeze`` +with Python 3.9.6. Then you can use ``pyenv activate airflow-breeze`` to activate it and install breeze +in editable mode with ``pip install -e ./dev/breeze``. The venv can be used for running breeze tests and for debugging breeze. While running tests should be usually "out-of-the-box" for most IDEs, once you configure ``./dev/breeze`` project to use the venv, diff --git a/dev/breeze/doc/ci/02_images.md b/dev/breeze/doc/ci/02_images.md index df8446f5a8606..310365aa80e58 100644 --- a/dev/breeze/doc/ci/02_images.md +++ b/dev/breeze/doc/ci/02_images.md @@ -568,7 +568,6 @@ percent-encoded when you access them via UI (/ = %2F) - \ might be either "main" or "v2-\*-test" - \ - Python version (Major + Minor).Should be one of \["3.9", "3.10", "3.11", "3.12" \]. - ---- Read next about [Github Variables](03_github_variables.md) diff --git a/dev/breeze/doc/ci/04_selective_checks.md b/dev/breeze/doc/ci/04_selective_checks.md index 2734fdda113af..b2b42edc0291a 100644 --- a/dev/breeze/doc/ci/04_selective_checks.md +++ b/dev/breeze/doc/ci/04_selective_checks.md @@ -144,7 +144,6 @@ when some files are not changed. Those are the rules implemented: * if no `All Airflow Python files` changed - `mypy-airflow` check is skipped * if no `All Docs Python files` changed - `mypy-docs` check is skipped * if no `All Dev Python files` changed - `mypy-dev` check is skipped - * if no `UI files` changed - `ts-compile-format-lint-ui` check is skipped * if no `WWW files` changed - `ts-compile-format-lint-www` check is skipped * if no `All Python files` changed - `flynt` check is skipped * if no `Helm files` changed - `lint-helm-chart` check is skipped diff --git a/dev/breeze/doc/images/output-commands.svg b/dev/breeze/doc/images/output-commands.svg index 08b135479de56..12e276fe01a49 100644 --- a/dev/breeze/doc/images/output-commands.svg +++ b/dev/breeze/doc/images/output-commands.svg @@ -299,8 +299,8 @@ ╭─ Execution mode ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ │--python-pPython major/minor version used in Airflow image for images.│ -│(>3.8< | 3.9 | 3.10 | 3.11 | 3.12)                          │ -│[default: 3.8]                                              │ +│(>3.9< | 3.10 | 3.11 | 3.12)                                │ +│[default: 3.9]                                              │ │--integrationCore Integrations to enable when running (can be more than one).                    │ │(all | all-testable | cassandra | celery | drill | kafka | kerberos | keycloak |    │ │mongo | mssql | openlineage | otel | pinot | qdrant | redis | statsd | trino | ydb) │ diff --git a/dev/breeze/doc/images/output_ci-image_build.svg b/dev/breeze/doc/images/output_ci-image_build.svg index beb8c95c2d598..08a8cd34c41ce 100644 --- a/dev/breeze/doc/images/output_ci-image_build.svg +++ b/dev/breeze/doc/images/output_ci-image_build.svg @@ -346,8 +346,8 @@ ╭─ Basic usage ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ │--python-pPython major/minor version used in Airflow image for images.│ -│(>3.8< | 3.9 | 3.10 | 3.11 | 3.12)                          │ -│[default: 3.8]                                              │ +│(>3.9< | 3.10 | 3.11 | 3.12)                                │ +│[default: 3.9]                                              │ │--upgrade-to-newer-dependencies-uWhen set, upgrade all PIP packages to latest.│ │--upgrade-on-failure/--no-upgrade-on-failureWhen set, attempt to run upgrade to newer dependencies when       │ │regular build fails. It is set to False by default on CI and True │ @@ -371,7 +371,7 @@ │(INTEGER RANGE)                                                            │ │[default: 4; 1<=x<=8]                                                      │ │--python-versionsSpace separated list of python versions used for build with multiple versions.(TEXT)│ -│[default: 3.8 3.9 3.10 3.11 3.12]                                             │ +│[default: 3.9 3.10 3.11 3.12]                                                 │ │--run-in-parallelRun the operation in parallel on all or selected subset of parameters.│ │--skip-cleanupSkip cleanup of temporary files created during parallel run.│ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/dev/breeze/doc/images/output_ci-image_build.txt b/dev/breeze/doc/images/output_ci-image_build.txt index f58b665cb89b1..e0662e0e667fc 100644 --- a/dev/breeze/doc/images/output_ci-image_build.txt +++ b/dev/breeze/doc/images/output_ci-image_build.txt @@ -1 +1 @@ -90140a7fba5288d9066334f1b63cc589 +ea5bd8f24822cf1def002aa1f92f0319 diff --git a/dev/breeze/doc/images/output_ci-image_load.svg b/dev/breeze/doc/images/output_ci-image_load.svg index 40a19e40a1c22..cb7036af1c946 100644 --- a/dev/breeze/doc/images/output_ci-image_load.svg +++ b/dev/breeze/doc/images/output_ci-image_load.svg @@ -168,8 +168,8 @@ ╭─ Load image flags ───────────────────────────────────────────────────────────────────────────────────────────────────╮ │--python-pPython major/minor version used in Airflow image for images.│ -│(>3.8< | 3.9 | 3.10 | 3.11 | 3.12)                          │ -│[default: 3.8]                                              │ +│(>3.9< | 3.10 | 3.11 | 3.12)                                │ +│[default: 3.9]                                              │ │--platformPlatform for Airflow image.(linux/amd64 | linux/arm64)│ │--image-fileOptional file name to load the image from - name must follow the                 │ │convention:`ci-image-save-{escaped_platform}-*-{python_version}.tar`. where      │ diff --git a/dev/breeze/doc/images/output_ci-image_load.txt b/dev/breeze/doc/images/output_ci-image_load.txt index d14a69e412474..9ab6e91ff9d6f 100644 --- a/dev/breeze/doc/images/output_ci-image_load.txt +++ b/dev/breeze/doc/images/output_ci-image_load.txt @@ -1 +1 @@ -0b24f3beaa6d117171ee85f5921125f4 +c7792f39775f01082f867e0ce0f38ecb diff --git a/dev/breeze/doc/images/output_ci-image_pull.svg b/dev/breeze/doc/images/output_ci-image_pull.svg index dfaf229682cae..a48fa5e840450 100644 --- a/dev/breeze/doc/images/output_ci-image_pull.svg +++ b/dev/breeze/doc/images/output_ci-image_pull.svg @@ -1,4 +1,4 @@ - + Pull and optionally verify CI images - possibly in parallel for all Python versions. ╭─ Pull image flags ───────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--python-pPython major/minor version used in Airflow image for images.│ -│(>3.8< | 3.9 | 3.10 | 3.11 | 3.12)                          │ -│[default: 3.8]                                              │ -│--verifyVerify image.│ -│--wait-for-imageWait until image is available.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Parallel running ───────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--run-in-parallelRun the operation in parallel on all or selected subset of parameters.│ -│--parallelismMaximum number of processes to use while running the operation in parallel.│ -│(INTEGER RANGE)                                                            │ -│[default: 4; 1<=x<=8]                                                      │ -│--python-versionsSpace separated list of python versions used for build with multiple versions.(TEXT)│ -│[default: 3.8 3.9 3.10 3.11 3.12]                                             │ -│--skip-cleanupSkip cleanup of temporary files created during parallel run.│ -│--debug-resourcesWhether to show resource information while running in parallel.│ -│--include-success-outputsWhether to include outputs of successful parallel runs (skipped by default).│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Github authentication ──────────────────────────────────────────────────────────────────────────────────────────────╮ -│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ -│--github-tokenThe token used to authenticate to GitHub.(TEXT)│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +│--python-pPython major/minor version used in Airflow image for images.(>3.9< | 3.10 | 3.11 | 3.12)│ +│[default: 3.9]                                              │ +│--verifyVerify image.│ +│--wait-for-imageWait until image is available.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Parallel running ───────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--run-in-parallelRun the operation in parallel on all or selected subset of parameters.│ +│--parallelismMaximum number of processes to use while running the operation in parallel.│ +│(INTEGER RANGE)                                                            │ +│[default: 4; 1<=x<=8]                                                      │ +│--python-versionsSpace separated list of python versions used for build with multiple versions.(TEXT)│ +│[default: 3.9 3.10 3.11 3.12]                                                 │ +│--skip-cleanupSkip cleanup of temporary files created during parallel run.│ +│--debug-resourcesWhether to show resource information while running in parallel.│ +│--include-success-outputsWhether to include outputs of successful parallel runs (skipped by default).│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Github authentication ──────────────────────────────────────────────────────────────────────────────────────────────╮ +│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ +│--github-tokenThe token used to authenticate to GitHub.(TEXT)│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/dev/breeze/doc/images/output_ci-image_pull.txt b/dev/breeze/doc/images/output_ci-image_pull.txt index c94c30a6f784a..ae612af5c80bc 100644 --- a/dev/breeze/doc/images/output_ci-image_pull.txt +++ b/dev/breeze/doc/images/output_ci-image_pull.txt @@ -1 +1 @@ -624ace37e6f1e1862923d56aae5136b8 +f032645c0767cde43842824f768940bd diff --git a/dev/breeze/doc/images/output_ci-image_save.svg b/dev/breeze/doc/images/output_ci-image_save.svg index 5c7ec5a388a00..6c4196005c7c8 100644 --- a/dev/breeze/doc/images/output_ci-image_save.svg +++ b/dev/breeze/doc/images/output_ci-image_save.svg @@ -1,4 +1,4 @@ - +