From 5281f6094c7022c31836746d777b646123b76530 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Wed, 30 Apr 2025 21:45:09 +0200 Subject: [PATCH] [v3-0-test] Add back missing `[sources]` link in generated documentation's includes (#49978) * Add back missing `[sources]` link in generated documentation's includes The exampleinclude of ours used to have generated "sources" link to link to sources of the examples. This was however gone after recent doc build refactoring. The problem was the example directive include import and lack of tests folder available on the python path. Both issues fixed in this PR. Fixes: #49893 * fixup! Add back missing `[sources]` link in generated documentation's includes (cherry picked from commit dd9df359574882f8001e66ecc425e3c0594a18fc) Co-authored-by: Jarek Potiuk --- Dockerfile.ci | 6 +++ airflow-core/docs/tutorial/taskflow.rst | 6 +-- .../airflow/dag_processing/bundles/manager.py | 26 ++++++++++++ airflow-core/src/airflow/models/dagbag.py | 8 ++++ airflow-core/src/airflow/utils/cli.py | 1 - .../routes/public/test_dag_parsing.py | 4 +- .../core_api/routes/public/test_dag_report.py | 7 +++- .../routes/public/test_dag_sources.py | 4 +- .../unit/cli/commands/test_task_command.py | 5 ++- .../tests/unit/jobs/test_scheduler_job.py | 8 ++-- airflow-core/tests/unit/models/test_dagbag.py | 16 ++++++-- clients/python/test_python_client.py | 2 +- devel-common/src/docs/provider_conf.py | 9 +---- .../sphinx_exts/docs_build/docs_builder.py | 11 +++++ .../src/sphinx_exts/exampleinclude.py | 24 ++++++----- .../src/tests_common/pytest_plugin.py | 3 +- docker-stack-docs/conf.py | 2 +- .../test_docker_compose_quick_start.py | 2 +- .../kubernetes_tests/test_other_executors.py | 4 +- providers-summary-docs/conf.py | 2 +- providers/common/messaging/docs/index.rst | 2 + providers/standard/docs/index.rst | 9 ++++- providers/standard/docs/operators/bash.rst | 18 ++++----- .../standard/docs/operators/datetime.rst | 8 ++-- .../standard/docs/operators/latest_only.rst | 2 +- providers/standard/docs/operators/python.rst | 40 +++++++++---------- .../docs/operators/trigger_dag_run.rst | 2 +- providers/standard/docs/sensors/bash.rst | 2 +- providers/standard/docs/sensors/datetime.rst | 10 ++--- .../docs/sensors/external_task_sensor.rst | 8 ++-- providers/standard/docs/sensors/file.rst | 4 +- providers/standard/docs/sensors/python.rst | 4 +- providers/standard/tests/system/__init__.py | 17 ++++++++ .../tests/system/standard/__init__.py | 16 ++++++++ .../standard}/example_bash_decorator.py | 6 +++ .../system/standard}/example_bash_operator.py | 6 +++ .../example_branch_datetime_operator.py | 8 ++++ .../example_branch_day_of_week_operator.py | 6 +++ .../standard}/example_branch_operator.py | 6 +++ .../example_branch_operator_decorator.py | 6 +++ .../example_external_task_child_deferrable.py | 0 .../example_external_task_marker_dag.py | 6 +++ ...example_external_task_parent_deferrable.py | 0 .../system/standard}/example_latest_only.py | 6 +++ .../standard}/example_python_decorator.py | 8 +++- .../standard}/example_python_operator.py | 8 +++- .../standard}/example_sensor_decorator.py | 6 +++ .../tests/system/standard}/example_sensors.py | 6 +++ .../example_short_circuit_decorator.py | 6 +++ .../example_short_circuit_operator.py | 8 +++- .../example_trigger_controller_dag.py | 6 +++ .../tests/system/standard/sql/__init__.py | 16 ++++++++ .../tests/system/standard/sql/sample.sql | 24 +++++++++++ scripts/ci/pre_commit/check_system_tests.py | 2 +- scripts/docker/entrypoint_ci.sh | 6 +++ 55 files changed, 341 insertions(+), 97 deletions(-) create mode 100644 providers/standard/tests/system/__init__.py create mode 100644 providers/standard/tests/system/standard/__init__.py rename {airflow-core/src/airflow/example_dags => providers/standard/tests/system/standard}/example_bash_decorator.py (95%) rename {airflow-core/src/airflow/example_dags => providers/standard/tests/system/standard}/example_bash_operator.py (92%) rename {airflow-core/src/airflow/example_dags => providers/standard/tests/system/standard}/example_branch_datetime_operator.py (93%) rename {airflow-core/src/airflow/example_dags => providers/standard/tests/system/standard}/example_branch_day_of_week_operator.py (92%) rename {airflow-core/src/airflow/example_dags => providers/standard/tests/system/standard}/example_branch_operator.py (96%) rename {airflow-core/src/airflow/example_dags => providers/standard/tests/system/standard}/example_branch_operator_decorator.py (96%) rename {airflow-core/tests/system/core => providers/standard/tests/system/standard}/example_external_task_child_deferrable.py (100%) rename {airflow-core/src/airflow/example_dags => providers/standard/tests/system/standard}/example_external_task_marker_dag.py (94%) rename {airflow-core/tests/system/core => providers/standard/tests/system/standard}/example_external_task_parent_deferrable.py (100%) rename {airflow-core/src/airflow/example_dags => providers/standard/tests/system/standard}/example_latest_only.py (88%) rename {airflow-core/src/airflow/example_dags => providers/standard/tests/system/standard}/example_python_decorator.py (94%) rename {airflow-core/src/airflow/example_dags => providers/standard/tests/system/standard}/example_python_operator.py (95%) rename {airflow-core/src/airflow/example_dags => providers/standard/tests/system/standard}/example_sensor_decorator.py (90%) rename {airflow-core/src/airflow/example_dags => providers/standard/tests/system/standard}/example_sensors.py (95%) rename {airflow-core/src/airflow/example_dags => providers/standard/tests/system/standard}/example_short_circuit_decorator.py (92%) rename {airflow-core/src/airflow/example_dags => providers/standard/tests/system/standard}/example_short_circuit_operator.py (92%) rename {airflow-core/src/airflow/example_dags => providers/standard/tests/system/standard}/example_trigger_controller_dag.py (90%) create mode 100644 providers/standard/tests/system/standard/sql/__init__.py create mode 100644 providers/standard/tests/system/standard/sql/sample.sql diff --git a/Dockerfile.ci b/Dockerfile.ci index d8b4d7f343e13..e68477ab09df0 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -950,6 +950,12 @@ function environment_initialization() { cd "${AIRFLOW_SOURCES}" + # Temporarily add /opt/airflow/providers/standard/tests to PYTHONPATH in order to see example dags + # in the UI when testing in Breeze. This might be solved differently in the future + if [[ -d /opt/airflow/providers/standard/tests ]]; then + export PYTHONPATH=${PYTHONPATH=}:/opt/airflow/providers/standard/tests + fi + if [[ ${START_AIRFLOW:="false"} == "true" || ${START_AIRFLOW} == "True" ]]; then export AIRFLOW__CORE__LOAD_EXAMPLES=${LOAD_EXAMPLES} wait_for_asset_compilation diff --git a/airflow-core/docs/tutorial/taskflow.rst b/airflow-core/docs/tutorial/taskflow.rst index 8ba2df9a51019..b5804dea94c18 100644 --- a/airflow-core/docs/tutorial/taskflow.rst +++ b/airflow-core/docs/tutorial/taskflow.rst @@ -269,7 +269,7 @@ system-level packages. TaskFlow supports multiple execution environments to isol Creates a temporary virtualenv at task runtime. Great for experimental or dynamic tasks, but may have cold start overhead. -.. exampleinclude:: /../src/airflow/example_dags/example_python_decorator.py +.. exampleinclude:: /../../providers/standard/tests/system/standard/example_python_decorator.py :language: python :dedent: 4 :start-after: [START howto_operator_python_venv] @@ -283,7 +283,7 @@ overhead. Executes the task using a pre-installed Python interpreter — ideal for consistent environments or shared virtualenvs. -.. exampleinclude:: /../src/airflow/example_dags/example_python_decorator.py +.. exampleinclude:: /../../providers/standard/tests/system/standard/example_python_decorator.py :language: python :dedent: 4 :start-after: [START howto_operator_external_python] @@ -333,7 +333,7 @@ Using Sensors Use ``@task.sensor`` to build lightweight, reusable sensors using Python functions. These support both poke and reschedule modes. -.. exampleinclude:: /../src/airflow/example_dags/example_sensor_decorator.py +.. exampleinclude:: /../../providers/standard/tests/system/standard/example_sensor_decorator.py :language: python :start-after: [START tutorial] :end-before: [END tutorial] diff --git a/airflow-core/src/airflow/dag_processing/bundles/manager.py b/airflow-core/src/airflow/dag_processing/bundles/manager.py index 9760da617a1d9..1ace2896028c6 100644 --- a/airflow-core/src/airflow/dag_processing/bundles/manager.py +++ b/airflow-core/src/airflow/dag_processing/bundles/manager.py @@ -16,6 +16,7 @@ # under the License. from __future__ import annotations +import os from typing import TYPE_CHECKING from airflow.configuration import conf @@ -34,6 +35,7 @@ from airflow.dag_processing.bundles.base import BaseDagBundle _example_dag_bundle_name = "example_dags" +_example_standard_dag_bundle_name = "example_standard_dags" def _bundle_item_exc(msg): @@ -80,6 +82,25 @@ def _add_example_dag_bundle(config_list): ) +def _add_example_standard_dag_bundle(config_list): + # TODO(potiuk): make it more generic - for now we only add standard example_dags if they are locally available + try: + from system import standard + except ImportError: + return + + example_dag_folder = next(iter(standard.__path__)) + config_list.append( + { + "name": _example_standard_dag_bundle_name, + "classpath": "airflow.dag_processing.bundles.local.LocalDagBundle", + "kwargs": { + "path": example_dag_folder, + }, + } + ) + + class DagBundlesManager(LoggingMixin): """Manager for DAG bundles.""" @@ -112,6 +133,11 @@ def parse_config(self) -> None: _validate_bundle_config(config_list) if conf.getboolean("core", "LOAD_EXAMPLES"): _add_example_dag_bundle(config_list) + if ( + os.environ.get("BREEZE", "").lower() == "true" + or os.environ.get("_IN_UNIT_TESTS", "").lower() == "true" + ): + _add_example_standard_dag_bundle(config_list) for cfg in config_list: name = cfg["name"] diff --git a/airflow-core/src/airflow/models/dagbag.py b/airflow-core/src/airflow/models/dagbag.py index 52c2337474d74..d0c8bf98f4152 100644 --- a/airflow-core/src/airflow/models/dagbag.py +++ b/airflow-core/src/airflow/models/dagbag.py @@ -586,6 +586,14 @@ def collect_dags( example_dag_folder = next(iter(example_dags.__path__)) files_to_parse.extend(list_py_file_paths(example_dag_folder, safe_mode=safe_mode)) + try: + from system import standard + + example_dag_folder_standard = next(iter(standard.__path__)) + files_to_parse.extend(list_py_file_paths(example_dag_folder_standard, safe_mode=safe_mode)) + except ImportError: + # Nothing happens - this should only work during tests + pass for filepath in files_to_parse: try: diff --git a/airflow-core/src/airflow/utils/cli.py b/airflow-core/src/airflow/utils/cli.py index 86ef56da69ca4..8da5c24b81f74 100644 --- a/airflow-core/src/airflow/utils/cli.py +++ b/airflow-core/src/airflow/utils/cli.py @@ -270,7 +270,6 @@ def get_dag(bundle_names: list | None, dag_id: str, from_db: bool = False) -> DA bundle_names = bundle_names or [] dag: DAG | None = None - if from_db: dagbag = DagBag(read_dags_from_db=True) dag = dagbag.get_dag(dag_id) # get_dag loads from the DB as requested diff --git a/airflow-core/tests/unit/api_fastapi/core_api/routes/public/test_dag_parsing.py b/airflow-core/tests/unit/api_fastapi/core_api/routes/public/test_dag_parsing.py index 11ecc484bebf3..f64eef028e273 100644 --- a/airflow-core/tests/unit/api_fastapi/core_api/routes/public/test_dag_parsing.py +++ b/airflow-core/tests/unit/api_fastapi/core_api/routes/public/test_dag_parsing.py @@ -28,8 +28,8 @@ pytestmark = pytest.mark.db_test -EXAMPLE_DAG_FILE = AIRFLOW_CORE_SOURCES_PATH / "airflow" / "example_dags" / "example_bash_operator.py" -TEST_DAG_ID = "example_bash_operator" +EXAMPLE_DAG_FILE = AIRFLOW_CORE_SOURCES_PATH / "airflow" / "example_dags" / "example_simplest_dag.py" +TEST_DAG_ID = "example_simplest_dag" NOT_READABLE_DAG_ID = "latest_only_with_trigger" TEST_MULTIPLE_DAGS_ID = "asset_produces_1" diff --git a/airflow-core/tests/unit/api_fastapi/core_api/routes/public/test_dag_report.py b/airflow-core/tests/unit/api_fastapi/core_api/routes/public/test_dag_report.py index 3938894001de9..d1204bedd3e3d 100644 --- a/airflow-core/tests/unit/api_fastapi/core_api/routes/public/test_dag_report.py +++ b/airflow-core/tests/unit/api_fastapi/core_api/routes/public/test_dag_report.py @@ -23,6 +23,7 @@ from airflow.utils.file import list_py_file_paths +import system.standard from tests_common.test_utils.config import conf_vars from tests_common.test_utils.db import clear_db_dags, parse_and_sync_to_db @@ -37,8 +38,10 @@ def get_corresponding_dag_file_count(dir: str, include_examples: bool = True) -> int: from airflow import example_dags - return len(list_py_file_paths(directory=dir)) + ( - len(list_py_file_paths(next(iter(example_dags.__path__)))) if include_examples else 0 + return ( + len(list_py_file_paths(directory=dir)) + + (len(list_py_file_paths(next(iter(example_dags.__path__)))) if include_examples else 0) + + (len(list_py_file_paths(next(iter(system.standard.__path__)))) if include_examples else 0) ) diff --git a/airflow-core/tests/unit/api_fastapi/core_api/routes/public/test_dag_sources.py b/airflow-core/tests/unit/api_fastapi/core_api/routes/public/test_dag_sources.py index 90ea45ece7b46..cd223e67c0c08 100644 --- a/airflow-core/tests/unit/api_fastapi/core_api/routes/public/test_dag_sources.py +++ b/airflow-core/tests/unit/api_fastapi/core_api/routes/public/test_dag_sources.py @@ -39,9 +39,9 @@ # Example bash operator located here: airflow/example_dags/example_bash_operator.py EXAMPLE_DAG_FILE = ( - AIRFLOW_REPO_ROOT_PATH / "airflow-core" / "src" / "airflow" / "example_dags" / "example_bash_operator.py" + AIRFLOW_REPO_ROOT_PATH / "airflow-core" / "src" / "airflow" / "example_dags" / "example_simplest_dag.py" ) -TEST_DAG_ID = "example_bash_operator" +TEST_DAG_ID = "example_simplest_dag" @pytest.fixture diff --git a/airflow-core/tests/unit/cli/commands/test_task_command.py b/airflow-core/tests/unit/cli/commands/test_task_command.py index 9421583e417c7..d864ffb808fee 100644 --- a/airflow-core/tests/unit/cli/commands/test_task_command.py +++ b/airflow-core/tests/unit/cli/commands/test_task_command.py @@ -47,6 +47,7 @@ from airflow.utils.state import State, TaskInstanceState from airflow.utils.types import DagRunTriggeredByType, DagRunType +from tests_common.test_utils.config import conf_vars from tests_common.test_utils.db import clear_db_runs, parse_and_sync_to_db pytestmark = pytest.mark.db_test @@ -74,7 +75,6 @@ def move_back(old_path, new_path): shutil.move(new_path, old_path) -# TODO: Check if tests needs side effects - locally there's missing DAG class TestCliTasks: run_id = "TEST_RUN_ID" dag_id = "example_python_operator" @@ -91,7 +91,7 @@ def setup_class(cls): cls.parser = cli_parser.get_parser() clear_db_runs() - cls.dagbag = DagBag(read_dags_from_db=True) + cls.dagbag = DagBag(read_dags_from_db=True, include_examples=True) cls.dag = cls.dagbag.get_dag(cls.dag_id) data_interval = cls.dag.timetable.infer_manual_data_interval(run_after=DEFAULT_DATE) cls.dag_run = cls.dag.create_dagrun( @@ -108,6 +108,7 @@ def setup_class(cls): def teardown_class(cls) -> None: clear_db_runs() + @conf_vars({("core", "load_examples"): "true"}) @pytest.mark.execution_timeout(120) def test_cli_list_tasks(self): for dag_id in self.dagbag.dags: diff --git a/airflow-core/tests/unit/jobs/test_scheduler_job.py b/airflow-core/tests/unit/jobs/test_scheduler_job.py index 201490633de13..2f0d71ac89fc9 100644 --- a/airflow-core/tests/unit/jobs/test_scheduler_job.py +++ b/airflow-core/tests/unit/jobs/test_scheduler_job.py @@ -37,7 +37,6 @@ from sqlalchemy import func, select, update from sqlalchemy.orm import joinedload -import airflow.example_dags from airflow import settings from airflow.assets.manager import AssetManager from airflow.callbacks.callback_requests import DagCallbackRequest, TaskCallbackRequest @@ -75,6 +74,7 @@ from airflow.utils.thread_safe_dict import ThreadSafeDict from airflow.utils.types import DagRunTriggeredByType, DagRunType +from system import standard from tests_common.test_utils.asserts import assert_queries_count from tests_common.test_utils.config import conf_vars, env_vars from tests_common.test_utils.db import ( @@ -104,7 +104,7 @@ ELASTIC_DAG_FILE = os.path.join(PERF_DAGS_FOLDER, "elastic_dag.py") TEST_DAG_FOLDER = os.environ["AIRFLOW__CORE__DAGS_FOLDER"] -EXAMPLE_DAGS_FOLDER = airflow.example_dags.__path__[0] +EXAMPLE_STANDARD_DAGS_FOLDER = standard.__path__[0] DEFAULT_DATE = timezone.datetime(2016, 1, 1) DEFAULT_LOGICAL_DATE = timezone.coerce_datetime(DEFAULT_DATE) TRY_NUMBER = 1 @@ -5707,7 +5707,7 @@ def test_find_and_purge_task_instances_without_heartbeats_nothing(self): @pytest.mark.usefixtures("testing_dag_bundle") def test_find_and_purge_task_instances_without_heartbeats(self, session, create_dagrun): - dagfile = os.path.join(EXAMPLE_DAGS_FOLDER, "example_branch_operator.py") + dagfile = os.path.join(EXAMPLE_STANDARD_DAGS_FOLDER, "example_branch_operator.py") dagbag = DagBag(dagfile) dag = dagbag.get_dag("example_branch_operator") dm = LazyDeserializedDAG(data=SerializedDAG.to_dict(dag)) @@ -5773,7 +5773,7 @@ def test_task_instance_heartbeat_timeout_message(self, session, create_dagrun): """ Check that the task instance heartbeat timeout message comes out as expected """ - dagfile = os.path.join(EXAMPLE_DAGS_FOLDER, "example_branch_operator.py") + dagfile = os.path.join(EXAMPLE_STANDARD_DAGS_FOLDER, "example_branch_operator.py") dagbag = DagBag(dagfile) dag = dagbag.get_dag("example_branch_operator") dm = LazyDeserializedDAG(data=SerializedDAG.to_dict(dag)) diff --git a/airflow-core/tests/unit/models/test_dagbag.py b/airflow-core/tests/unit/models/test_dagbag.py index a4cc7da9e5bff..a9ba6669af96f 100644 --- a/airflow-core/tests/unit/models/test_dagbag.py +++ b/airflow-core/tests/unit/models/test_dagbag.py @@ -42,6 +42,7 @@ from airflow.serialization.serialized_objects import SerializedDAG from airflow.utils import timezone as tz from airflow.utils.session import create_session +from scripts.ci.pre_commit.common_precommit_utils import AIRFLOW_ROOT_PATH from tests_common.test_utils import db from tests_common.test_utils.asserts import assert_queries_count @@ -52,6 +53,12 @@ pytestmark = pytest.mark.db_test example_dags_folder = pathlib.Path(airflow.example_dags.__path__[0]) # type: ignore[attr-defined] +try: + import system.standard + + example_standard_dags_folder = pathlib.Path(system.standard.__path__[0]) # type: ignore[attr-defined] +except ImportError: + example_standard_dags_folder = pathlib.Path(airflow.example_dags.__path__[0]) # type: ignore[attr-defined] PY311 = sys.version_info >= (3, 11) @@ -359,13 +366,16 @@ def process_file(self, filepath, only_if_updated=True, safe_mode=True): ("file_to_load", "expected"), ( pytest.param( - pathlib.Path(example_dags_folder) / "example_bash_operator.py", - {"example_bash_operator": "airflow/example_dags/example_bash_operator.py"}, + pathlib.Path(example_standard_dags_folder) / "example_bash_operator.py", + { + "example_bash_operator": f"{example_standard_dags_folder.relative_to(AIRFLOW_ROOT_PATH) / 'example_bash_operator.py'}" + }, id="example_bash_operator", ), ), ) def test_get_dag_registration(self, file_to_load, expected): + pytest.importorskip("system.standard") dagbag = DagBag(dag_folder=os.devnull, include_examples=False) dagbag.process_file(os.fspath(file_to_load)) for dag_id, path in expected.items(): @@ -420,7 +430,7 @@ def test_refresh_py_dag(self, mock_dagmodel, tmp_path): Test that we can refresh an ordinary .py DAG """ dag_id = "example_bash_operator" - fileloc = str(example_dags_folder / "example_bash_operator.py") + fileloc = str(example_standard_dags_folder / "example_bash_operator.py") mock_dagmodel.return_value = DagModel() mock_dagmodel.return_value.last_expired = datetime.max.replace(tzinfo=timezone.utc) diff --git a/clients/python/test_python_client.py b/clients/python/test_python_client.py index 61bd3df6619de..d5979b40b9964 100644 --- a/clients/python/test_python_client.py +++ b/clients/python/test_python_client.py @@ -69,7 +69,7 @@ # Make sure in the [core] section, the `load_examples` config is set to True in your airflow.cfg # or AIRFLOW__CORE__LOAD_EXAMPLES environment variable set to True -DAG_ID = "example_bash_operator" +DAG_ID = "example_simplest_dag" # Enter a context with an instance of the API client diff --git a/devel-common/src/docs/provider_conf.py b/devel-common/src/docs/provider_conf.py index 16b0ad3c952c8..4b3f433ab763f 100644 --- a/devel-common/src/docs/provider_conf.py +++ b/devel-common/src/docs/provider_conf.py @@ -155,15 +155,8 @@ "operators/_partials", "_api/airflow/index.rst", "_api/airflow/providers/index.rst", - "_api/airflow/providers/apache/index.rst", - "_api/airflow/providers/atlassian/index.rst", - "_api/airflow/providers/cncf/index.rst", - "_api/airflow/providers/common/index.rst", - "_api/airflow/providers/common/messaging/providers/base_provider/index.rst", - "_api/airflow/providers/common/messaging/providers/sqs/index.rst", - "_api/airflow/providers/dbt/index.rst", - "_api/airflow/providers/microsoft/index.rst", "_api/docs/conf", + *[f"_api/airflow/providers/{subpackage}/index.rst" for subpackage in empty_subpackages], *[f"_api/system/{subpackage}/index.rst" for subpackage in empty_subpackages], *[f"_api/tests/system/{subpackage}/index.rst" for subpackage in empty_subpackages], ] diff --git a/devel-common/src/sphinx_exts/docs_build/docs_builder.py b/devel-common/src/sphinx_exts/docs_build/docs_builder.py index 1a5d46c6d7f60..44ba83e6f7a3d 100644 --- a/devel-common/src/sphinx_exts/docs_build/docs_builder.py +++ b/devel-common/src/sphinx_exts/docs_build/docs_builder.py @@ -120,6 +120,13 @@ def _src_dir(self) -> Path: console.print(f"[red]Unknown package name: {self.package_name}") sys.exit(1) + @property + def pythonpath(self) -> list[Path]: + path = [] + if (self._src_dir.parent / "tests").exists(): + path.append(self._src_dir.parent.joinpath("tests").resolve()) + return path + @property def _generated_api_dir(self) -> Path: return self._build_dir.resolve() / "_api" @@ -167,6 +174,8 @@ def check_spelling(self, verbose: bool) -> tuple[list[SpellingError], list[DocBu console.print("[yellow]Command to run:[/] ", " ".join([shlex.quote(arg) for arg in build_cmd])) env = os.environ.copy() env["AIRFLOW_PACKAGE_NAME"] = self.package_name + if self.pythonpath: + env["PYTHONPATH"] = ":".join([path.as_posix() for path in self.pythonpath]) if verbose: console.print( f"[bright_blue]{self.package_name:60}:[/] The output is hidden until an error occurs." @@ -246,6 +255,8 @@ def build_sphinx_docs(self, verbose: bool) -> list[DocBuildError]: console.print("[yellow]Command to run:[/] ", " ".join([shlex.quote(arg) for arg in build_cmd])) env = os.environ.copy() env["AIRFLOW_PACKAGE_NAME"] = self.package_name + if self.pythonpath: + env["PYTHONPATH"] = ":".join([path.as_posix() for path in self.pythonpath]) if verbose: console.print( f"[bright_blue]{self.package_name:60}:[/] Running sphinx. " diff --git a/devel-common/src/sphinx_exts/exampleinclude.py b/devel-common/src/sphinx_exts/exampleinclude.py index decf049d83a0b..3feaed367c35a 100644 --- a/devel-common/src/sphinx_exts/exampleinclude.py +++ b/devel-common/src/sphinx_exts/exampleinclude.py @@ -17,8 +17,8 @@ from __future__ import annotations """Nice formatted include for examples""" -import os import traceback +from pathlib import Path from docutils import nodes @@ -139,6 +139,7 @@ def register_source(app, env, modname): """ if modname is None: return False + entry = env._viewcode_modules.get(modname, None) if entry is False: print(f"[{modname}] Entry is false for ") @@ -227,6 +228,7 @@ def doctree_read(app, doctree): :return None """ + env = app.builder.env if not hasattr(env, "_viewcode_modules"): env._viewcode_modules = {} @@ -235,19 +237,23 @@ def doctree_read(app, doctree): return for objnode in doctree.traverse(ExampleHeader): - filepath = objnode.get("filename") - relative_path = os.path.relpath( - filepath, os.path.commonprefix([app.config.exampleinclude_sourceroot, filepath]) - ) - if relative_path.endswith(".py"): - modname = relative_path.replace("/", ".")[-3] - split_modname = modname.split(".") + source_root_path = Path(app.config.exampleinclude_sourceroot) + filepath = Path(objnode.get("filename")) + if filepath.is_relative_to(source_root_path) and filepath.name.endswith(".py"): + module_path = filepath.relative_to(source_root_path) + split_modname = module_path.parts if "src" in split_modname: modname = ".".join(split_modname[split_modname.index("src") + 1 :]) + elif "tests" in split_modname: + modname = ".".join(split_modname[split_modname.index("tests") + 1 :]) + else: + modname = ".".join(split_modname) + modname = modname.replace(".py", "") else: modname = None + module_path = filepath.resolve() show_button = register_source(app, env, modname) - onlynode = create_node(env, relative_path, show_button) + onlynode = create_node(env, module_path.as_posix(), show_button) objnode.replace_self(onlynode) diff --git a/devel-common/src/tests_common/pytest_plugin.py b/devel-common/src/tests_common/pytest_plugin.py index dff39c5d7a067..a86377b5ac3a7 100644 --- a/devel-common/src/tests_common/pytest_plugin.py +++ b/devel-common/src/tests_common/pytest_plugin.py @@ -135,7 +135,6 @@ # Make sure sqlalchemy will not be usable for pure unit tests even if initialized os.environ["AIRFLOW__CORE__SQL_ALCHEMY_CONN"] = "bad_schema:///" os.environ["AIRFLOW__DATABASE__SQL_ALCHEMY_CONN"] = "bad_schema:///" - os.environ["_IN_UNIT_TESTS"] = "true" # Set it here to pass the flag to python-xdist spawned processes os.environ["_AIRFLOW_SKIP_DB_TESTS"] = "true" @@ -143,6 +142,8 @@ # Set it here to pass the flag to python-xdist spawned processes os.environ["_AIRFLOW_RUN_DB_TESTS_ONLY"] = "true" +os.environ["_IN_UNIT_TESTS"] = "true" + _airflow_sources = os.getenv("AIRFLOW_SOURCES", None) AIRFLOW_ROOT_PATH = (Path(_airflow_sources) if _airflow_sources else Path(__file__).parents[3]).resolve() AIRFLOW_CORE_SOURCES_PATH = AIRFLOW_ROOT_PATH / "airflow-core" / "src" diff --git a/docker-stack-docs/conf.py b/docker-stack-docs/conf.py index 5d0e9f06869b7..c42e311844cb6 100644 --- a/docker-stack-docs/conf.py +++ b/docker-stack-docs/conf.py @@ -222,7 +222,7 @@ suppress_warnings = SUPPRESS_WARNINGS # -- Options for ext.exampleinclude -------------------------------------------- -exampleinclude_sourceroot = os.path.abspath("..") +exampleinclude_sourceroot = os.path.abspath(".") # -- Options for ext.redirects ------------------------------------------------- redirects_file = "redirects.txt" diff --git a/docker-tests/tests/docker_tests/test_docker_compose_quick_start.py b/docker-tests/tests/docker_tests/test_docker_compose_quick_start.py index 789a0f734294f..6241e49d15878 100644 --- a/docker-tests/tests/docker_tests/test_docker_compose_quick_start.py +++ b/docker-tests/tests/docker_tests/test_docker_compose_quick_start.py @@ -41,7 +41,7 @@ DOCKER_COMPOSE_HOST_PORT = os.environ.get("HOST_PORT", "localhost:8080") AIRFLOW_WWW_USER_USERNAME = os.environ.get("_AIRFLOW_WWW_USER_USERNAME", "airflow") AIRFLOW_WWW_USER_PASSWORD = os.environ.get("_AIRFLOW_WWW_USER_PASSWORD", "airflow") -DAG_ID = "example_bash_operator" +DAG_ID = "example_simplest_dag" DAG_RUN_ID = "test_dag_run_id" diff --git a/kubernetes-tests/tests/kubernetes_tests/test_other_executors.py b/kubernetes-tests/tests/kubernetes_tests/test_other_executors.py index 7b49bf9f5639b..28bad807da01e 100644 --- a/kubernetes-tests/tests/kubernetes_tests/test_other_executors.py +++ b/kubernetes-tests/tests/kubernetes_tests/test_other_executors.py @@ -30,7 +30,7 @@ @pytest.mark.skipif(EXECUTOR == "KubernetesExecutor", reason="Does not run on KubernetesExecutor") class TestCeleryAndLocalExecutor(BaseK8STest): def test_integration_run_dag(self): - dag_id = "example_bash_operator" + dag_id = "example_simplest_dag" dag_run_id, logical_date = self.start_job_in_kubernetes(dag_id, self.host) print(f"Found the job with logical_date {logical_date}") @@ -39,7 +39,7 @@ def test_integration_run_dag(self): host=self.host, dag_run_id=dag_run_id, dag_id=dag_id, - task_id="run_after_loop", + task_id="my_task", expected_final_state="success", timeout=300, ) diff --git a/providers-summary-docs/conf.py b/providers-summary-docs/conf.py index 85dfc4c1e1e47..2219f0bcb8ab6 100644 --- a/providers-summary-docs/conf.py +++ b/providers-summary-docs/conf.py @@ -241,7 +241,7 @@ suppress_warnings = SUPPRESS_WARNINGS # -- Options for ext.exampleinclude -------------------------------------------- -exampleinclude_sourceroot = os.path.abspath("..") +exampleinclude_sourceroot = os.path.abspath(".") # -- Options for ext.redirects ------------------------------------------------- redirects_file = "redirects.txt" diff --git a/providers/common/messaging/docs/index.rst b/providers/common/messaging/docs/index.rst index 10a349a362bad..0f50a2c2eb39d 100644 --- a/providers/common/messaging/docs/index.rst +++ b/providers/common/messaging/docs/index.rst @@ -43,6 +43,8 @@ :caption: References Python API <_api/airflow/providers/common/messaging/index> + Base Provider <_api/airflow/providers/common/messaging/providers/base_provider/index> + SQS provider <_api/airflow/providers/common/messaging/providers/sqs/index> .. toctree:: :hidden: diff --git a/providers/standard/docs/index.rst b/providers/standard/docs/index.rst index 43cf71f2c1f0e..5ddaa7eef2484 100644 --- a/providers/standard/docs/index.rst +++ b/providers/standard/docs/index.rst @@ -38,17 +38,24 @@ Sensors Configuration +.. toctree:: + :hidden: + :maxdepth: 1 + :caption: System tests + + System Tests <_api/tests/system/standard/index> + .. toctree:: :hidden: :maxdepth: 1 :caption: Resources + Example DAGs PyPI Repository Installing from sources Python API <_api/airflow/providers/standard/index> - .. THE REMAINDER OF THE FILE IS AUTOMATICALLY GENERATED. IT WILL BE OVERWRITTEN AT RELEASE TIME! diff --git a/providers/standard/docs/operators/bash.rst b/providers/standard/docs/operators/bash.rst index 113ca81e246d9..0831403020c99 100644 --- a/providers/standard/docs/operators/bash.rst +++ b/providers/standard/docs/operators/bash.rst @@ -41,7 +41,7 @@ determined by: .. tab-item:: @task.bash :sync: taskflow - .. exampleinclude:: /../../../airflow-core/src/airflow/example_dags/example_bash_decorator.py + .. exampleinclude:: /../tests/system/standard/example_bash_decorator.py :language: python :dedent: 4 :start-after: [START howto_decorator_bash] @@ -50,7 +50,7 @@ determined by: .. tab-item:: BashOperator :sync: operator - .. exampleinclude:: /../../../airflow-core/src/airflow/example_dags/example_bash_operator.py + .. exampleinclude:: /../tests/system/standard/example_bash_operator.py :language: python :dedent: 4 :start-after: [START howto_operator_bash] @@ -67,7 +67,7 @@ You can use :ref:`Jinja templates ` to parameterize t .. tab-item:: @task.bash :sync: taskflow - .. exampleinclude:: /../../../airflow-core/src/airflow/example_dags/example_bash_decorator.py + .. exampleinclude:: /../tests/system/standard/example_bash_decorator.py :language: python :dedent: 4 :start-after: [START howto_decorator_bash_template] @@ -76,7 +76,7 @@ You can use :ref:`Jinja templates ` to parameterize t .. tab-item:: BashOperator :sync: operator - .. exampleinclude:: /../../../airflow-core/src/airflow/example_dags/example_bash_operator.py + .. exampleinclude:: /../tests/system/standard/example_bash_operator.py :language: python :dedent: 4 :start-after: [START howto_operator_bash_template] @@ -85,7 +85,7 @@ You can use :ref:`Jinja templates ` to parameterize t Using the ``@task.bash`` TaskFlow decorator allows you to return a formatted string and take advantage of having all :ref:`execution context variables directly accessible to decorated tasks `. -.. exampleinclude:: /../../../airflow-core/src/airflow/example_dags/example_bash_decorator.py +.. exampleinclude:: /../tests/system/standard/example_bash_decorator.py :language: python :dedent: 4 :start-after: [START howto_decorator_bash_context_vars] @@ -169,7 +169,7 @@ exit code if you pass ``skip_on_exit_code``). .. tab-item:: @task.bash :sync: taskflow - .. exampleinclude:: /../../../airflow-core/src/airflow/example_dags/example_bash_decorator.py + .. exampleinclude:: /../tests/system/standard/example_bash_decorator.py :language: python :dedent: 4 :start-after: [START howto_decorator_bash_skip] @@ -178,7 +178,7 @@ exit code if you pass ``skip_on_exit_code``). .. tab-item:: BashOperator :sync: operator - .. exampleinclude:: /../../../airflow-core/src/airflow/example_dags/example_bash_operator.py + .. exampleinclude:: /../tests/system/standard/example_bash_operator.py :language: python :start-after: [START howto_operator_bash_skip] :end-before: [END howto_operator_bash_skip] @@ -388,7 +388,7 @@ or even build the Bash command(s) to execute. For example, use conditional logic to determine task behavior: -.. exampleinclude:: /../../../airflow-core/src/airflow/example_dags/example_bash_decorator.py +.. exampleinclude:: /../tests/system/standard/example_bash_decorator.py :language: python :dedent: 4 :start-after: [START howto_decorator_bash_conditional] @@ -396,7 +396,7 @@ For example, use conditional logic to determine task behavior: Or call a function to help build a Bash command: -.. exampleinclude:: /../../../airflow-core/src/airflow/example_dags/example_bash_decorator.py +.. exampleinclude:: /../tests/system/standard/example_bash_decorator.py :language: python :dedent: 4 :start-after: [START howto_decorator_bash_build_cmd] diff --git a/providers/standard/docs/operators/datetime.rst b/providers/standard/docs/operators/datetime.rst index ea8b9d3b31fa8..251e933cbad40 100644 --- a/providers/standard/docs/operators/datetime.rst +++ b/providers/standard/docs/operators/datetime.rst @@ -39,7 +39,7 @@ take some time between when the DAGRun was scheduled and executed and it might m the DAGRun was scheduled properly, the actual time used for branching decision will be different than the schedule time and the branching decision might be different depending on those delays. -.. exampleinclude:: /../../../airflow-core/src/airflow/example_dags/example_branch_datetime_operator.py +.. exampleinclude:: /../tests/system/standard/example_branch_datetime_operator.py :language: python :start-after: [START howto_branch_datetime_operator] :end-before: [END howto_branch_datetime_operator] @@ -50,7 +50,7 @@ the current date in order to allow comparisons with it. In the event that ``targ to a ``datetime.time`` that occurs before the given ``target_lower``, a day will be added to ``target_upper``. This is done to allow for time periods that span over two dates. -.. exampleinclude:: /../../../airflow-core/src/airflow/example_dags/example_branch_datetime_operator.py +.. exampleinclude:: /../tests/system/standard/example_branch_datetime_operator.py :language: python :start-after: [START howto_branch_datetime_operator_next_day] :end-before: [END howto_branch_datetime_operator_next_day] @@ -66,7 +66,7 @@ The usage is much more "data range" friendly. The ``logical_date`` does not chan it is not affected by execution delays, so this approach is suitable for idempotent DAG runs that might be back-filled. -.. exampleinclude:: /../../../airflow-core/src/airflow/example_dags/example_branch_datetime_operator.py +.. exampleinclude:: /../tests/system/standard/example_branch_datetime_operator.py :language: python :start-after: [START howto_branch_datetime_operator_logical_date] :end-before: [END howto_branch_datetime_operator_logical_date] @@ -78,7 +78,7 @@ BranchDayOfWeekOperator Use the :class:`~airflow.operators.weekday.BranchDayOfWeekOperator` to branch your workflow based on week day value. -.. exampleinclude:: /../../../airflow-core/src/airflow/example_dags/example_branch_day_of_week_operator.py +.. exampleinclude:: /../tests/system/standard/example_branch_day_of_week_operator.py :language: python :dedent: 4 :start-after: [START howto_operator_day_of_week_branch] diff --git a/providers/standard/docs/operators/latest_only.rst b/providers/standard/docs/operators/latest_only.rst index d3fc30937a2ed..21e66a124917e 100644 --- a/providers/standard/docs/operators/latest_only.rst +++ b/providers/standard/docs/operators/latest_only.rst @@ -24,7 +24,7 @@ LatestOnlyOperator Use the :class:`~airflow.providers.standard.operators.latest_only.LatestOnlyOperator`. -.. exampleinclude:: /../../../airflow-core/src/airflow/example_dags/example_latest_only.py +.. exampleinclude:: /../tests/system/standard/example_latest_only.py :language: python :dedent: 4 :start-after: [START howto_operator_latest_only] diff --git a/providers/standard/docs/operators/python.rst b/providers/standard/docs/operators/python.rst index adb7f55828619..8f6f8d7d68159 100644 --- a/providers/standard/docs/operators/python.rst +++ b/providers/standard/docs/operators/python.rst @@ -32,7 +32,7 @@ Use the :class:`~airflow.providers.standard.operators.python.PythonOperator` to .. tab-item:: @task :sync: taskflow - .. exampleinclude:: /../../../airflow-core/src/airflow/example_dags/example_python_decorator.py + .. exampleinclude:: /../tests/system/standard/example_python_decorator.py :language: python :dedent: 4 :start-after: [START howto_operator_python] @@ -41,7 +41,7 @@ Use the :class:`~airflow.providers.standard.operators.python.PythonOperator` to .. tab-item:: PythonOperator :sync: operator - .. exampleinclude:: /../../../airflow-core/src/airflow/example_dags/example_python_operator.py + .. exampleinclude:: /../tests/system/standard/example_python_operator.py :language: python :dedent: 4 :start-after: [START howto_operator_python] @@ -57,7 +57,7 @@ Pass extra arguments to the ``@task`` decorated function as you would with a nor .. tab-item:: @task :sync: taskflow - .. exampleinclude:: /../../../airflow-core/src/airflow/example_dags/example_python_decorator.py + .. exampleinclude:: /../tests/system/standard/example_python_decorator.py :language: python :dedent: 4 :start-after: [START howto_operator_python_kwargs] @@ -66,7 +66,7 @@ Pass extra arguments to the ``@task`` decorated function as you would with a nor .. tab-item:: PythonOperator :sync: operator - .. exampleinclude:: /../../../airflow-core/src/airflow/example_dags/example_python_operator.py + .. exampleinclude:: /../tests/system/standard/example_python_operator.py :language: python :dedent: 4 :start-after: [START howto_operator_python_kwargs] @@ -87,7 +87,7 @@ is evaluated as a :ref:`Jinja template `. .. tab-item:: @task :sync: taskflow - .. exampleinclude:: /../../../airflow-core/src/airflow/example_dags/example_python_decorator.py + .. exampleinclude:: /../tests/system/standard/example_python_decorator.py :language: python :dedent: 4 :start-after: [START howto_operator_python_render_sql] @@ -96,7 +96,7 @@ is evaluated as a :ref:`Jinja template `. .. tab-item:: PythonOperator :sync: operator - .. exampleinclude:: /../../../airflow-core/src/airflow/example_dags/example_python_operator.py + .. exampleinclude:: /../tests/system/standard/example_python_operator.py :language: python :dedent: 4 :start-after: [START howto_operator_python_render_sql] @@ -137,7 +137,7 @@ smoother data exchange, while still effectively handling common Python objects a .. tab-item:: @task.virtualenv :sync: taskflow - .. exampleinclude:: /../../../airflow-core/src/airflow/example_dags/example_python_decorator.py + .. exampleinclude:: /../tests/system/standard/example_python_decorator.py :language: python :dedent: 4 :start-after: [START howto_operator_python_venv] @@ -146,7 +146,7 @@ smoother data exchange, while still effectively handling common Python objects a .. tab-item:: PythonVirtualenvOperator :sync: operator - .. exampleinclude:: /../../../airflow-core/src/airflow/example_dags/example_python_operator.py + .. exampleinclude:: /../tests/system/standard/example_python_operator.py :language: python :dedent: 4 :start-after: [START howto_operator_python_venv] @@ -251,7 +251,7 @@ in main Airflow environment). .. tab-item:: @task.external_python :sync: taskflow - .. exampleinclude:: /../../../airflow-core/src/airflow/example_dags/example_python_decorator.py + .. exampleinclude:: /../tests/system/standard/example_python_decorator.py :language: python :dedent: 4 :start-after: [START howto_operator_external_python] @@ -260,7 +260,7 @@ in main Airflow environment). .. tab-item:: ExternalPythonOperator :sync: operator - .. exampleinclude:: /../../../airflow-core/src/airflow/example_dags/example_python_operator.py + .. exampleinclude:: /../tests/system/standard/example_python_operator.py :language: python :dedent: 4 :start-after: [START howto_operator_external_python] @@ -308,7 +308,7 @@ tasks. .. tab-item:: @task.branch :sync: taskflow - .. exampleinclude:: /../../../airflow-core/src/airflow/example_dags/example_branch_operator_decorator.py + .. exampleinclude:: /../tests/system/standard/example_branch_operator_decorator.py :language: python :dedent: 4 :start-after: [START howto_operator_branch_python] @@ -317,7 +317,7 @@ tasks. .. tab-item:: PythonBranchOperator :sync: operator - .. exampleinclude:: /../../../airflow-core/src/airflow/example_dags/example_branch_operator.py + .. exampleinclude:: /../tests/system/standard/example_branch_operator.py :language: python :dedent: 4 :start-after: [START howto_operator_branch_python] @@ -345,7 +345,7 @@ tasks and is a hybrid of the :class:`~airflow.providers.standard.operators.pytho .. tab-item:: @task.branch_virtualenv :sync: taskflow - .. exampleinclude:: /../../../airflow-core/src/airflow/example_dags/example_branch_operator_decorator.py + .. exampleinclude:: /../tests/system/standard/example_branch_operator_decorator.py :language: python :dedent: 4 :start-after: [START howto_operator_branch_virtualenv] @@ -354,7 +354,7 @@ tasks and is a hybrid of the :class:`~airflow.providers.standard.operators.pytho .. tab-item:: BranchPythonVirtualenvOperator :sync: operator - .. exampleinclude:: /../../../airflow-core/src/airflow/example_dags/example_branch_operator.py + .. exampleinclude:: /../tests/system/standard/example_branch_operator.py :language: python :dedent: 4 :start-after: [START howto_operator_branch_virtualenv] @@ -383,7 +383,7 @@ external Python environment. .. tab-item:: @task.branch_external_python :sync: taskflow - .. exampleinclude:: /../../../airflow-core/src/airflow/example_dags/example_branch_operator_decorator.py + .. exampleinclude:: /../tests/system/standard/example_branch_operator_decorator.py :language: python :dedent: 4 :start-after: [START howto_operator_branch_ext_py] @@ -392,7 +392,7 @@ external Python environment. .. tab-item:: BranchExternalPythonOperator :sync: operator - .. exampleinclude:: /../../../airflow-core/src/airflow/example_dags/example_branch_operator.py + .. exampleinclude:: /../tests/system/standard/example_branch_operator.py :language: python :dedent: 4 :start-after: [START howto_operator_branch_ext_py] @@ -428,7 +428,7 @@ skipped. .. tab-item:: @task.short_circuit :sync: taskflow - .. exampleinclude:: /../../../airflow-core/src/airflow/example_dags/example_short_circuit_decorator.py + .. exampleinclude:: /../tests/system/standard/example_short_circuit_decorator.py :language: python :dedent: 4 :start-after: [START howto_operator_short_circuit] @@ -437,7 +437,7 @@ skipped. .. tab-item:: ShortCircuitOperator :sync: operator - .. exampleinclude:: /../../../airflow-core/src/airflow/example_dags/example_short_circuit_operator.py + .. exampleinclude:: /../tests/system/standard/example_short_circuit_operator.py :language: python :dedent: 4 :start-after: [START howto_operator_short_circuit] @@ -463,7 +463,7 @@ tasks have completed running regardless of status (i.e. the ``TriggerRule.ALL_DO .. tab-item:: @task.short_circuit :sync: taskflow - .. exampleinclude:: /../../../airflow-core/src/airflow/example_dags/example_short_circuit_decorator.py + .. exampleinclude:: /../tests/system/standard/example_short_circuit_decorator.py :language: python :dedent: 4 :start-after: [START howto_operator_short_circuit_trigger_rules] @@ -472,7 +472,7 @@ tasks have completed running regardless of status (i.e. the ``TriggerRule.ALL_DO .. tab-item:: ShortCircuitOperator :sync: operator - .. exampleinclude:: /../../../airflow-core/src/airflow/example_dags/example_short_circuit_operator.py + .. exampleinclude:: /../tests/system/standard/example_short_circuit_operator.py :language: python :dedent: 4 :start-after: [START howto_operator_short_circuit_trigger_rules] diff --git a/providers/standard/docs/operators/trigger_dag_run.rst b/providers/standard/docs/operators/trigger_dag_run.rst index b23e79f0ff34e..1db20b35eceef 100644 --- a/providers/standard/docs/operators/trigger_dag_run.rst +++ b/providers/standard/docs/operators/trigger_dag_run.rst @@ -24,7 +24,7 @@ TriggerDagRunOperator Use the :class:`~airflow.providers.standard.operators.trigger_dagrun.TriggerDagRunOperator` to trigger dag from another dag. -.. exampleinclude:: /../../../airflow-core/src/airflow/example_dags/example_trigger_controller_dag.py +.. exampleinclude:: /../tests/system/standard/example_trigger_controller_dag.py :language: python :dedent: 4 :start-after: [START howto_operator_trigger_dagrun] diff --git a/providers/standard/docs/sensors/bash.rst b/providers/standard/docs/sensors/bash.rst index 2fdb32419cc66..5acabb1172cf1 100644 --- a/providers/standard/docs/sensors/bash.rst +++ b/providers/standard/docs/sensors/bash.rst @@ -25,7 +25,7 @@ BashSensor Use the :class:`~airflow.providers.standard.sensors.bash.BashSensor` to use arbitrary command for sensing. The command should return 0 when it succeeds, any other value otherwise. -.. exampleinclude:: /../../../airflow-core/src/airflow/example_dags/example_sensors.py +.. exampleinclude:: /../tests/system/standard/example_sensors.py :language: python :dedent: 4 :start-after: [START example_bash_sensors] diff --git a/providers/standard/docs/sensors/datetime.rst b/providers/standard/docs/sensors/datetime.rst index b222650b57f82..aabff0f866dce 100644 --- a/providers/standard/docs/sensors/datetime.rst +++ b/providers/standard/docs/sensors/datetime.rst @@ -25,7 +25,7 @@ TimeDeltaSensor Use the :class:`~airflow.providers.standard.sensors.time_delta.TimeDeltaSensor` to end sensing after specific time. -.. exampleinclude:: /../../../airflow-core/src/airflow/example_dags/example_sensors.py +.. exampleinclude:: /../tests/system/standard/example_sensors.py :language: python :dedent: 4 :start-after: [START example_time_delta_sensor] @@ -41,7 +41,7 @@ Use the :class:`~airflow.providers.standard.sensors.time_delta.TimeDeltaSensorAs It is an async version of the operator and requires Triggerer to run. -.. exampleinclude:: /../../../airflow-core/src/airflow/example_dags/example_sensors.py +.. exampleinclude:: /../tests/system/standard/example_sensors.py :language: python :dedent: 4 :start-after: [START example_time_delta_sensor_async] @@ -58,7 +58,7 @@ Use the :class:`~airflow.providers.standard.sensors.time_sensor.TimeSensor` to e Time will be evaluated against ``data_interval_end`` if present for the dag run, otherwise ``run_after`` will be used. -.. exampleinclude:: /../../../airflow-core/src/airflow/example_dags/example_sensors.py +.. exampleinclude:: /../tests/system/standard/example_sensors.py :language: python :dedent: 4 :start-after: [START example_time_sensors] @@ -75,7 +75,7 @@ It is an async version of the operator and requires Triggerer to run. Time will be evaluated against ``data_interval_end`` if present for the dag run, otherwise ``run_after`` will be used. -.. exampleinclude:: /../../../airflow-core/src/airflow/example_dags/example_sensors.py +.. exampleinclude:: /../tests/system/standard/example_sensors.py :language: python :dedent: 4 :start-after: [START example_time_sensors_async] @@ -88,7 +88,7 @@ DayOfWeekSensor Use the :class:`~airflow.sensors.weekday.DayOfWeekSensor` to sense for day of week. -.. exampleinclude:: /../../../airflow-core/src/airflow/example_dags/example_sensors.py +.. exampleinclude:: /../tests/system/standard/example_sensors.py :language: python :dedent: 4 :start-after: [START example_day_of_week_sensor] diff --git a/providers/standard/docs/sensors/external_task_sensor.rst b/providers/standard/docs/sensors/external_task_sensor.rst index 30cbe4fdf0f1f..8aec0bb785891 100644 --- a/providers/standard/docs/sensors/external_task_sensor.rst +++ b/providers/standard/docs/sensors/external_task_sensor.rst @@ -47,7 +47,7 @@ wait for another task on a different DAG for a specific ``execution_date``. ExternalTaskSensor also provide options to set if the Task on a remote DAG succeeded or failed via ``allowed_states`` and ``failed_states`` parameters. -.. exampleinclude:: /../../../airflow-core/src/airflow/example_dags/example_external_task_marker_dag.py +.. exampleinclude:: /../tests/system/standard/example_external_task_marker_dag.py :language: python :dedent: 4 :start-after: [START howto_operator_external_task_sensor] @@ -55,7 +55,7 @@ via ``allowed_states`` and ``failed_states`` parameters. Also for this action you can use sensor in the deferrable mode: -.. exampleinclude:: /../../../airflow-core/tests/system/core/example_external_task_parent_deferrable.py +.. exampleinclude:: /../tests/system/standard/example_external_task_parent_deferrable.py :language: python :dedent: 4 :start-after: [START howto_external_task_async_sensor] @@ -67,7 +67,7 @@ ExternalTaskSensor with task_group dependency In Addition, we can also use the :class:`~airflow.providers.standard.sensors.external_task.ExternalTaskSensor` to make tasks on a DAG wait for another ``task_group`` on a different DAG for a specific ``execution_date``. -.. exampleinclude:: /../../../airflow-core/src/airflow/example_dags/example_external_task_marker_dag.py +.. exampleinclude:: /../tests/system/standard/example_external_task_marker_dag.py :language: python :dedent: 4 :start-after: [START howto_operator_external_task_sensor_with_task_group] @@ -81,7 +81,7 @@ on ``child_dag`` for a specific ``execution_date`` should also be cleared, ``Ext should be used. Note that ``child_task1`` will only be cleared if "Recursive" is selected when the user clears ``parent_task``. -.. exampleinclude:: /../../../airflow-core/src/airflow/example_dags/example_external_task_marker_dag.py +.. exampleinclude:: /../tests/system/standard/example_external_task_marker_dag.py :language: python :dedent: 4 :start-after: [START howto_operator_external_task_marker] diff --git a/providers/standard/docs/sensors/file.rst b/providers/standard/docs/sensors/file.rst index 03b4d57722b70..a2f2b42ce3fe7 100644 --- a/providers/standard/docs/sensors/file.rst +++ b/providers/standard/docs/sensors/file.rst @@ -26,7 +26,7 @@ Use the :class:`~airflow.providers.standard.sensors.filesystem.FileSensor` to de filesystem. You need to have connection defined to use it (pass connection id via ``fs_conn_id``). Default connection is ``fs_default``. -.. exampleinclude:: /../../../airflow-core/src/airflow/example_dags/example_sensors.py +.. exampleinclude:: /../tests/system/standard/example_sensors.py :language: python :dedent: 4 :start-after: [START example_file_sensor] @@ -34,7 +34,7 @@ Default connection is ``fs_default``. Also for this job you can use sensor in the deferrable mode: -.. exampleinclude:: /../../../airflow-core/src/airflow/example_dags/example_sensors.py +.. exampleinclude:: /../tests/system/standard/example_sensors.py :language: python :dedent: 4 :start-after: [START example_file_sensor_async] diff --git a/providers/standard/docs/sensors/python.rst b/providers/standard/docs/sensors/python.rst index 2702ef6408fe0..1850a272d3050 100644 --- a/providers/standard/docs/sensors/python.rst +++ b/providers/standard/docs/sensors/python.rst @@ -34,7 +34,7 @@ value to be True. .. tab-item:: @task.sensor :sync: taskflow - .. exampleinclude:: /../../../airflow-core/src/airflow/example_dags/example_sensor_decorator.py + .. exampleinclude:: /../tests/system/standard/example_sensor_decorator.py :language: python :dedent: 4 :start-after: [START wait_function] @@ -43,7 +43,7 @@ value to be True. .. tab-item:: PythonSensor :sync: operator - .. exampleinclude:: /../../../airflow-core/src/airflow/example_dags/example_sensors.py + .. exampleinclude:: /../tests/system/standard/example_sensors.py :language: python :dedent: 4 :start-after: [START example_python_sensors] diff --git a/providers/standard/tests/system/__init__.py b/providers/standard/tests/system/__init__.py new file mode 100644 index 0000000000000..e8fd22856438c --- /dev/null +++ b/providers/standard/tests/system/__init__.py @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +__path__ = __import__("pkgutil").extend_path(__path__, __name__) # type: ignore diff --git a/providers/standard/tests/system/standard/__init__.py b/providers/standard/tests/system/standard/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/providers/standard/tests/system/standard/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/airflow-core/src/airflow/example_dags/example_bash_decorator.py b/providers/standard/tests/system/standard/example_bash_decorator.py similarity index 95% rename from airflow-core/src/airflow/example_dags/example_bash_decorator.py rename to providers/standard/tests/system/standard/example_bash_decorator.py index 89335c0686465..ce95b62ce6f4c 100644 --- a/airflow-core/src/airflow/example_dags/example_bash_decorator.py +++ b/providers/standard/tests/system/standard/example_bash_decorator.py @@ -112,3 +112,9 @@ def get_file_stats() -> str: example_bash_decorator() + + +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 + +# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +test_run = get_test_run(dag) diff --git a/airflow-core/src/airflow/example_dags/example_bash_operator.py b/providers/standard/tests/system/standard/example_bash_operator.py similarity index 92% rename from airflow-core/src/airflow/example_dags/example_bash_operator.py rename to providers/standard/tests/system/standard/example_bash_operator.py index a4fb3161ab67d..7ef7c0db2efbe 100644 --- a/airflow-core/src/airflow/example_dags/example_bash_operator.py +++ b/providers/standard/tests/system/standard/example_bash_operator.py @@ -72,3 +72,9 @@ ) # [END howto_operator_bash_skip] this_will_skip >> run_this_last + + +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 + +# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +test_run = get_test_run(dag) diff --git a/airflow-core/src/airflow/example_dags/example_branch_datetime_operator.py b/providers/standard/tests/system/standard/example_branch_datetime_operator.py similarity index 93% rename from airflow-core/src/airflow/example_dags/example_branch_datetime_operator.py rename to providers/standard/tests/system/standard/example_branch_datetime_operator.py index 710bdb1de898c..b1e7564133323 100644 --- a/airflow-core/src/airflow/example_dags/example_branch_datetime_operator.py +++ b/providers/standard/tests/system/standard/example_branch_datetime_operator.py @@ -103,3 +103,11 @@ # Run empty_task_13 if cond3 executes between 2020-10-10 14:00:00 and 2020-10-10 15:00:00 cond3 >> [empty_task_13, empty_task_23] # [END howto_branch_datetime_operator_logical_date] + + +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 + +# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +test_run = get_test_run(dag1) +test_run2 = get_test_run(dag2) +test_run3 = get_test_run(dag3) diff --git a/airflow-core/src/airflow/example_dags/example_branch_day_of_week_operator.py b/providers/standard/tests/system/standard/example_branch_day_of_week_operator.py similarity index 92% rename from airflow-core/src/airflow/example_dags/example_branch_day_of_week_operator.py rename to providers/standard/tests/system/standard/example_branch_day_of_week_operator.py index 43400522468f7..3b541d2712d44 100644 --- a/airflow-core/src/airflow/example_dags/example_branch_day_of_week_operator.py +++ b/providers/standard/tests/system/standard/example_branch_day_of_week_operator.py @@ -59,3 +59,9 @@ # Run empty_task_3 if it's a weekend, empty_task_4 otherwise empty_task_2 >> branch_weekend >> [empty_task_3, empty_task_4] # [END howto_operator_day_of_week_branch] + + +from tests_common.test_utils.system_tests import get_test_run + +# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +test_run = get_test_run(dag) diff --git a/airflow-core/src/airflow/example_dags/example_branch_operator.py b/providers/standard/tests/system/standard/example_branch_operator.py similarity index 96% rename from airflow-core/src/airflow/example_dags/example_branch_operator.py rename to providers/standard/tests/system/standard/example_branch_operator.py index 0bb429a6f8fe0..98c85444cdd3a 100644 --- a/airflow-core/src/airflow/example_dags/example_branch_operator.py +++ b/providers/standard/tests/system/standard/example_branch_operator.py @@ -164,3 +164,9 @@ def hello_world_with_venv(): # Label is optional here, but it can help identify more complex branches branching_venv >> Label(option) >> t >> join_venv + + +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 + +# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +test_run = get_test_run(dag) diff --git a/airflow-core/src/airflow/example_dags/example_branch_operator_decorator.py b/providers/standard/tests/system/standard/example_branch_operator_decorator.py similarity index 96% rename from airflow-core/src/airflow/example_dags/example_branch_operator_decorator.py rename to providers/standard/tests/system/standard/example_branch_operator_decorator.py index d3634acb998eb..3a3b34e50d11f 100644 --- a/airflow-core/src/airflow/example_dags/example_branch_operator_decorator.py +++ b/providers/standard/tests/system/standard/example_branch_operator_decorator.py @@ -140,3 +140,9 @@ def some_venv_task(): # Label is optional here, but it can help identify more complex branches random_choice_venv >> Label(option) >> t >> join_venv + + +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 + +# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +test_run = get_test_run(dag) diff --git a/airflow-core/tests/system/core/example_external_task_child_deferrable.py b/providers/standard/tests/system/standard/example_external_task_child_deferrable.py similarity index 100% rename from airflow-core/tests/system/core/example_external_task_child_deferrable.py rename to providers/standard/tests/system/standard/example_external_task_child_deferrable.py diff --git a/airflow-core/src/airflow/example_dags/example_external_task_marker_dag.py b/providers/standard/tests/system/standard/example_external_task_marker_dag.py similarity index 94% rename from airflow-core/src/airflow/example_dags/example_external_task_marker_dag.py rename to providers/standard/tests/system/standard/example_external_task_marker_dag.py index b8fad182549f1..d310f369611bf 100644 --- a/airflow-core/src/airflow/example_dags/example_external_task_marker_dag.py +++ b/providers/standard/tests/system/standard/example_external_task_marker_dag.py @@ -96,3 +96,9 @@ child_task3 = EmptyOperator(task_id="child_task3") child_task1 >> child_task2 >> child_task3 + + +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 + +# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +test_run = get_test_run(parent_dag) diff --git a/airflow-core/tests/system/core/example_external_task_parent_deferrable.py b/providers/standard/tests/system/standard/example_external_task_parent_deferrable.py similarity index 100% rename from airflow-core/tests/system/core/example_external_task_parent_deferrable.py rename to providers/standard/tests/system/standard/example_external_task_parent_deferrable.py diff --git a/airflow-core/src/airflow/example_dags/example_latest_only.py b/providers/standard/tests/system/standard/example_latest_only.py similarity index 88% rename from airflow-core/src/airflow/example_dags/example_latest_only.py rename to providers/standard/tests/system/standard/example_latest_only.py index 745100def2af2..983db906bdb5b 100644 --- a/airflow-core/src/airflow/example_dags/example_latest_only.py +++ b/providers/standard/tests/system/standard/example_latest_only.py @@ -38,3 +38,9 @@ task1 = EmptyOperator(task_id="task1") latest_only >> task1 + + +from tests_common.test_utils.system_tests import get_test_run + +# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +test_run = get_test_run(dag) diff --git a/airflow-core/src/airflow/example_dags/example_python_decorator.py b/providers/standard/tests/system/standard/example_python_decorator.py similarity index 94% rename from airflow-core/src/airflow/example_dags/example_python_decorator.py rename to providers/standard/tests/system/standard/example_python_decorator.py index 4f9c05f7940c8..8b26d94842632 100644 --- a/airflow-core/src/airflow/example_dags/example_python_decorator.py +++ b/providers/standard/tests/system/standard/example_python_decorator.py @@ -129,4 +129,10 @@ def callable_external_python(): run_this >> external_python_task >> virtualenv_task -example_python_decorator() +example_dag = example_python_decorator() + + +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 + +# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +test_run = get_test_run(example_dag) diff --git a/airflow-core/src/airflow/example_dags/example_python_operator.py b/providers/standard/tests/system/standard/example_python_operator.py similarity index 95% rename from airflow-core/src/airflow/example_dags/example_python_operator.py rename to providers/standard/tests/system/standard/example_python_operator.py index d48bcae483c92..27ca2db12e995 100644 --- a/airflow-core/src/airflow/example_dags/example_python_operator.py +++ b/providers/standard/tests/system/standard/example_python_operator.py @@ -47,7 +47,7 @@ start_date=pendulum.datetime(2021, 1, 1, tz="UTC"), catchup=False, tags=["example"], -): +) as dag: # [START howto_operator_python] def print_context(ds=None, **kwargs): """Print the Airflow context and ds variable from the context.""" @@ -145,3 +145,9 @@ def callable_external_python(): # [END howto_operator_external_python] run_this >> external_python_task >> virtualenv_task + + +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 + +# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +test_run = get_test_run(dag) diff --git a/airflow-core/src/airflow/example_dags/example_sensor_decorator.py b/providers/standard/tests/system/standard/example_sensor_decorator.py similarity index 90% rename from airflow-core/src/airflow/example_dags/example_sensor_decorator.py rename to providers/standard/tests/system/standard/example_sensor_decorator.py index 64ea80400ceb8..eaceebab9f387 100644 --- a/airflow-core/src/airflow/example_dags/example_sensor_decorator.py +++ b/providers/standard/tests/system/standard/example_sensor_decorator.py @@ -64,3 +64,9 @@ def dummy_operator() -> None: # [END dag_invocation] # [END tutorial] + + +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 + +# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +test_run = get_test_run(dag) diff --git a/airflow-core/src/airflow/example_dags/example_sensors.py b/providers/standard/tests/system/standard/example_sensors.py similarity index 95% rename from airflow-core/src/airflow/example_dags/example_sensors.py rename to providers/standard/tests/system/standard/example_sensors.py index 5ec33bad51bf5..b5ea2f458fd94 100644 --- a/airflow-core/src/airflow/example_dags/example_sensors.py +++ b/providers/standard/tests/system/standard/example_sensors.py @@ -130,3 +130,9 @@ def failure_callable(): t8 >> tx [t9, t10] >> tx t11 >> tx + + +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 + +# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +test_run = get_test_run(dag) diff --git a/airflow-core/src/airflow/example_dags/example_short_circuit_decorator.py b/providers/standard/tests/system/standard/example_short_circuit_decorator.py similarity index 92% rename from airflow-core/src/airflow/example_dags/example_short_circuit_decorator.py rename to providers/standard/tests/system/standard/example_short_circuit_decorator.py index 714325b1739d2..93b32a924502d 100644 --- a/airflow-core/src/airflow/example_dags/example_short_circuit_decorator.py +++ b/providers/standard/tests/system/standard/example_short_circuit_decorator.py @@ -58,3 +58,9 @@ def check_condition(condition): example_dag = example_short_circuit_decorator() + + +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 + +# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +test_run = get_test_run(example_dag) diff --git a/airflow-core/src/airflow/example_dags/example_short_circuit_operator.py b/providers/standard/tests/system/standard/example_short_circuit_operator.py similarity index 92% rename from airflow-core/src/airflow/example_dags/example_short_circuit_operator.py rename to providers/standard/tests/system/standard/example_short_circuit_operator.py index 494bd55a8697a..892be8934ae3a 100644 --- a/airflow-core/src/airflow/example_dags/example_short_circuit_operator.py +++ b/providers/standard/tests/system/standard/example_short_circuit_operator.py @@ -32,7 +32,7 @@ start_date=pendulum.datetime(2021, 1, 1, tz="UTC"), catchup=False, tags=["example"], -): +) as dag: # [START howto_operator_short_circuit] cond_true = ShortCircuitOperator( task_id="condition_is_True", @@ -64,3 +64,9 @@ chain(task_1, [task_2, short_circuit], [task_3, task_4], [task_5, task_6], task_7) # [END howto_operator_short_circuit_trigger_rules] + + +from tests_common.test_utils.system_tests import get_test_run + +# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +test_run = get_test_run(dag) diff --git a/airflow-core/src/airflow/example_dags/example_trigger_controller_dag.py b/providers/standard/tests/system/standard/example_trigger_controller_dag.py similarity index 90% rename from airflow-core/src/airflow/example_dags/example_trigger_controller_dag.py rename to providers/standard/tests/system/standard/example_trigger_controller_dag.py index 8ca49e34b3dcd..3a85fb7e0aff1 100644 --- a/airflow-core/src/airflow/example_dags/example_trigger_controller_dag.py +++ b/providers/standard/tests/system/standard/example_trigger_controller_dag.py @@ -44,3 +44,9 @@ ) # [END howto_operator_trigger_dagrun] + + +from tests_common.test_utils.system_tests import get_test_run + +# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +test_run = get_test_run(dag) diff --git a/providers/standard/tests/system/standard/sql/__init__.py b/providers/standard/tests/system/standard/sql/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/providers/standard/tests/system/standard/sql/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/providers/standard/tests/system/standard/sql/sample.sql b/providers/standard/tests/system/standard/sql/sample.sql new file mode 100644 index 0000000000000..23af6ab4b9bb3 --- /dev/null +++ b/providers/standard/tests/system/standard/sql/sample.sql @@ -0,0 +1,24 @@ +/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/ + +CREATE TABLE Orders ( + order_id INT PRIMARY KEY, + name TEXT, + description TEXT +) diff --git a/scripts/ci/pre_commit/check_system_tests.py b/scripts/ci/pre_commit/check_system_tests.py index 3d5c743b54f78..15f94a8cdfa6a 100755 --- a/scripts/ci/pre_commit/check_system_tests.py +++ b/scripts/ci/pre_commit/check_system_tests.py @@ -46,7 +46,7 @@ PYTEST_FUNCTION_PATTERN = re.compile( r"from tests_common\.test_utils\.system_tests import get_test_run(?: # noqa: E402)?\s+" r"(?:# .+\))?\s+" - r"test_run = get_test_run\(dag\)" + r"test_run = get_test_run" ) diff --git a/scripts/docker/entrypoint_ci.sh b/scripts/docker/entrypoint_ci.sh index a99c961c67528..adf37ce738592 100755 --- a/scripts/docker/entrypoint_ci.sh +++ b/scripts/docker/entrypoint_ci.sh @@ -178,6 +178,12 @@ function environment_initialization() { cd "${AIRFLOW_SOURCES}" + # Temporarily add /opt/airflow/providers/standard/tests to PYTHONPATH in order to see example dags + # in the UI when testing in Breeze. This might be solved differently in the future + if [[ -d /opt/airflow/providers/standard/tests ]]; then + export PYTHONPATH=${PYTHONPATH=}:/opt/airflow/providers/standard/tests + fi + if [[ ${START_AIRFLOW:="false"} == "true" || ${START_AIRFLOW} == "True" ]]; then export AIRFLOW__CORE__LOAD_EXAMPLES=${LOAD_EXAMPLES} wait_for_asset_compilation