diff --git a/Dockerfile.ci b/Dockerfile.ci index d8b4d7f343e13..e68477ab09df0 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -950,6 +950,12 @@ function environment_initialization() { cd "${AIRFLOW_SOURCES}" + # Temporarily add /opt/airflow/providers/standard/tests to PYTHONPATH in order to see example dags + # in the UI when testing in Breeze. This might be solved differently in the future + if [[ -d /opt/airflow/providers/standard/tests ]]; then + export PYTHONPATH=${PYTHONPATH=}:/opt/airflow/providers/standard/tests + fi + if [[ ${START_AIRFLOW:="false"} == "true" || ${START_AIRFLOW} == "True" ]]; then export AIRFLOW__CORE__LOAD_EXAMPLES=${LOAD_EXAMPLES} wait_for_asset_compilation diff --git a/airflow-core/docs/tutorial/taskflow.rst b/airflow-core/docs/tutorial/taskflow.rst index 8ba2df9a51019..b5804dea94c18 100644 --- a/airflow-core/docs/tutorial/taskflow.rst +++ b/airflow-core/docs/tutorial/taskflow.rst @@ -269,7 +269,7 @@ system-level packages. TaskFlow supports multiple execution environments to isol Creates a temporary virtualenv at task runtime. Great for experimental or dynamic tasks, but may have cold start overhead. -.. exampleinclude:: /../src/airflow/example_dags/example_python_decorator.py +.. exampleinclude:: /../../providers/standard/tests/system/standard/example_python_decorator.py :language: python :dedent: 4 :start-after: [START howto_operator_python_venv] @@ -283,7 +283,7 @@ overhead. Executes the task using a pre-installed Python interpreter — ideal for consistent environments or shared virtualenvs. -.. exampleinclude:: /../src/airflow/example_dags/example_python_decorator.py +.. exampleinclude:: /../../providers/standard/tests/system/standard/example_python_decorator.py :language: python :dedent: 4 :start-after: [START howto_operator_external_python] @@ -333,7 +333,7 @@ Using Sensors Use ``@task.sensor`` to build lightweight, reusable sensors using Python functions. These support both poke and reschedule modes. -.. exampleinclude:: /../src/airflow/example_dags/example_sensor_decorator.py +.. exampleinclude:: /../../providers/standard/tests/system/standard/example_sensor_decorator.py :language: python :start-after: [START tutorial] :end-before: [END tutorial] diff --git a/airflow-core/src/airflow/dag_processing/bundles/manager.py b/airflow-core/src/airflow/dag_processing/bundles/manager.py index 9760da617a1d9..1ace2896028c6 100644 --- a/airflow-core/src/airflow/dag_processing/bundles/manager.py +++ b/airflow-core/src/airflow/dag_processing/bundles/manager.py @@ -16,6 +16,7 @@ # under the License. from __future__ import annotations +import os from typing import TYPE_CHECKING from airflow.configuration import conf @@ -34,6 +35,7 @@ from airflow.dag_processing.bundles.base import BaseDagBundle _example_dag_bundle_name = "example_dags" +_example_standard_dag_bundle_name = "example_standard_dags" def _bundle_item_exc(msg): @@ -80,6 +82,25 @@ def _add_example_dag_bundle(config_list): ) +def _add_example_standard_dag_bundle(config_list): + # TODO(potiuk): make it more generic - for now we only add standard example_dags if they are locally available + try: + from system import standard + except ImportError: + return + + example_dag_folder = next(iter(standard.__path__)) + config_list.append( + { + "name": _example_standard_dag_bundle_name, + "classpath": "airflow.dag_processing.bundles.local.LocalDagBundle", + "kwargs": { + "path": example_dag_folder, + }, + } + ) + + class DagBundlesManager(LoggingMixin): """Manager for DAG bundles.""" @@ -112,6 +133,11 @@ def parse_config(self) -> None: _validate_bundle_config(config_list) if conf.getboolean("core", "LOAD_EXAMPLES"): _add_example_dag_bundle(config_list) + if ( + os.environ.get("BREEZE", "").lower() == "true" + or os.environ.get("_IN_UNIT_TESTS", "").lower() == "true" + ): + _add_example_standard_dag_bundle(config_list) for cfg in config_list: name = cfg["name"] diff --git a/airflow-core/src/airflow/models/dagbag.py b/airflow-core/src/airflow/models/dagbag.py index 52c2337474d74..d0c8bf98f4152 100644 --- a/airflow-core/src/airflow/models/dagbag.py +++ b/airflow-core/src/airflow/models/dagbag.py @@ -586,6 +586,14 @@ def collect_dags( example_dag_folder = next(iter(example_dags.__path__)) files_to_parse.extend(list_py_file_paths(example_dag_folder, safe_mode=safe_mode)) + try: + from system import standard + + example_dag_folder_standard = next(iter(standard.__path__)) + files_to_parse.extend(list_py_file_paths(example_dag_folder_standard, safe_mode=safe_mode)) + except ImportError: + # Nothing happens - this should only work during tests + pass for filepath in files_to_parse: try: diff --git a/airflow-core/src/airflow/utils/cli.py b/airflow-core/src/airflow/utils/cli.py index 86ef56da69ca4..8da5c24b81f74 100644 --- a/airflow-core/src/airflow/utils/cli.py +++ b/airflow-core/src/airflow/utils/cli.py @@ -270,7 +270,6 @@ def get_dag(bundle_names: list | None, dag_id: str, from_db: bool = False) -> DA bundle_names = bundle_names or [] dag: DAG | None = None - if from_db: dagbag = DagBag(read_dags_from_db=True) dag = dagbag.get_dag(dag_id) # get_dag loads from the DB as requested diff --git a/airflow-core/tests/unit/api_fastapi/core_api/routes/public/test_dag_parsing.py b/airflow-core/tests/unit/api_fastapi/core_api/routes/public/test_dag_parsing.py index 11ecc484bebf3..f64eef028e273 100644 --- a/airflow-core/tests/unit/api_fastapi/core_api/routes/public/test_dag_parsing.py +++ b/airflow-core/tests/unit/api_fastapi/core_api/routes/public/test_dag_parsing.py @@ -28,8 +28,8 @@ pytestmark = pytest.mark.db_test -EXAMPLE_DAG_FILE = AIRFLOW_CORE_SOURCES_PATH / "airflow" / "example_dags" / "example_bash_operator.py" -TEST_DAG_ID = "example_bash_operator" +EXAMPLE_DAG_FILE = AIRFLOW_CORE_SOURCES_PATH / "airflow" / "example_dags" / "example_simplest_dag.py" +TEST_DAG_ID = "example_simplest_dag" NOT_READABLE_DAG_ID = "latest_only_with_trigger" TEST_MULTIPLE_DAGS_ID = "asset_produces_1" diff --git a/airflow-core/tests/unit/api_fastapi/core_api/routes/public/test_dag_report.py b/airflow-core/tests/unit/api_fastapi/core_api/routes/public/test_dag_report.py index 3938894001de9..d1204bedd3e3d 100644 --- a/airflow-core/tests/unit/api_fastapi/core_api/routes/public/test_dag_report.py +++ b/airflow-core/tests/unit/api_fastapi/core_api/routes/public/test_dag_report.py @@ -23,6 +23,7 @@ from airflow.utils.file import list_py_file_paths +import system.standard from tests_common.test_utils.config import conf_vars from tests_common.test_utils.db import clear_db_dags, parse_and_sync_to_db @@ -37,8 +38,10 @@ def get_corresponding_dag_file_count(dir: str, include_examples: bool = True) -> int: from airflow import example_dags - return len(list_py_file_paths(directory=dir)) + ( - len(list_py_file_paths(next(iter(example_dags.__path__)))) if include_examples else 0 + return ( + len(list_py_file_paths(directory=dir)) + + (len(list_py_file_paths(next(iter(example_dags.__path__)))) if include_examples else 0) + + (len(list_py_file_paths(next(iter(system.standard.__path__)))) if include_examples else 0) ) diff --git a/airflow-core/tests/unit/api_fastapi/core_api/routes/public/test_dag_sources.py b/airflow-core/tests/unit/api_fastapi/core_api/routes/public/test_dag_sources.py index 90ea45ece7b46..cd223e67c0c08 100644 --- a/airflow-core/tests/unit/api_fastapi/core_api/routes/public/test_dag_sources.py +++ b/airflow-core/tests/unit/api_fastapi/core_api/routes/public/test_dag_sources.py @@ -39,9 +39,9 @@ # Example bash operator located here: airflow/example_dags/example_bash_operator.py EXAMPLE_DAG_FILE = ( - AIRFLOW_REPO_ROOT_PATH / "airflow-core" / "src" / "airflow" / "example_dags" / "example_bash_operator.py" + AIRFLOW_REPO_ROOT_PATH / "airflow-core" / "src" / "airflow" / "example_dags" / "example_simplest_dag.py" ) -TEST_DAG_ID = "example_bash_operator" +TEST_DAG_ID = "example_simplest_dag" @pytest.fixture diff --git a/airflow-core/tests/unit/cli/commands/test_task_command.py b/airflow-core/tests/unit/cli/commands/test_task_command.py index 9421583e417c7..d864ffb808fee 100644 --- a/airflow-core/tests/unit/cli/commands/test_task_command.py +++ b/airflow-core/tests/unit/cli/commands/test_task_command.py @@ -47,6 +47,7 @@ from airflow.utils.state import State, TaskInstanceState from airflow.utils.types import DagRunTriggeredByType, DagRunType +from tests_common.test_utils.config import conf_vars from tests_common.test_utils.db import clear_db_runs, parse_and_sync_to_db pytestmark = pytest.mark.db_test @@ -74,7 +75,6 @@ def move_back(old_path, new_path): shutil.move(new_path, old_path) -# TODO: Check if tests needs side effects - locally there's missing DAG class TestCliTasks: run_id = "TEST_RUN_ID" dag_id = "example_python_operator" @@ -91,7 +91,7 @@ def setup_class(cls): cls.parser = cli_parser.get_parser() clear_db_runs() - cls.dagbag = DagBag(read_dags_from_db=True) + cls.dagbag = DagBag(read_dags_from_db=True, include_examples=True) cls.dag = cls.dagbag.get_dag(cls.dag_id) data_interval = cls.dag.timetable.infer_manual_data_interval(run_after=DEFAULT_DATE) cls.dag_run = cls.dag.create_dagrun( @@ -108,6 +108,7 @@ def setup_class(cls): def teardown_class(cls) -> None: clear_db_runs() + @conf_vars({("core", "load_examples"): "true"}) @pytest.mark.execution_timeout(120) def test_cli_list_tasks(self): for dag_id in self.dagbag.dags: diff --git a/airflow-core/tests/unit/jobs/test_scheduler_job.py b/airflow-core/tests/unit/jobs/test_scheduler_job.py index 201490633de13..2f0d71ac89fc9 100644 --- a/airflow-core/tests/unit/jobs/test_scheduler_job.py +++ b/airflow-core/tests/unit/jobs/test_scheduler_job.py @@ -37,7 +37,6 @@ from sqlalchemy import func, select, update from sqlalchemy.orm import joinedload -import airflow.example_dags from airflow import settings from airflow.assets.manager import AssetManager from airflow.callbacks.callback_requests import DagCallbackRequest, TaskCallbackRequest @@ -75,6 +74,7 @@ from airflow.utils.thread_safe_dict import ThreadSafeDict from airflow.utils.types import DagRunTriggeredByType, DagRunType +from system import standard from tests_common.test_utils.asserts import assert_queries_count from tests_common.test_utils.config import conf_vars, env_vars from tests_common.test_utils.db import ( @@ -104,7 +104,7 @@ ELASTIC_DAG_FILE = os.path.join(PERF_DAGS_FOLDER, "elastic_dag.py") TEST_DAG_FOLDER = os.environ["AIRFLOW__CORE__DAGS_FOLDER"] -EXAMPLE_DAGS_FOLDER = airflow.example_dags.__path__[0] +EXAMPLE_STANDARD_DAGS_FOLDER = standard.__path__[0] DEFAULT_DATE = timezone.datetime(2016, 1, 1) DEFAULT_LOGICAL_DATE = timezone.coerce_datetime(DEFAULT_DATE) TRY_NUMBER = 1 @@ -5707,7 +5707,7 @@ def test_find_and_purge_task_instances_without_heartbeats_nothing(self): @pytest.mark.usefixtures("testing_dag_bundle") def test_find_and_purge_task_instances_without_heartbeats(self, session, create_dagrun): - dagfile = os.path.join(EXAMPLE_DAGS_FOLDER, "example_branch_operator.py") + dagfile = os.path.join(EXAMPLE_STANDARD_DAGS_FOLDER, "example_branch_operator.py") dagbag = DagBag(dagfile) dag = dagbag.get_dag("example_branch_operator") dm = LazyDeserializedDAG(data=SerializedDAG.to_dict(dag)) @@ -5773,7 +5773,7 @@ def test_task_instance_heartbeat_timeout_message(self, session, create_dagrun): """ Check that the task instance heartbeat timeout message comes out as expected """ - dagfile = os.path.join(EXAMPLE_DAGS_FOLDER, "example_branch_operator.py") + dagfile = os.path.join(EXAMPLE_STANDARD_DAGS_FOLDER, "example_branch_operator.py") dagbag = DagBag(dagfile) dag = dagbag.get_dag("example_branch_operator") dm = LazyDeserializedDAG(data=SerializedDAG.to_dict(dag)) diff --git a/airflow-core/tests/unit/models/test_dagbag.py b/airflow-core/tests/unit/models/test_dagbag.py index a4cc7da9e5bff..a9ba6669af96f 100644 --- a/airflow-core/tests/unit/models/test_dagbag.py +++ b/airflow-core/tests/unit/models/test_dagbag.py @@ -42,6 +42,7 @@ from airflow.serialization.serialized_objects import SerializedDAG from airflow.utils import timezone as tz from airflow.utils.session import create_session +from scripts.ci.pre_commit.common_precommit_utils import AIRFLOW_ROOT_PATH from tests_common.test_utils import db from tests_common.test_utils.asserts import assert_queries_count @@ -52,6 +53,12 @@ pytestmark = pytest.mark.db_test example_dags_folder = pathlib.Path(airflow.example_dags.__path__[0]) # type: ignore[attr-defined] +try: + import system.standard + + example_standard_dags_folder = pathlib.Path(system.standard.__path__[0]) # type: ignore[attr-defined] +except ImportError: + example_standard_dags_folder = pathlib.Path(airflow.example_dags.__path__[0]) # type: ignore[attr-defined] PY311 = sys.version_info >= (3, 11) @@ -359,13 +366,16 @@ def process_file(self, filepath, only_if_updated=True, safe_mode=True): ("file_to_load", "expected"), ( pytest.param( - pathlib.Path(example_dags_folder) / "example_bash_operator.py", - {"example_bash_operator": "airflow/example_dags/example_bash_operator.py"}, + pathlib.Path(example_standard_dags_folder) / "example_bash_operator.py", + { + "example_bash_operator": f"{example_standard_dags_folder.relative_to(AIRFLOW_ROOT_PATH) / 'example_bash_operator.py'}" + }, id="example_bash_operator", ), ), ) def test_get_dag_registration(self, file_to_load, expected): + pytest.importorskip("system.standard") dagbag = DagBag(dag_folder=os.devnull, include_examples=False) dagbag.process_file(os.fspath(file_to_load)) for dag_id, path in expected.items(): @@ -420,7 +430,7 @@ def test_refresh_py_dag(self, mock_dagmodel, tmp_path): Test that we can refresh an ordinary .py DAG """ dag_id = "example_bash_operator" - fileloc = str(example_dags_folder / "example_bash_operator.py") + fileloc = str(example_standard_dags_folder / "example_bash_operator.py") mock_dagmodel.return_value = DagModel() mock_dagmodel.return_value.last_expired = datetime.max.replace(tzinfo=timezone.utc) diff --git a/clients/python/test_python_client.py b/clients/python/test_python_client.py index 61bd3df6619de..d5979b40b9964 100644 --- a/clients/python/test_python_client.py +++ b/clients/python/test_python_client.py @@ -69,7 +69,7 @@ # Make sure in the [core] section, the `load_examples` config is set to True in your airflow.cfg # or AIRFLOW__CORE__LOAD_EXAMPLES environment variable set to True -DAG_ID = "example_bash_operator" +DAG_ID = "example_simplest_dag" # Enter a context with an instance of the API client diff --git a/devel-common/src/docs/provider_conf.py b/devel-common/src/docs/provider_conf.py index 16b0ad3c952c8..4b3f433ab763f 100644 --- a/devel-common/src/docs/provider_conf.py +++ b/devel-common/src/docs/provider_conf.py @@ -155,15 +155,8 @@ "operators/_partials", "_api/airflow/index.rst", "_api/airflow/providers/index.rst", - "_api/airflow/providers/apache/index.rst", - "_api/airflow/providers/atlassian/index.rst", - "_api/airflow/providers/cncf/index.rst", - "_api/airflow/providers/common/index.rst", - "_api/airflow/providers/common/messaging/providers/base_provider/index.rst", - "_api/airflow/providers/common/messaging/providers/sqs/index.rst", - "_api/airflow/providers/dbt/index.rst", - "_api/airflow/providers/microsoft/index.rst", "_api/docs/conf", + *[f"_api/airflow/providers/{subpackage}/index.rst" for subpackage in empty_subpackages], *[f"_api/system/{subpackage}/index.rst" for subpackage in empty_subpackages], *[f"_api/tests/system/{subpackage}/index.rst" for subpackage in empty_subpackages], ] diff --git a/devel-common/src/sphinx_exts/docs_build/docs_builder.py b/devel-common/src/sphinx_exts/docs_build/docs_builder.py index 1a5d46c6d7f60..44ba83e6f7a3d 100644 --- a/devel-common/src/sphinx_exts/docs_build/docs_builder.py +++ b/devel-common/src/sphinx_exts/docs_build/docs_builder.py @@ -120,6 +120,13 @@ def _src_dir(self) -> Path: console.print(f"[red]Unknown package name: {self.package_name}") sys.exit(1) + @property + def pythonpath(self) -> list[Path]: + path = [] + if (self._src_dir.parent / "tests").exists(): + path.append(self._src_dir.parent.joinpath("tests").resolve()) + return path + @property def _generated_api_dir(self) -> Path: return self._build_dir.resolve() / "_api" @@ -167,6 +174,8 @@ def check_spelling(self, verbose: bool) -> tuple[list[SpellingError], list[DocBu console.print("[yellow]Command to run:[/] ", " ".join([shlex.quote(arg) for arg in build_cmd])) env = os.environ.copy() env["AIRFLOW_PACKAGE_NAME"] = self.package_name + if self.pythonpath: + env["PYTHONPATH"] = ":".join([path.as_posix() for path in self.pythonpath]) if verbose: console.print( f"[bright_blue]{self.package_name:60}:[/] The output is hidden until an error occurs." @@ -246,6 +255,8 @@ def build_sphinx_docs(self, verbose: bool) -> list[DocBuildError]: console.print("[yellow]Command to run:[/] ", " ".join([shlex.quote(arg) for arg in build_cmd])) env = os.environ.copy() env["AIRFLOW_PACKAGE_NAME"] = self.package_name + if self.pythonpath: + env["PYTHONPATH"] = ":".join([path.as_posix() for path in self.pythonpath]) if verbose: console.print( f"[bright_blue]{self.package_name:60}:[/] Running sphinx. " diff --git a/devel-common/src/sphinx_exts/exampleinclude.py b/devel-common/src/sphinx_exts/exampleinclude.py index decf049d83a0b..3feaed367c35a 100644 --- a/devel-common/src/sphinx_exts/exampleinclude.py +++ b/devel-common/src/sphinx_exts/exampleinclude.py @@ -17,8 +17,8 @@ from __future__ import annotations """Nice formatted include for examples""" -import os import traceback +from pathlib import Path from docutils import nodes @@ -139,6 +139,7 @@ def register_source(app, env, modname): """ if modname is None: return False + entry = env._viewcode_modules.get(modname, None) if entry is False: print(f"[{modname}] Entry is false for ") @@ -227,6 +228,7 @@ def doctree_read(app, doctree): :return None """ + env = app.builder.env if not hasattr(env, "_viewcode_modules"): env._viewcode_modules = {} @@ -235,19 +237,23 @@ def doctree_read(app, doctree): return for objnode in doctree.traverse(ExampleHeader): - filepath = objnode.get("filename") - relative_path = os.path.relpath( - filepath, os.path.commonprefix([app.config.exampleinclude_sourceroot, filepath]) - ) - if relative_path.endswith(".py"): - modname = relative_path.replace("/", ".")[-3] - split_modname = modname.split(".") + source_root_path = Path(app.config.exampleinclude_sourceroot) + filepath = Path(objnode.get("filename")) + if filepath.is_relative_to(source_root_path) and filepath.name.endswith(".py"): + module_path = filepath.relative_to(source_root_path) + split_modname = module_path.parts if "src" in split_modname: modname = ".".join(split_modname[split_modname.index("src") + 1 :]) + elif "tests" in split_modname: + modname = ".".join(split_modname[split_modname.index("tests") + 1 :]) + else: + modname = ".".join(split_modname) + modname = modname.replace(".py", "") else: modname = None + module_path = filepath.resolve() show_button = register_source(app, env, modname) - onlynode = create_node(env, relative_path, show_button) + onlynode = create_node(env, module_path.as_posix(), show_button) objnode.replace_self(onlynode) diff --git a/devel-common/src/tests_common/pytest_plugin.py b/devel-common/src/tests_common/pytest_plugin.py index dff39c5d7a067..a86377b5ac3a7 100644 --- a/devel-common/src/tests_common/pytest_plugin.py +++ b/devel-common/src/tests_common/pytest_plugin.py @@ -135,7 +135,6 @@ # Make sure sqlalchemy will not be usable for pure unit tests even if initialized os.environ["AIRFLOW__CORE__SQL_ALCHEMY_CONN"] = "bad_schema:///" os.environ["AIRFLOW__DATABASE__SQL_ALCHEMY_CONN"] = "bad_schema:///" - os.environ["_IN_UNIT_TESTS"] = "true" # Set it here to pass the flag to python-xdist spawned processes os.environ["_AIRFLOW_SKIP_DB_TESTS"] = "true" @@ -143,6 +142,8 @@ # Set it here to pass the flag to python-xdist spawned processes os.environ["_AIRFLOW_RUN_DB_TESTS_ONLY"] = "true" +os.environ["_IN_UNIT_TESTS"] = "true" + _airflow_sources = os.getenv("AIRFLOW_SOURCES", None) AIRFLOW_ROOT_PATH = (Path(_airflow_sources) if _airflow_sources else Path(__file__).parents[3]).resolve() AIRFLOW_CORE_SOURCES_PATH = AIRFLOW_ROOT_PATH / "airflow-core" / "src" diff --git a/docker-stack-docs/conf.py b/docker-stack-docs/conf.py index 5d0e9f06869b7..c42e311844cb6 100644 --- a/docker-stack-docs/conf.py +++ b/docker-stack-docs/conf.py @@ -222,7 +222,7 @@ suppress_warnings = SUPPRESS_WARNINGS # -- Options for ext.exampleinclude -------------------------------------------- -exampleinclude_sourceroot = os.path.abspath("..") +exampleinclude_sourceroot = os.path.abspath(".") # -- Options for ext.redirects ------------------------------------------------- redirects_file = "redirects.txt" diff --git a/docker-tests/tests/docker_tests/test_docker_compose_quick_start.py b/docker-tests/tests/docker_tests/test_docker_compose_quick_start.py index 789a0f734294f..6241e49d15878 100644 --- a/docker-tests/tests/docker_tests/test_docker_compose_quick_start.py +++ b/docker-tests/tests/docker_tests/test_docker_compose_quick_start.py @@ -41,7 +41,7 @@ DOCKER_COMPOSE_HOST_PORT = os.environ.get("HOST_PORT", "localhost:8080") AIRFLOW_WWW_USER_USERNAME = os.environ.get("_AIRFLOW_WWW_USER_USERNAME", "airflow") AIRFLOW_WWW_USER_PASSWORD = os.environ.get("_AIRFLOW_WWW_USER_PASSWORD", "airflow") -DAG_ID = "example_bash_operator" +DAG_ID = "example_simplest_dag" DAG_RUN_ID = "test_dag_run_id" diff --git a/kubernetes-tests/tests/kubernetes_tests/test_other_executors.py b/kubernetes-tests/tests/kubernetes_tests/test_other_executors.py index 7b49bf9f5639b..28bad807da01e 100644 --- a/kubernetes-tests/tests/kubernetes_tests/test_other_executors.py +++ b/kubernetes-tests/tests/kubernetes_tests/test_other_executors.py @@ -30,7 +30,7 @@ @pytest.mark.skipif(EXECUTOR == "KubernetesExecutor", reason="Does not run on KubernetesExecutor") class TestCeleryAndLocalExecutor(BaseK8STest): def test_integration_run_dag(self): - dag_id = "example_bash_operator" + dag_id = "example_simplest_dag" dag_run_id, logical_date = self.start_job_in_kubernetes(dag_id, self.host) print(f"Found the job with logical_date {logical_date}") @@ -39,7 +39,7 @@ def test_integration_run_dag(self): host=self.host, dag_run_id=dag_run_id, dag_id=dag_id, - task_id="run_after_loop", + task_id="my_task", expected_final_state="success", timeout=300, ) diff --git a/providers-summary-docs/conf.py b/providers-summary-docs/conf.py index 85dfc4c1e1e47..2219f0bcb8ab6 100644 --- a/providers-summary-docs/conf.py +++ b/providers-summary-docs/conf.py @@ -241,7 +241,7 @@ suppress_warnings = SUPPRESS_WARNINGS # -- Options for ext.exampleinclude -------------------------------------------- -exampleinclude_sourceroot = os.path.abspath("..") +exampleinclude_sourceroot = os.path.abspath(".") # -- Options for ext.redirects ------------------------------------------------- redirects_file = "redirects.txt" diff --git a/providers/common/messaging/docs/index.rst b/providers/common/messaging/docs/index.rst index 10a349a362bad..0f50a2c2eb39d 100644 --- a/providers/common/messaging/docs/index.rst +++ b/providers/common/messaging/docs/index.rst @@ -43,6 +43,8 @@ :caption: References Python API <_api/airflow/providers/common/messaging/index> + Base Provider <_api/airflow/providers/common/messaging/providers/base_provider/index> + SQS provider <_api/airflow/providers/common/messaging/providers/sqs/index> .. toctree:: :hidden: diff --git a/providers/standard/docs/index.rst b/providers/standard/docs/index.rst index 43cf71f2c1f0e..5ddaa7eef2484 100644 --- a/providers/standard/docs/index.rst +++ b/providers/standard/docs/index.rst @@ -38,17 +38,24 @@ Sensors Configuration +.. toctree:: + :hidden: + :maxdepth: 1 + :caption: System tests + + System Tests <_api/tests/system/standard/index> + .. toctree:: :hidden: :maxdepth: 1 :caption: Resources + Example DAGs PyPI Repository Installing from sources Python API <_api/airflow/providers/standard/index> - .. THE REMAINDER OF THE FILE IS AUTOMATICALLY GENERATED. IT WILL BE OVERWRITTEN AT RELEASE TIME! diff --git a/providers/standard/docs/operators/bash.rst b/providers/standard/docs/operators/bash.rst index 113ca81e246d9..0831403020c99 100644 --- a/providers/standard/docs/operators/bash.rst +++ b/providers/standard/docs/operators/bash.rst @@ -41,7 +41,7 @@ determined by: .. tab-item:: @task.bash :sync: taskflow - .. exampleinclude:: /../../../airflow-core/src/airflow/example_dags/example_bash_decorator.py + .. exampleinclude:: /../tests/system/standard/example_bash_decorator.py :language: python :dedent: 4 :start-after: [START howto_decorator_bash] @@ -50,7 +50,7 @@ determined by: .. tab-item:: BashOperator :sync: operator - .. exampleinclude:: /../../../airflow-core/src/airflow/example_dags/example_bash_operator.py + .. exampleinclude:: /../tests/system/standard/example_bash_operator.py :language: python :dedent: 4 :start-after: [START howto_operator_bash] @@ -67,7 +67,7 @@ You can use :ref:`Jinja templates ` to parameterize t .. tab-item:: @task.bash :sync: taskflow - .. exampleinclude:: /../../../airflow-core/src/airflow/example_dags/example_bash_decorator.py + .. exampleinclude:: /../tests/system/standard/example_bash_decorator.py :language: python :dedent: 4 :start-after: [START howto_decorator_bash_template] @@ -76,7 +76,7 @@ You can use :ref:`Jinja templates ` to parameterize t .. tab-item:: BashOperator :sync: operator - .. exampleinclude:: /../../../airflow-core/src/airflow/example_dags/example_bash_operator.py + .. exampleinclude:: /../tests/system/standard/example_bash_operator.py :language: python :dedent: 4 :start-after: [START howto_operator_bash_template] @@ -85,7 +85,7 @@ You can use :ref:`Jinja templates ` to parameterize t Using the ``@task.bash`` TaskFlow decorator allows you to return a formatted string and take advantage of having all :ref:`execution context variables directly accessible to decorated tasks `. -.. exampleinclude:: /../../../airflow-core/src/airflow/example_dags/example_bash_decorator.py +.. exampleinclude:: /../tests/system/standard/example_bash_decorator.py :language: python :dedent: 4 :start-after: [START howto_decorator_bash_context_vars] @@ -169,7 +169,7 @@ exit code if you pass ``skip_on_exit_code``). .. tab-item:: @task.bash :sync: taskflow - .. exampleinclude:: /../../../airflow-core/src/airflow/example_dags/example_bash_decorator.py + .. exampleinclude:: /../tests/system/standard/example_bash_decorator.py :language: python :dedent: 4 :start-after: [START howto_decorator_bash_skip] @@ -178,7 +178,7 @@ exit code if you pass ``skip_on_exit_code``). .. tab-item:: BashOperator :sync: operator - .. exampleinclude:: /../../../airflow-core/src/airflow/example_dags/example_bash_operator.py + .. exampleinclude:: /../tests/system/standard/example_bash_operator.py :language: python :start-after: [START howto_operator_bash_skip] :end-before: [END howto_operator_bash_skip] @@ -388,7 +388,7 @@ or even build the Bash command(s) to execute. For example, use conditional logic to determine task behavior: -.. exampleinclude:: /../../../airflow-core/src/airflow/example_dags/example_bash_decorator.py +.. exampleinclude:: /../tests/system/standard/example_bash_decorator.py :language: python :dedent: 4 :start-after: [START howto_decorator_bash_conditional] @@ -396,7 +396,7 @@ For example, use conditional logic to determine task behavior: Or call a function to help build a Bash command: -.. exampleinclude:: /../../../airflow-core/src/airflow/example_dags/example_bash_decorator.py +.. exampleinclude:: /../tests/system/standard/example_bash_decorator.py :language: python :dedent: 4 :start-after: [START howto_decorator_bash_build_cmd] diff --git a/providers/standard/docs/operators/datetime.rst b/providers/standard/docs/operators/datetime.rst index ea8b9d3b31fa8..251e933cbad40 100644 --- a/providers/standard/docs/operators/datetime.rst +++ b/providers/standard/docs/operators/datetime.rst @@ -39,7 +39,7 @@ take some time between when the DAGRun was scheduled and executed and it might m the DAGRun was scheduled properly, the actual time used for branching decision will be different than the schedule time and the branching decision might be different depending on those delays. -.. exampleinclude:: /../../../airflow-core/src/airflow/example_dags/example_branch_datetime_operator.py +.. exampleinclude:: /../tests/system/standard/example_branch_datetime_operator.py :language: python :start-after: [START howto_branch_datetime_operator] :end-before: [END howto_branch_datetime_operator] @@ -50,7 +50,7 @@ the current date in order to allow comparisons with it. In the event that ``targ to a ``datetime.time`` that occurs before the given ``target_lower``, a day will be added to ``target_upper``. This is done to allow for time periods that span over two dates. -.. exampleinclude:: /../../../airflow-core/src/airflow/example_dags/example_branch_datetime_operator.py +.. exampleinclude:: /../tests/system/standard/example_branch_datetime_operator.py :language: python :start-after: [START howto_branch_datetime_operator_next_day] :end-before: [END howto_branch_datetime_operator_next_day] @@ -66,7 +66,7 @@ The usage is much more "data range" friendly. The ``logical_date`` does not chan it is not affected by execution delays, so this approach is suitable for idempotent DAG runs that might be back-filled. -.. exampleinclude:: /../../../airflow-core/src/airflow/example_dags/example_branch_datetime_operator.py +.. exampleinclude:: /../tests/system/standard/example_branch_datetime_operator.py :language: python :start-after: [START howto_branch_datetime_operator_logical_date] :end-before: [END howto_branch_datetime_operator_logical_date] @@ -78,7 +78,7 @@ BranchDayOfWeekOperator Use the :class:`~airflow.operators.weekday.BranchDayOfWeekOperator` to branch your workflow based on week day value. -.. exampleinclude:: /../../../airflow-core/src/airflow/example_dags/example_branch_day_of_week_operator.py +.. exampleinclude:: /../tests/system/standard/example_branch_day_of_week_operator.py :language: python :dedent: 4 :start-after: [START howto_operator_day_of_week_branch] diff --git a/providers/standard/docs/operators/latest_only.rst b/providers/standard/docs/operators/latest_only.rst index d3fc30937a2ed..21e66a124917e 100644 --- a/providers/standard/docs/operators/latest_only.rst +++ b/providers/standard/docs/operators/latest_only.rst @@ -24,7 +24,7 @@ LatestOnlyOperator Use the :class:`~airflow.providers.standard.operators.latest_only.LatestOnlyOperator`. -.. exampleinclude:: /../../../airflow-core/src/airflow/example_dags/example_latest_only.py +.. exampleinclude:: /../tests/system/standard/example_latest_only.py :language: python :dedent: 4 :start-after: [START howto_operator_latest_only] diff --git a/providers/standard/docs/operators/python.rst b/providers/standard/docs/operators/python.rst index adb7f55828619..8f6f8d7d68159 100644 --- a/providers/standard/docs/operators/python.rst +++ b/providers/standard/docs/operators/python.rst @@ -32,7 +32,7 @@ Use the :class:`~airflow.providers.standard.operators.python.PythonOperator` to .. tab-item:: @task :sync: taskflow - .. exampleinclude:: /../../../airflow-core/src/airflow/example_dags/example_python_decorator.py + .. exampleinclude:: /../tests/system/standard/example_python_decorator.py :language: python :dedent: 4 :start-after: [START howto_operator_python] @@ -41,7 +41,7 @@ Use the :class:`~airflow.providers.standard.operators.python.PythonOperator` to .. tab-item:: PythonOperator :sync: operator - .. exampleinclude:: /../../../airflow-core/src/airflow/example_dags/example_python_operator.py + .. exampleinclude:: /../tests/system/standard/example_python_operator.py :language: python :dedent: 4 :start-after: [START howto_operator_python] @@ -57,7 +57,7 @@ Pass extra arguments to the ``@task`` decorated function as you would with a nor .. tab-item:: @task :sync: taskflow - .. exampleinclude:: /../../../airflow-core/src/airflow/example_dags/example_python_decorator.py + .. exampleinclude:: /../tests/system/standard/example_python_decorator.py :language: python :dedent: 4 :start-after: [START howto_operator_python_kwargs] @@ -66,7 +66,7 @@ Pass extra arguments to the ``@task`` decorated function as you would with a nor .. tab-item:: PythonOperator :sync: operator - .. exampleinclude:: /../../../airflow-core/src/airflow/example_dags/example_python_operator.py + .. exampleinclude:: /../tests/system/standard/example_python_operator.py :language: python :dedent: 4 :start-after: [START howto_operator_python_kwargs] @@ -87,7 +87,7 @@ is evaluated as a :ref:`Jinja template `. .. tab-item:: @task :sync: taskflow - .. exampleinclude:: /../../../airflow-core/src/airflow/example_dags/example_python_decorator.py + .. exampleinclude:: /../tests/system/standard/example_python_decorator.py :language: python :dedent: 4 :start-after: [START howto_operator_python_render_sql] @@ -96,7 +96,7 @@ is evaluated as a :ref:`Jinja template `. .. tab-item:: PythonOperator :sync: operator - .. exampleinclude:: /../../../airflow-core/src/airflow/example_dags/example_python_operator.py + .. exampleinclude:: /../tests/system/standard/example_python_operator.py :language: python :dedent: 4 :start-after: [START howto_operator_python_render_sql] @@ -137,7 +137,7 @@ smoother data exchange, while still effectively handling common Python objects a .. tab-item:: @task.virtualenv :sync: taskflow - .. exampleinclude:: /../../../airflow-core/src/airflow/example_dags/example_python_decorator.py + .. exampleinclude:: /../tests/system/standard/example_python_decorator.py :language: python :dedent: 4 :start-after: [START howto_operator_python_venv] @@ -146,7 +146,7 @@ smoother data exchange, while still effectively handling common Python objects a .. tab-item:: PythonVirtualenvOperator :sync: operator - .. exampleinclude:: /../../../airflow-core/src/airflow/example_dags/example_python_operator.py + .. exampleinclude:: /../tests/system/standard/example_python_operator.py :language: python :dedent: 4 :start-after: [START howto_operator_python_venv] @@ -251,7 +251,7 @@ in main Airflow environment). .. tab-item:: @task.external_python :sync: taskflow - .. exampleinclude:: /../../../airflow-core/src/airflow/example_dags/example_python_decorator.py + .. exampleinclude:: /../tests/system/standard/example_python_decorator.py :language: python :dedent: 4 :start-after: [START howto_operator_external_python] @@ -260,7 +260,7 @@ in main Airflow environment). .. tab-item:: ExternalPythonOperator :sync: operator - .. exampleinclude:: /../../../airflow-core/src/airflow/example_dags/example_python_operator.py + .. exampleinclude:: /../tests/system/standard/example_python_operator.py :language: python :dedent: 4 :start-after: [START howto_operator_external_python] @@ -308,7 +308,7 @@ tasks. .. tab-item:: @task.branch :sync: taskflow - .. exampleinclude:: /../../../airflow-core/src/airflow/example_dags/example_branch_operator_decorator.py + .. exampleinclude:: /../tests/system/standard/example_branch_operator_decorator.py :language: python :dedent: 4 :start-after: [START howto_operator_branch_python] @@ -317,7 +317,7 @@ tasks. .. tab-item:: PythonBranchOperator :sync: operator - .. exampleinclude:: /../../../airflow-core/src/airflow/example_dags/example_branch_operator.py + .. exampleinclude:: /../tests/system/standard/example_branch_operator.py :language: python :dedent: 4 :start-after: [START howto_operator_branch_python] @@ -345,7 +345,7 @@ tasks and is a hybrid of the :class:`~airflow.providers.standard.operators.pytho .. tab-item:: @task.branch_virtualenv :sync: taskflow - .. exampleinclude:: /../../../airflow-core/src/airflow/example_dags/example_branch_operator_decorator.py + .. exampleinclude:: /../tests/system/standard/example_branch_operator_decorator.py :language: python :dedent: 4 :start-after: [START howto_operator_branch_virtualenv] @@ -354,7 +354,7 @@ tasks and is a hybrid of the :class:`~airflow.providers.standard.operators.pytho .. tab-item:: BranchPythonVirtualenvOperator :sync: operator - .. exampleinclude:: /../../../airflow-core/src/airflow/example_dags/example_branch_operator.py + .. exampleinclude:: /../tests/system/standard/example_branch_operator.py :language: python :dedent: 4 :start-after: [START howto_operator_branch_virtualenv] @@ -383,7 +383,7 @@ external Python environment. .. tab-item:: @task.branch_external_python :sync: taskflow - .. exampleinclude:: /../../../airflow-core/src/airflow/example_dags/example_branch_operator_decorator.py + .. exampleinclude:: /../tests/system/standard/example_branch_operator_decorator.py :language: python :dedent: 4 :start-after: [START howto_operator_branch_ext_py] @@ -392,7 +392,7 @@ external Python environment. .. tab-item:: BranchExternalPythonOperator :sync: operator - .. exampleinclude:: /../../../airflow-core/src/airflow/example_dags/example_branch_operator.py + .. exampleinclude:: /../tests/system/standard/example_branch_operator.py :language: python :dedent: 4 :start-after: [START howto_operator_branch_ext_py] @@ -428,7 +428,7 @@ skipped. .. tab-item:: @task.short_circuit :sync: taskflow - .. exampleinclude:: /../../../airflow-core/src/airflow/example_dags/example_short_circuit_decorator.py + .. exampleinclude:: /../tests/system/standard/example_short_circuit_decorator.py :language: python :dedent: 4 :start-after: [START howto_operator_short_circuit] @@ -437,7 +437,7 @@ skipped. .. tab-item:: ShortCircuitOperator :sync: operator - .. exampleinclude:: /../../../airflow-core/src/airflow/example_dags/example_short_circuit_operator.py + .. exampleinclude:: /../tests/system/standard/example_short_circuit_operator.py :language: python :dedent: 4 :start-after: [START howto_operator_short_circuit] @@ -463,7 +463,7 @@ tasks have completed running regardless of status (i.e. the ``TriggerRule.ALL_DO .. tab-item:: @task.short_circuit :sync: taskflow - .. exampleinclude:: /../../../airflow-core/src/airflow/example_dags/example_short_circuit_decorator.py + .. exampleinclude:: /../tests/system/standard/example_short_circuit_decorator.py :language: python :dedent: 4 :start-after: [START howto_operator_short_circuit_trigger_rules] @@ -472,7 +472,7 @@ tasks have completed running regardless of status (i.e. the ``TriggerRule.ALL_DO .. tab-item:: ShortCircuitOperator :sync: operator - .. exampleinclude:: /../../../airflow-core/src/airflow/example_dags/example_short_circuit_operator.py + .. exampleinclude:: /../tests/system/standard/example_short_circuit_operator.py :language: python :dedent: 4 :start-after: [START howto_operator_short_circuit_trigger_rules] diff --git a/providers/standard/docs/operators/trigger_dag_run.rst b/providers/standard/docs/operators/trigger_dag_run.rst index b23e79f0ff34e..1db20b35eceef 100644 --- a/providers/standard/docs/operators/trigger_dag_run.rst +++ b/providers/standard/docs/operators/trigger_dag_run.rst @@ -24,7 +24,7 @@ TriggerDagRunOperator Use the :class:`~airflow.providers.standard.operators.trigger_dagrun.TriggerDagRunOperator` to trigger dag from another dag. -.. exampleinclude:: /../../../airflow-core/src/airflow/example_dags/example_trigger_controller_dag.py +.. exampleinclude:: /../tests/system/standard/example_trigger_controller_dag.py :language: python :dedent: 4 :start-after: [START howto_operator_trigger_dagrun] diff --git a/providers/standard/docs/sensors/bash.rst b/providers/standard/docs/sensors/bash.rst index 2fdb32419cc66..5acabb1172cf1 100644 --- a/providers/standard/docs/sensors/bash.rst +++ b/providers/standard/docs/sensors/bash.rst @@ -25,7 +25,7 @@ BashSensor Use the :class:`~airflow.providers.standard.sensors.bash.BashSensor` to use arbitrary command for sensing. The command should return 0 when it succeeds, any other value otherwise. -.. exampleinclude:: /../../../airflow-core/src/airflow/example_dags/example_sensors.py +.. exampleinclude:: /../tests/system/standard/example_sensors.py :language: python :dedent: 4 :start-after: [START example_bash_sensors] diff --git a/providers/standard/docs/sensors/datetime.rst b/providers/standard/docs/sensors/datetime.rst index b222650b57f82..aabff0f866dce 100644 --- a/providers/standard/docs/sensors/datetime.rst +++ b/providers/standard/docs/sensors/datetime.rst @@ -25,7 +25,7 @@ TimeDeltaSensor Use the :class:`~airflow.providers.standard.sensors.time_delta.TimeDeltaSensor` to end sensing after specific time. -.. exampleinclude:: /../../../airflow-core/src/airflow/example_dags/example_sensors.py +.. exampleinclude:: /../tests/system/standard/example_sensors.py :language: python :dedent: 4 :start-after: [START example_time_delta_sensor] @@ -41,7 +41,7 @@ Use the :class:`~airflow.providers.standard.sensors.time_delta.TimeDeltaSensorAs It is an async version of the operator and requires Triggerer to run. -.. exampleinclude:: /../../../airflow-core/src/airflow/example_dags/example_sensors.py +.. exampleinclude:: /../tests/system/standard/example_sensors.py :language: python :dedent: 4 :start-after: [START example_time_delta_sensor_async] @@ -58,7 +58,7 @@ Use the :class:`~airflow.providers.standard.sensors.time_sensor.TimeSensor` to e Time will be evaluated against ``data_interval_end`` if present for the dag run, otherwise ``run_after`` will be used. -.. exampleinclude:: /../../../airflow-core/src/airflow/example_dags/example_sensors.py +.. exampleinclude:: /../tests/system/standard/example_sensors.py :language: python :dedent: 4 :start-after: [START example_time_sensors] @@ -75,7 +75,7 @@ It is an async version of the operator and requires Triggerer to run. Time will be evaluated against ``data_interval_end`` if present for the dag run, otherwise ``run_after`` will be used. -.. exampleinclude:: /../../../airflow-core/src/airflow/example_dags/example_sensors.py +.. exampleinclude:: /../tests/system/standard/example_sensors.py :language: python :dedent: 4 :start-after: [START example_time_sensors_async] @@ -88,7 +88,7 @@ DayOfWeekSensor Use the :class:`~airflow.sensors.weekday.DayOfWeekSensor` to sense for day of week. -.. exampleinclude:: /../../../airflow-core/src/airflow/example_dags/example_sensors.py +.. exampleinclude:: /../tests/system/standard/example_sensors.py :language: python :dedent: 4 :start-after: [START example_day_of_week_sensor] diff --git a/providers/standard/docs/sensors/external_task_sensor.rst b/providers/standard/docs/sensors/external_task_sensor.rst index 30cbe4fdf0f1f..8aec0bb785891 100644 --- a/providers/standard/docs/sensors/external_task_sensor.rst +++ b/providers/standard/docs/sensors/external_task_sensor.rst @@ -47,7 +47,7 @@ wait for another task on a different DAG for a specific ``execution_date``. ExternalTaskSensor also provide options to set if the Task on a remote DAG succeeded or failed via ``allowed_states`` and ``failed_states`` parameters. -.. exampleinclude:: /../../../airflow-core/src/airflow/example_dags/example_external_task_marker_dag.py +.. exampleinclude:: /../tests/system/standard/example_external_task_marker_dag.py :language: python :dedent: 4 :start-after: [START howto_operator_external_task_sensor] @@ -55,7 +55,7 @@ via ``allowed_states`` and ``failed_states`` parameters. Also for this action you can use sensor in the deferrable mode: -.. exampleinclude:: /../../../airflow-core/tests/system/core/example_external_task_parent_deferrable.py +.. exampleinclude:: /../tests/system/standard/example_external_task_parent_deferrable.py :language: python :dedent: 4 :start-after: [START howto_external_task_async_sensor] @@ -67,7 +67,7 @@ ExternalTaskSensor with task_group dependency In Addition, we can also use the :class:`~airflow.providers.standard.sensors.external_task.ExternalTaskSensor` to make tasks on a DAG wait for another ``task_group`` on a different DAG for a specific ``execution_date``. -.. exampleinclude:: /../../../airflow-core/src/airflow/example_dags/example_external_task_marker_dag.py +.. exampleinclude:: /../tests/system/standard/example_external_task_marker_dag.py :language: python :dedent: 4 :start-after: [START howto_operator_external_task_sensor_with_task_group] @@ -81,7 +81,7 @@ on ``child_dag`` for a specific ``execution_date`` should also be cleared, ``Ext should be used. Note that ``child_task1`` will only be cleared if "Recursive" is selected when the user clears ``parent_task``. -.. exampleinclude:: /../../../airflow-core/src/airflow/example_dags/example_external_task_marker_dag.py +.. exampleinclude:: /../tests/system/standard/example_external_task_marker_dag.py :language: python :dedent: 4 :start-after: [START howto_operator_external_task_marker] diff --git a/providers/standard/docs/sensors/file.rst b/providers/standard/docs/sensors/file.rst index 03b4d57722b70..a2f2b42ce3fe7 100644 --- a/providers/standard/docs/sensors/file.rst +++ b/providers/standard/docs/sensors/file.rst @@ -26,7 +26,7 @@ Use the :class:`~airflow.providers.standard.sensors.filesystem.FileSensor` to de filesystem. You need to have connection defined to use it (pass connection id via ``fs_conn_id``). Default connection is ``fs_default``. -.. exampleinclude:: /../../../airflow-core/src/airflow/example_dags/example_sensors.py +.. exampleinclude:: /../tests/system/standard/example_sensors.py :language: python :dedent: 4 :start-after: [START example_file_sensor] @@ -34,7 +34,7 @@ Default connection is ``fs_default``. Also for this job you can use sensor in the deferrable mode: -.. exampleinclude:: /../../../airflow-core/src/airflow/example_dags/example_sensors.py +.. exampleinclude:: /../tests/system/standard/example_sensors.py :language: python :dedent: 4 :start-after: [START example_file_sensor_async] diff --git a/providers/standard/docs/sensors/python.rst b/providers/standard/docs/sensors/python.rst index 2702ef6408fe0..1850a272d3050 100644 --- a/providers/standard/docs/sensors/python.rst +++ b/providers/standard/docs/sensors/python.rst @@ -34,7 +34,7 @@ value to be True. .. tab-item:: @task.sensor :sync: taskflow - .. exampleinclude:: /../../../airflow-core/src/airflow/example_dags/example_sensor_decorator.py + .. exampleinclude:: /../tests/system/standard/example_sensor_decorator.py :language: python :dedent: 4 :start-after: [START wait_function] @@ -43,7 +43,7 @@ value to be True. .. tab-item:: PythonSensor :sync: operator - .. exampleinclude:: /../../../airflow-core/src/airflow/example_dags/example_sensors.py + .. exampleinclude:: /../tests/system/standard/example_sensors.py :language: python :dedent: 4 :start-after: [START example_python_sensors] diff --git a/providers/standard/tests/system/__init__.py b/providers/standard/tests/system/__init__.py new file mode 100644 index 0000000000000..e8fd22856438c --- /dev/null +++ b/providers/standard/tests/system/__init__.py @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +__path__ = __import__("pkgutil").extend_path(__path__, __name__) # type: ignore diff --git a/providers/standard/tests/system/standard/__init__.py b/providers/standard/tests/system/standard/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/providers/standard/tests/system/standard/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/airflow-core/src/airflow/example_dags/example_bash_decorator.py b/providers/standard/tests/system/standard/example_bash_decorator.py similarity index 95% rename from airflow-core/src/airflow/example_dags/example_bash_decorator.py rename to providers/standard/tests/system/standard/example_bash_decorator.py index 89335c0686465..ce95b62ce6f4c 100644 --- a/airflow-core/src/airflow/example_dags/example_bash_decorator.py +++ b/providers/standard/tests/system/standard/example_bash_decorator.py @@ -112,3 +112,9 @@ def get_file_stats() -> str: example_bash_decorator() + + +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 + +# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +test_run = get_test_run(dag) diff --git a/airflow-core/src/airflow/example_dags/example_bash_operator.py b/providers/standard/tests/system/standard/example_bash_operator.py similarity index 92% rename from airflow-core/src/airflow/example_dags/example_bash_operator.py rename to providers/standard/tests/system/standard/example_bash_operator.py index a4fb3161ab67d..7ef7c0db2efbe 100644 --- a/airflow-core/src/airflow/example_dags/example_bash_operator.py +++ b/providers/standard/tests/system/standard/example_bash_operator.py @@ -72,3 +72,9 @@ ) # [END howto_operator_bash_skip] this_will_skip >> run_this_last + + +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 + +# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +test_run = get_test_run(dag) diff --git a/airflow-core/src/airflow/example_dags/example_branch_datetime_operator.py b/providers/standard/tests/system/standard/example_branch_datetime_operator.py similarity index 93% rename from airflow-core/src/airflow/example_dags/example_branch_datetime_operator.py rename to providers/standard/tests/system/standard/example_branch_datetime_operator.py index 710bdb1de898c..b1e7564133323 100644 --- a/airflow-core/src/airflow/example_dags/example_branch_datetime_operator.py +++ b/providers/standard/tests/system/standard/example_branch_datetime_operator.py @@ -103,3 +103,11 @@ # Run empty_task_13 if cond3 executes between 2020-10-10 14:00:00 and 2020-10-10 15:00:00 cond3 >> [empty_task_13, empty_task_23] # [END howto_branch_datetime_operator_logical_date] + + +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 + +# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +test_run = get_test_run(dag1) +test_run2 = get_test_run(dag2) +test_run3 = get_test_run(dag3) diff --git a/airflow-core/src/airflow/example_dags/example_branch_day_of_week_operator.py b/providers/standard/tests/system/standard/example_branch_day_of_week_operator.py similarity index 92% rename from airflow-core/src/airflow/example_dags/example_branch_day_of_week_operator.py rename to providers/standard/tests/system/standard/example_branch_day_of_week_operator.py index 43400522468f7..3b541d2712d44 100644 --- a/airflow-core/src/airflow/example_dags/example_branch_day_of_week_operator.py +++ b/providers/standard/tests/system/standard/example_branch_day_of_week_operator.py @@ -59,3 +59,9 @@ # Run empty_task_3 if it's a weekend, empty_task_4 otherwise empty_task_2 >> branch_weekend >> [empty_task_3, empty_task_4] # [END howto_operator_day_of_week_branch] + + +from tests_common.test_utils.system_tests import get_test_run + +# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +test_run = get_test_run(dag) diff --git a/airflow-core/src/airflow/example_dags/example_branch_operator.py b/providers/standard/tests/system/standard/example_branch_operator.py similarity index 96% rename from airflow-core/src/airflow/example_dags/example_branch_operator.py rename to providers/standard/tests/system/standard/example_branch_operator.py index 0bb429a6f8fe0..98c85444cdd3a 100644 --- a/airflow-core/src/airflow/example_dags/example_branch_operator.py +++ b/providers/standard/tests/system/standard/example_branch_operator.py @@ -164,3 +164,9 @@ def hello_world_with_venv(): # Label is optional here, but it can help identify more complex branches branching_venv >> Label(option) >> t >> join_venv + + +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 + +# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +test_run = get_test_run(dag) diff --git a/airflow-core/src/airflow/example_dags/example_branch_operator_decorator.py b/providers/standard/tests/system/standard/example_branch_operator_decorator.py similarity index 96% rename from airflow-core/src/airflow/example_dags/example_branch_operator_decorator.py rename to providers/standard/tests/system/standard/example_branch_operator_decorator.py index d3634acb998eb..3a3b34e50d11f 100644 --- a/airflow-core/src/airflow/example_dags/example_branch_operator_decorator.py +++ b/providers/standard/tests/system/standard/example_branch_operator_decorator.py @@ -140,3 +140,9 @@ def some_venv_task(): # Label is optional here, but it can help identify more complex branches random_choice_venv >> Label(option) >> t >> join_venv + + +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 + +# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +test_run = get_test_run(dag) diff --git a/airflow-core/tests/system/core/example_external_task_child_deferrable.py b/providers/standard/tests/system/standard/example_external_task_child_deferrable.py similarity index 100% rename from airflow-core/tests/system/core/example_external_task_child_deferrable.py rename to providers/standard/tests/system/standard/example_external_task_child_deferrable.py diff --git a/airflow-core/src/airflow/example_dags/example_external_task_marker_dag.py b/providers/standard/tests/system/standard/example_external_task_marker_dag.py similarity index 94% rename from airflow-core/src/airflow/example_dags/example_external_task_marker_dag.py rename to providers/standard/tests/system/standard/example_external_task_marker_dag.py index b8fad182549f1..d310f369611bf 100644 --- a/airflow-core/src/airflow/example_dags/example_external_task_marker_dag.py +++ b/providers/standard/tests/system/standard/example_external_task_marker_dag.py @@ -96,3 +96,9 @@ child_task3 = EmptyOperator(task_id="child_task3") child_task1 >> child_task2 >> child_task3 + + +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 + +# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +test_run = get_test_run(parent_dag) diff --git a/airflow-core/tests/system/core/example_external_task_parent_deferrable.py b/providers/standard/tests/system/standard/example_external_task_parent_deferrable.py similarity index 100% rename from airflow-core/tests/system/core/example_external_task_parent_deferrable.py rename to providers/standard/tests/system/standard/example_external_task_parent_deferrable.py diff --git a/airflow-core/src/airflow/example_dags/example_latest_only.py b/providers/standard/tests/system/standard/example_latest_only.py similarity index 88% rename from airflow-core/src/airflow/example_dags/example_latest_only.py rename to providers/standard/tests/system/standard/example_latest_only.py index 745100def2af2..983db906bdb5b 100644 --- a/airflow-core/src/airflow/example_dags/example_latest_only.py +++ b/providers/standard/tests/system/standard/example_latest_only.py @@ -38,3 +38,9 @@ task1 = EmptyOperator(task_id="task1") latest_only >> task1 + + +from tests_common.test_utils.system_tests import get_test_run + +# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +test_run = get_test_run(dag) diff --git a/airflow-core/src/airflow/example_dags/example_python_decorator.py b/providers/standard/tests/system/standard/example_python_decorator.py similarity index 94% rename from airflow-core/src/airflow/example_dags/example_python_decorator.py rename to providers/standard/tests/system/standard/example_python_decorator.py index 4f9c05f7940c8..8b26d94842632 100644 --- a/airflow-core/src/airflow/example_dags/example_python_decorator.py +++ b/providers/standard/tests/system/standard/example_python_decorator.py @@ -129,4 +129,10 @@ def callable_external_python(): run_this >> external_python_task >> virtualenv_task -example_python_decorator() +example_dag = example_python_decorator() + + +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 + +# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +test_run = get_test_run(example_dag) diff --git a/airflow-core/src/airflow/example_dags/example_python_operator.py b/providers/standard/tests/system/standard/example_python_operator.py similarity index 95% rename from airflow-core/src/airflow/example_dags/example_python_operator.py rename to providers/standard/tests/system/standard/example_python_operator.py index d48bcae483c92..27ca2db12e995 100644 --- a/airflow-core/src/airflow/example_dags/example_python_operator.py +++ b/providers/standard/tests/system/standard/example_python_operator.py @@ -47,7 +47,7 @@ start_date=pendulum.datetime(2021, 1, 1, tz="UTC"), catchup=False, tags=["example"], -): +) as dag: # [START howto_operator_python] def print_context(ds=None, **kwargs): """Print the Airflow context and ds variable from the context.""" @@ -145,3 +145,9 @@ def callable_external_python(): # [END howto_operator_external_python] run_this >> external_python_task >> virtualenv_task + + +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 + +# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +test_run = get_test_run(dag) diff --git a/airflow-core/src/airflow/example_dags/example_sensor_decorator.py b/providers/standard/tests/system/standard/example_sensor_decorator.py similarity index 90% rename from airflow-core/src/airflow/example_dags/example_sensor_decorator.py rename to providers/standard/tests/system/standard/example_sensor_decorator.py index 64ea80400ceb8..eaceebab9f387 100644 --- a/airflow-core/src/airflow/example_dags/example_sensor_decorator.py +++ b/providers/standard/tests/system/standard/example_sensor_decorator.py @@ -64,3 +64,9 @@ def dummy_operator() -> None: # [END dag_invocation] # [END tutorial] + + +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 + +# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +test_run = get_test_run(dag) diff --git a/airflow-core/src/airflow/example_dags/example_sensors.py b/providers/standard/tests/system/standard/example_sensors.py similarity index 95% rename from airflow-core/src/airflow/example_dags/example_sensors.py rename to providers/standard/tests/system/standard/example_sensors.py index 5ec33bad51bf5..b5ea2f458fd94 100644 --- a/airflow-core/src/airflow/example_dags/example_sensors.py +++ b/providers/standard/tests/system/standard/example_sensors.py @@ -130,3 +130,9 @@ def failure_callable(): t8 >> tx [t9, t10] >> tx t11 >> tx + + +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 + +# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +test_run = get_test_run(dag) diff --git a/airflow-core/src/airflow/example_dags/example_short_circuit_decorator.py b/providers/standard/tests/system/standard/example_short_circuit_decorator.py similarity index 92% rename from airflow-core/src/airflow/example_dags/example_short_circuit_decorator.py rename to providers/standard/tests/system/standard/example_short_circuit_decorator.py index 714325b1739d2..93b32a924502d 100644 --- a/airflow-core/src/airflow/example_dags/example_short_circuit_decorator.py +++ b/providers/standard/tests/system/standard/example_short_circuit_decorator.py @@ -58,3 +58,9 @@ def check_condition(condition): example_dag = example_short_circuit_decorator() + + +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 + +# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +test_run = get_test_run(example_dag) diff --git a/airflow-core/src/airflow/example_dags/example_short_circuit_operator.py b/providers/standard/tests/system/standard/example_short_circuit_operator.py similarity index 92% rename from airflow-core/src/airflow/example_dags/example_short_circuit_operator.py rename to providers/standard/tests/system/standard/example_short_circuit_operator.py index 494bd55a8697a..892be8934ae3a 100644 --- a/airflow-core/src/airflow/example_dags/example_short_circuit_operator.py +++ b/providers/standard/tests/system/standard/example_short_circuit_operator.py @@ -32,7 +32,7 @@ start_date=pendulum.datetime(2021, 1, 1, tz="UTC"), catchup=False, tags=["example"], -): +) as dag: # [START howto_operator_short_circuit] cond_true = ShortCircuitOperator( task_id="condition_is_True", @@ -64,3 +64,9 @@ chain(task_1, [task_2, short_circuit], [task_3, task_4], [task_5, task_6], task_7) # [END howto_operator_short_circuit_trigger_rules] + + +from tests_common.test_utils.system_tests import get_test_run + +# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +test_run = get_test_run(dag) diff --git a/airflow-core/src/airflow/example_dags/example_trigger_controller_dag.py b/providers/standard/tests/system/standard/example_trigger_controller_dag.py similarity index 90% rename from airflow-core/src/airflow/example_dags/example_trigger_controller_dag.py rename to providers/standard/tests/system/standard/example_trigger_controller_dag.py index 8ca49e34b3dcd..3a85fb7e0aff1 100644 --- a/airflow-core/src/airflow/example_dags/example_trigger_controller_dag.py +++ b/providers/standard/tests/system/standard/example_trigger_controller_dag.py @@ -44,3 +44,9 @@ ) # [END howto_operator_trigger_dagrun] + + +from tests_common.test_utils.system_tests import get_test_run + +# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +test_run = get_test_run(dag) diff --git a/providers/standard/tests/system/standard/sql/__init__.py b/providers/standard/tests/system/standard/sql/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/providers/standard/tests/system/standard/sql/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/providers/standard/tests/system/standard/sql/sample.sql b/providers/standard/tests/system/standard/sql/sample.sql new file mode 100644 index 0000000000000..23af6ab4b9bb3 --- /dev/null +++ b/providers/standard/tests/system/standard/sql/sample.sql @@ -0,0 +1,24 @@ +/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/ + +CREATE TABLE Orders ( + order_id INT PRIMARY KEY, + name TEXT, + description TEXT +) diff --git a/scripts/ci/pre_commit/check_system_tests.py b/scripts/ci/pre_commit/check_system_tests.py index 3d5c743b54f78..15f94a8cdfa6a 100755 --- a/scripts/ci/pre_commit/check_system_tests.py +++ b/scripts/ci/pre_commit/check_system_tests.py @@ -46,7 +46,7 @@ PYTEST_FUNCTION_PATTERN = re.compile( r"from tests_common\.test_utils\.system_tests import get_test_run(?: # noqa: E402)?\s+" r"(?:# .+\))?\s+" - r"test_run = get_test_run\(dag\)" + r"test_run = get_test_run" ) diff --git a/scripts/docker/entrypoint_ci.sh b/scripts/docker/entrypoint_ci.sh index a99c961c67528..adf37ce738592 100755 --- a/scripts/docker/entrypoint_ci.sh +++ b/scripts/docker/entrypoint_ci.sh @@ -178,6 +178,12 @@ function environment_initialization() { cd "${AIRFLOW_SOURCES}" + # Temporarily add /opt/airflow/providers/standard/tests to PYTHONPATH in order to see example dags + # in the UI when testing in Breeze. This might be solved differently in the future + if [[ -d /opt/airflow/providers/standard/tests ]]; then + export PYTHONPATH=${PYTHONPATH=}:/opt/airflow/providers/standard/tests + fi + if [[ ${START_AIRFLOW:="false"} == "true" || ${START_AIRFLOW} == "True" ]]; then export AIRFLOW__CORE__LOAD_EXAMPLES=${LOAD_EXAMPLES} wait_for_asset_compilation