From 725e024307a60009f80204a85c4cdd8f348997ab Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Thu, 29 May 2025 21:01:54 +0200 Subject: [PATCH] [v3-0-test] Bring back "standard" example dags to the airflow-core package (#51192) As we are working on a longer-term solution for example-dags, the short-term one is to bring the standard example_dags back to the airflow-core via symbolic link to the standard package folder. This makes the dags to be copied to the airflow-core from the latest main version (but those dags don't change) while not requiring to duplicate the dags and keep them in standard provider where the documentation for them is kept. (cherry picked from commit 46908a146bd2ac076ba0bd63146eb41decd88adc) Co-authored-by: Jarek Potiuk Co-authored-by: Kaxil Naik --- .../airflow/dag_processing/bundles/manager.py | 26 ------------------- .../src/airflow/example_dags/standard | 1 + airflow-core/src/airflow/models/dagbag.py | 8 ------ .../core_api/routes/public/test_dag_report.py | 11 ++------ airflow-core/tests/unit/models/test_dagbag.py | 2 +- 5 files changed, 4 insertions(+), 44 deletions(-) create mode 120000 airflow-core/src/airflow/example_dags/standard diff --git a/airflow-core/src/airflow/dag_processing/bundles/manager.py b/airflow-core/src/airflow/dag_processing/bundles/manager.py index 1ace2896028c6..9760da617a1d9 100644 --- a/airflow-core/src/airflow/dag_processing/bundles/manager.py +++ b/airflow-core/src/airflow/dag_processing/bundles/manager.py @@ -16,7 +16,6 @@ # under the License. from __future__ import annotations -import os from typing import TYPE_CHECKING from airflow.configuration import conf @@ -35,7 +34,6 @@ from airflow.dag_processing.bundles.base import BaseDagBundle _example_dag_bundle_name = "example_dags" -_example_standard_dag_bundle_name = "example_standard_dags" def _bundle_item_exc(msg): @@ -82,25 +80,6 @@ def _add_example_dag_bundle(config_list): ) -def _add_example_standard_dag_bundle(config_list): - # TODO(potiuk): make it more generic - for now we only add standard example_dags if they are locally available - try: - from system import standard - except ImportError: - return - - example_dag_folder = next(iter(standard.__path__)) - config_list.append( - { - "name": _example_standard_dag_bundle_name, - "classpath": "airflow.dag_processing.bundles.local.LocalDagBundle", - "kwargs": { - "path": example_dag_folder, - }, - } - ) - - class DagBundlesManager(LoggingMixin): """Manager for DAG bundles.""" @@ -133,11 +112,6 @@ def parse_config(self) -> None: _validate_bundle_config(config_list) if conf.getboolean("core", "LOAD_EXAMPLES"): _add_example_dag_bundle(config_list) - if ( - os.environ.get("BREEZE", "").lower() == "true" - or os.environ.get("_IN_UNIT_TESTS", "").lower() == "true" - ): - _add_example_standard_dag_bundle(config_list) for cfg in config_list: name = cfg["name"] diff --git a/airflow-core/src/airflow/example_dags/standard b/airflow-core/src/airflow/example_dags/standard new file mode 120000 index 0000000000000..c5f1f37b5270a --- /dev/null +++ b/airflow-core/src/airflow/example_dags/standard @@ -0,0 +1 @@ +../../../../providers/standard/tests/system/standard/ \ No newline at end of file diff --git a/airflow-core/src/airflow/models/dagbag.py b/airflow-core/src/airflow/models/dagbag.py index d0c8bf98f4152..52c2337474d74 100644 --- a/airflow-core/src/airflow/models/dagbag.py +++ b/airflow-core/src/airflow/models/dagbag.py @@ -586,14 +586,6 @@ def collect_dags( example_dag_folder = next(iter(example_dags.__path__)) files_to_parse.extend(list_py_file_paths(example_dag_folder, safe_mode=safe_mode)) - try: - from system import standard - - example_dag_folder_standard = next(iter(standard.__path__)) - files_to_parse.extend(list_py_file_paths(example_dag_folder_standard, safe_mode=safe_mode)) - except ImportError: - # Nothing happens - this should only work during tests - pass for filepath in files_to_parse: try: diff --git a/airflow-core/tests/unit/api_fastapi/core_api/routes/public/test_dag_report.py b/airflow-core/tests/unit/api_fastapi/core_api/routes/public/test_dag_report.py index 314d6126b9669..3938894001de9 100644 --- a/airflow-core/tests/unit/api_fastapi/core_api/routes/public/test_dag_report.py +++ b/airflow-core/tests/unit/api_fastapi/core_api/routes/public/test_dag_report.py @@ -23,7 +23,6 @@ from airflow.utils.file import list_py_file_paths -from tests_common.pytest_plugin import AIRFLOW_ROOT_PATH from tests_common.test_utils.config import conf_vars from tests_common.test_utils.db import clear_db_dags, parse_and_sync_to_db @@ -34,18 +33,12 @@ TEST_DAG_FOLDER_INVALID = "/invalid/path" TEST_DAG_FOLDER_INVALID_2 = "/root/airflow/tests/dags/" -STANDARD_PROVIDER_SYSTEM_TESTS_PATH = ( - AIRFLOW_ROOT_PATH / "providers" / "standard" / "tests" / "system" / "standard" -) - def get_corresponding_dag_file_count(dir: str, include_examples: bool = True) -> int: from airflow import example_dags - return ( - len(list_py_file_paths(directory=dir)) - + (len(list_py_file_paths(next(iter(example_dags.__path__)))) if include_examples else 0) - + (len(list_py_file_paths(STANDARD_PROVIDER_SYSTEM_TESTS_PATH.as_posix())) if include_examples else 0) + return len(list_py_file_paths(directory=dir)) + ( + len(list_py_file_paths(next(iter(example_dags.__path__)))) if include_examples else 0 ) diff --git a/airflow-core/tests/unit/models/test_dagbag.py b/airflow-core/tests/unit/models/test_dagbag.py index 44eb06229ee89..6d6ebbf306988 100644 --- a/airflow-core/tests/unit/models/test_dagbag.py +++ b/airflow-core/tests/unit/models/test_dagbag.py @@ -52,7 +52,7 @@ pytestmark = pytest.mark.db_test -example_dags_folder = AIRFLOW_ROOT_PATH / "providers" / "standard" / "tests" / "system" / "standard" +example_dags_folder = AIRFLOW_ROOT_PATH / "airflow-core" / "src" / "airflow" / "example_dags" / "standard" PY311 = sys.version_info >= (3, 11)