diff --git a/generated/provider_dependencies.json b/generated/provider_dependencies.json index d7970a8c116f8..f09c78d75c91a 100644 --- a/generated/provider_dependencies.json +++ b/generated/provider_dependencies.json @@ -495,6 +495,7 @@ "dbt.cloud": { "deps": [ "aiohttp>=3.9.2", + "apache-airflow-providers-common-compat>=1.6.0", "apache-airflow-providers-http", "apache-airflow>=2.9.0", "asgiref>=2.3.0" diff --git a/providers/dbt/cloud/README.rst b/providers/dbt/cloud/README.rst index 31d55c2dd18d4..bbf2ab3079b26 100644 --- a/providers/dbt/cloud/README.rst +++ b/providers/dbt/cloud/README.rst @@ -50,14 +50,15 @@ The package supports the following python versions: 3.9,3.10,3.11,3.12 Requirements ------------ -================================= ================== -PIP package Version required -================================= ================== -``apache-airflow`` ``>=2.9.0`` +========================================== ================== +PIP package Version required +========================================== ================== +``apache-airflow`` ``>=2.9.0`` +``apache-airflow-providers-common-compat`` ``>=1.6.0`` ``apache-airflow-providers-http`` -``asgiref`` ``>=2.3.0`` -``aiohttp`` ``>=3.9.2`` -================================= ================== +``asgiref`` ``>=2.3.0`` +``aiohttp`` ``>=3.9.2`` +========================================== ================== Cross provider package dependencies ----------------------------------- diff --git a/providers/dbt/cloud/pyproject.toml b/providers/dbt/cloud/pyproject.toml index ba0f168022186..45eea06e4aef7 100644 --- a/providers/dbt/cloud/pyproject.toml +++ b/providers/dbt/cloud/pyproject.toml @@ -58,6 +58,7 @@ requires-python = "~=3.9" # After you modify the dependencies, and rebuild your Breeze CI image with ``breeze ci-image build`` dependencies = [ "apache-airflow>=2.9.0", + "apache-airflow-providers-common-compat>=1.6.0", "apache-airflow-providers-http", "asgiref>=2.3.0", "aiohttp>=3.9.2", @@ -68,10 +69,7 @@ dependencies = [ [project.optional-dependencies] # pip install apache-airflow-providers-dbt-cloud[openlineage] "openlineage" = [ - "apache-airflow-providers-openlineage>=1.7.0", -] -"common.compat" = [ - "apache-airflow-providers-common-compat" + "apache-airflow-providers-openlineage>=2.0.0", ] [dependency-groups] diff --git a/providers/dbt/cloud/src/airflow/providers/dbt/cloud/get_provider_info.py b/providers/dbt/cloud/src/airflow/providers/dbt/cloud/get_provider_info.py index 450c58410f482..6d0813d9648cc 100644 --- a/providers/dbt/cloud/src/airflow/providers/dbt/cloud/get_provider_info.py +++ b/providers/dbt/cloud/src/airflow/providers/dbt/cloud/get_provider_info.py @@ -96,13 +96,11 @@ def get_provider_info(): "extra-links": ["airflow.providers.dbt.cloud.operators.dbt.DbtCloudRunJobOperatorLink"], "dependencies": [ "apache-airflow>=2.9.0", + "apache-airflow-providers-common-compat>=1.6.0", "apache-airflow-providers-http", "asgiref>=2.3.0", "aiohttp>=3.9.2", ], - "optional-dependencies": { - "openlineage": ["apache-airflow-providers-openlineage>=1.7.0"], - "common.compat": ["apache-airflow-providers-common-compat"], - }, + "optional-dependencies": {"openlineage": ["apache-airflow-providers-openlineage>=2.0.0"]}, "devel-dependencies": [], } diff --git a/providers/dbt/cloud/src/airflow/providers/dbt/cloud/utils/openlineage.py b/providers/dbt/cloud/src/airflow/providers/dbt/cloud/utils/openlineage.py index cd3fd9271623e..983ee48e63c99 100644 --- a/providers/dbt/cloud/src/airflow/providers/dbt/cloud/utils/openlineage.py +++ b/providers/dbt/cloud/src/airflow/providers/dbt/cloud/utils/openlineage.py @@ -21,6 +21,7 @@ import re from typing import TYPE_CHECKING +from airflow.providers.common.compat.openlineage.check import require_openlineage_version from airflow.providers.dbt.cloud.version_compat import AIRFLOW_V_2_10_PLUS, AIRFLOW_V_3_0_PLUS if TYPE_CHECKING: @@ -54,6 +55,7 @@ def _get_try_number(val): return val.try_number - 1 +@require_openlineage_version(provider_min_version="2.0.0") def generate_openlineage_events_from_dbt_cloud_run( operator: DbtCloudRunJobOperator | DbtCloudJobRunSensor, task_instance: TaskInstance ) -> OperatorLineage: @@ -73,14 +75,7 @@ def generate_openlineage_events_from_dbt_cloud_run( """ from openlineage.common.provider.dbt import DbtCloudArtifactProcessor, ParentRunMetadata - try: - from airflow.providers.openlineage.conf import namespace - except ModuleNotFoundError as e: - from airflow.exceptions import AirflowOptionalProviderFeatureException - - msg = "Please install `apache-airflow-providers-openlineage>=1.7.0`" - raise AirflowOptionalProviderFeatureException(e, msg) - + from airflow.providers.openlineage.conf import namespace from airflow.providers.openlineage.extractors import OperatorLineage from airflow.providers.openlineage.plugins.adapter import ( _PRODUCER, diff --git a/providers/dbt/cloud/tests/unit/dbt/cloud/utils/test_openlineage.py b/providers/dbt/cloud/tests/unit/dbt/cloud/utils/test_openlineage.py index d7d753cc2a465..b96cb549eb7b9 100644 --- a/providers/dbt/cloud/tests/unit/dbt/cloud/utils/test_openlineage.py +++ b/providers/dbt/cloud/tests/unit/dbt/cloud/utils/test_openlineage.py @@ -89,23 +89,24 @@ def get_dbt_artifact(*args, **kwargs): def test_previous_version_openlineage_provider(): - """When using OpenLineage, the dbt-cloud provider now depends on openlineage provider >= 1.7""" - original_import = __import__ + """When using OpenLineage, the dbt-cloud provider now depends on openlineage provider >= 2.0""" - def custom_import(name, *args, **kwargs): - if name == "airflow.providers.openlineage.conf": - raise ModuleNotFoundError("No module named 'airflow.providers.openlineage.conf") - else: - return original_import(name, *args, **kwargs) + def _mock_version(package): + if package == "apache-airflow-providers-openlineage": + return "1.99.0" + raise Exception("Unexpected package") mock_operator = MagicMock() mock_task_instance = MagicMock() - with patch("builtins.__import__", side_effect=custom_import): - with pytest.raises(AirflowOptionalProviderFeatureException) as exc: + expected_err = ( + "OpenLineage provider version `1.99.0` is lower than required `2.0.0`, " + "skipping function `generate_openlineage_events_from_dbt_cloud_run` execution" + ) + + with patch("importlib.metadata.version", side_effect=_mock_version): + with pytest.raises(AirflowOptionalProviderFeatureException, match=expected_err): generate_openlineage_events_from_dbt_cloud_run(mock_operator, mock_task_instance) - assert str(exc.value.args[0]) == "No module named 'airflow.providers.openlineage.conf" - assert str(exc.value.args[1]) == "Please install `apache-airflow-providers-openlineage>=1.7.0`" class TestGenerateOpenLineageEventsFromDbtCloudRun: