From 38905af77c082c6b6311f56ed4899cae97063e93 Mon Sep 17 00:00:00 2001 From: Your Name Date: Wed, 22 Oct 2025 23:05:03 +0800 Subject: [PATCH 1/4] Apache/druid Migrate remaining providers to common.compat compatibility layer --- providers/apache/drill/pyproject.toml | 2 ++ providers/apache/druid/pyproject.toml | 2 ++ .../airflow/providers/apache/druid/hooks/druid.py | 2 +- .../providers/apache/druid/operators/druid.py | 4 ++-- .../apache/druid/transfers/hive_to_druid.py | 4 ++-- .../providers/apache/druid/version_compat.py | 15 --------------- 6 files changed, 9 insertions(+), 20 deletions(-) diff --git a/providers/apache/drill/pyproject.toml b/providers/apache/drill/pyproject.toml index b9c38c30c663e..b21dd6a675e9c 100644 --- a/providers/apache/drill/pyproject.toml +++ b/providers/apache/drill/pyproject.toml @@ -59,6 +59,7 @@ requires-python = ">=3.10" dependencies = [ "apache-airflow>=2.10.0", "apache-airflow-providers-common-sql>=1.26.0", + "apache-airflow-providers-common-compat>=1.7.4", # + TODO: bump to next version # Workaround until we get https://github.com/JohnOmernik/sqlalchemy-drill/issues/94 fixed. "sqlalchemy-drill>=1.1.0,!=1.1.6,!=1.1.7", ] @@ -69,6 +70,7 @@ dev = [ "apache-airflow-task-sdk", "apache-airflow-devel-common", "apache-airflow-providers-common-sql", + "apache-airflow-providers-common-compat", # Additional devel dependencies (do not remove this line and add extra development dependencies) "apache-airflow-providers-common-sql[pandas,polars]", ] diff --git a/providers/apache/druid/pyproject.toml b/providers/apache/druid/pyproject.toml index 9eb64fff9ef46..25632ac90abf3 100644 --- a/providers/apache/druid/pyproject.toml +++ b/providers/apache/druid/pyproject.toml @@ -59,6 +59,7 @@ requires-python = ">=3.10" dependencies = [ "apache-airflow>=2.10.0", "apache-airflow-providers-common-sql>=1.26.0", + "apache-airflow-providers-common-compat>=1.7.4", # + TODO: bump to next version "pydruid>=0.6.6", ] @@ -76,6 +77,7 @@ dev = [ "apache-airflow-devel-common", "apache-airflow-providers-apache-hive", "apache-airflow-providers-common-sql", + "apache-airflow-providers-common-compat", # Additional devel dependencies (do not remove this line and add extra development dependencies) "apache-airflow-providers-common-sql[pandas,polars]", ] diff --git a/providers/apache/druid/src/airflow/providers/apache/druid/hooks/druid.py b/providers/apache/druid/src/airflow/providers/apache/druid/hooks/druid.py index 4977970d9588d..682f7fafe2beb 100644 --- a/providers/apache/druid/src/airflow/providers/apache/druid/hooks/druid.py +++ b/providers/apache/druid/src/airflow/providers/apache/druid/hooks/druid.py @@ -27,7 +27,7 @@ from pydruid.db import connect from airflow.exceptions import AirflowException -from airflow.providers.apache.druid.version_compat import BaseHook +from airflow.providers.common.compat.sdk import BaseHook from airflow.providers.common.sql.hooks.sql import DbApiHook if TYPE_CHECKING: diff --git a/providers/apache/druid/src/airflow/providers/apache/druid/operators/druid.py b/providers/apache/druid/src/airflow/providers/apache/druid/operators/druid.py index cd9c70a06e06b..a8e5f0247a67b 100644 --- a/providers/apache/druid/src/airflow/providers/apache/druid/operators/druid.py +++ b/providers/apache/druid/src/airflow/providers/apache/druid/operators/druid.py @@ -21,10 +21,10 @@ from typing import TYPE_CHECKING, Any from airflow.providers.apache.druid.hooks.druid import DruidHook, IngestionType -from airflow.providers.apache.druid.version_compat import BaseOperator +from airflow.providers.common.compat.sdk import BaseOperator if TYPE_CHECKING: - from airflow.providers.apache.druid.version_compat import Context + from airflow.providers.common.compat.sdk import Context class DruidOperator(BaseOperator): diff --git a/providers/apache/druid/src/airflow/providers/apache/druid/transfers/hive_to_druid.py b/providers/apache/druid/src/airflow/providers/apache/druid/transfers/hive_to_druid.py index 5cd4db23c469e..97a2c85e5968f 100644 --- a/providers/apache/druid/src/airflow/providers/apache/druid/transfers/hive_to_druid.py +++ b/providers/apache/druid/src/airflow/providers/apache/druid/transfers/hive_to_druid.py @@ -23,11 +23,11 @@ from typing import TYPE_CHECKING, Any from airflow.providers.apache.druid.hooks.druid import DruidHook -from airflow.providers.apache.druid.version_compat import BaseOperator +from airflow.providers.common.compat.sdk import BaseOperator from airflow.providers.apache.hive.hooks.hive import HiveCliHook, HiveMetastoreHook if TYPE_CHECKING: - from airflow.providers.apache.druid.version_compat import Context + from airflow.providers.common.compat.sdk import Context LOAD_CHECK_INTERVAL = 5 DEFAULT_TARGET_PARTITION_SIZE = 5000000 diff --git a/providers/apache/druid/src/airflow/providers/apache/druid/version_compat.py b/providers/apache/druid/src/airflow/providers/apache/druid/version_compat.py index a9ce50c32c7a2..a4fae50317e81 100644 --- a/providers/apache/druid/src/airflow/providers/apache/druid/version_compat.py +++ b/providers/apache/druid/src/airflow/providers/apache/druid/version_compat.py @@ -35,22 +35,7 @@ def get_base_airflow_version_tuple() -> tuple[int, int, int]: AIRFLOW_V_3_0_PLUS = get_base_airflow_version_tuple() >= (3, 0, 0) AIRFLOW_V_3_1_PLUS: bool = get_base_airflow_version_tuple() >= (3, 1, 0) -if AIRFLOW_V_3_1_PLUS: - from airflow.sdk import BaseHook -else: - from airflow.hooks.base import BaseHook # type: ignore[attr-defined,no-redef] - -if AIRFLOW_V_3_0_PLUS: - from airflow.sdk import BaseOperator - from airflow.sdk.definitions.context import Context -else: - from airflow.models import BaseOperator - from airflow.utils.context import Context - __all__ = [ "AIRFLOW_V_3_0_PLUS", "AIRFLOW_V_3_1_PLUS", - "BaseHook", - "BaseOperator", - "Context", ] From bda3d201cb81dd9f05ad09c36b86dbbdb94e9903 Mon Sep 17 00:00:00 2001 From: Your Name Date: Thu, 23 Oct 2025 15:01:52 +0800 Subject: [PATCH 2/4] fix apache/druid static check --- .../airflow/providers/apache/druid/transfers/hive_to_druid.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/providers/apache/druid/src/airflow/providers/apache/druid/transfers/hive_to_druid.py b/providers/apache/druid/src/airflow/providers/apache/druid/transfers/hive_to_druid.py index 97a2c85e5968f..d78aabb9b8dc3 100644 --- a/providers/apache/druid/src/airflow/providers/apache/druid/transfers/hive_to_druid.py +++ b/providers/apache/druid/src/airflow/providers/apache/druid/transfers/hive_to_druid.py @@ -23,8 +23,8 @@ from typing import TYPE_CHECKING, Any from airflow.providers.apache.druid.hooks.druid import DruidHook -from airflow.providers.common.compat.sdk import BaseOperator from airflow.providers.apache.hive.hooks.hive import HiveCliHook, HiveMetastoreHook +from airflow.providers.common.compat.sdk import BaseOperator if TYPE_CHECKING: from airflow.providers.common.compat.sdk import Context From 09d3cbc4c167fd05991653acc5905a1522117315 Mon Sep 17 00:00:00 2001 From: Kaxil Naik Date: Sun, 26 Oct 2025 23:17:11 +0000 Subject: [PATCH 3/4] Fix tests --- providers/apache/drill/pyproject.toml | 3 +-- providers/apache/druid/pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/providers/apache/drill/pyproject.toml b/providers/apache/drill/pyproject.toml index b21dd6a675e9c..c3dbc2a066cc3 100644 --- a/providers/apache/drill/pyproject.toml +++ b/providers/apache/drill/pyproject.toml @@ -59,7 +59,7 @@ requires-python = ">=3.10" dependencies = [ "apache-airflow>=2.10.0", "apache-airflow-providers-common-sql>=1.26.0", - "apache-airflow-providers-common-compat>=1.7.4", # + TODO: bump to next version + "apache-airflow-providers-common-compat>=1.8.0", # Workaround until we get https://github.com/JohnOmernik/sqlalchemy-drill/issues/94 fixed. "sqlalchemy-drill>=1.1.0,!=1.1.6,!=1.1.7", ] @@ -70,7 +70,6 @@ dev = [ "apache-airflow-task-sdk", "apache-airflow-devel-common", "apache-airflow-providers-common-sql", - "apache-airflow-providers-common-compat", # Additional devel dependencies (do not remove this line and add extra development dependencies) "apache-airflow-providers-common-sql[pandas,polars]", ] diff --git a/providers/apache/druid/pyproject.toml b/providers/apache/druid/pyproject.toml index 25632ac90abf3..76c7ccab54c3b 100644 --- a/providers/apache/druid/pyproject.toml +++ b/providers/apache/druid/pyproject.toml @@ -76,8 +76,8 @@ dev = [ "apache-airflow-task-sdk", "apache-airflow-devel-common", "apache-airflow-providers-apache-hive", - "apache-airflow-providers-common-sql", "apache-airflow-providers-common-compat", + "apache-airflow-providers-common-sql", # Additional devel dependencies (do not remove this line and add extra development dependencies) "apache-airflow-providers-common-sql[pandas,polars]", ] From b9d5f402796eae457f15d069b584bb460cc730d7 Mon Sep 17 00:00:00 2001 From: Kaxil Naik Date: Sun, 26 Oct 2025 23:19:38 +0000 Subject: [PATCH 4/4] Fix tests --- providers/apache/druid/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/providers/apache/druid/pyproject.toml b/providers/apache/druid/pyproject.toml index 76c7ccab54c3b..e6484d5b0fd2c 100644 --- a/providers/apache/druid/pyproject.toml +++ b/providers/apache/druid/pyproject.toml @@ -59,7 +59,7 @@ requires-python = ">=3.10" dependencies = [ "apache-airflow>=2.10.0", "apache-airflow-providers-common-sql>=1.26.0", - "apache-airflow-providers-common-compat>=1.7.4", # + TODO: bump to next version + "apache-airflow-providers-common-compat>=1.8.0", "pydruid>=0.6.6", ]