From 1d00dc4e46bb8a859f3a5f9f9ed791252a662f1e Mon Sep 17 00:00:00 2001 From: dabla Date: Fri, 12 Jul 2024 16:00:39 +0200 Subject: [PATCH 01/32] refactor: Moved connection_extra_lower from OdbcHook and JdbcHook to DbApiHook and added cached connection_extra property --- airflow/providers/common/sql/hooks/sql.py | 17 +++++++++++++++-- airflow/providers/common/sql/hooks/sql.pyi | 4 ++++ airflow/providers/jdbc/hooks/jdbc.py | 11 ----------- airflow/providers/odbc/hooks/odbc.py | 11 +---------- 4 files changed, 20 insertions(+), 23 deletions(-) diff --git a/airflow/providers/common/sql/hooks/sql.py b/airflow/providers/common/sql/hooks/sql.py index 4dba2d843a38..d2d2da96909d 100644 --- a/airflow/providers/common/sql/hooks/sql.py +++ b/airflow/providers/common/sql/hooks/sql.py @@ -188,8 +188,7 @@ def get_conn_id(self) -> str: @cached_property def placeholder(self): - conn = self.get_connection(self.get_conn_id()) - placeholder = conn.extra_dejson.get("placeholder") + placeholder = self.connection_extra.get("placeholder") if placeholder: if placeholder in SQL_PLACEHOLDERS: return placeholder @@ -202,6 +201,20 @@ def placeholder(self): ) return self._placeholder + @cached_property + def connection_extra(self) -> dict: + conn = self.get_connection(self.get_conn_id()) + return conn.extra_dejson + + @cached_property + def connection_extra_lower(self) -> dict: + """ + ``connection.extra_dejson`` but where keys are converted to lower case. + + This is used internally for case-insensitive access of jdbc params. + """ + return {k.lower(): v for k, v in self.connection_extra.items()} + def get_conn(self): """Return a connection object.""" db = self.get_connection(self.get_conn_id()) diff --git a/airflow/providers/common/sql/hooks/sql.pyi b/airflow/providers/common/sql/hooks/sql.pyi index 27142aeaf2a0..dae8235a2403 100644 --- a/airflow/providers/common/sql/hooks/sql.pyi +++ b/airflow/providers/common/sql/hooks/sql.pyi @@ -68,6 +68,10 @@ class DbApiHook(BaseHook): def get_conn_id(self) -> str: ... @cached_property def placeholder(self): ... + @cached_property + def connection_extra(self) -> dict: ... + @cached_property + def connection_extra_lower(self) -> dict: ... def get_conn(self): ... def get_uri(self) -> str: ... @property diff --git a/airflow/providers/jdbc/hooks/jdbc.py b/airflow/providers/jdbc/hooks/jdbc.py index 5d89ada9d987..61d69be5c103 100644 --- a/airflow/providers/jdbc/hooks/jdbc.py +++ b/airflow/providers/jdbc/hooks/jdbc.py @@ -98,17 +98,6 @@ def get_ui_field_behaviour(cls) -> dict[str, Any]: "hidden_fields": ["port", "schema"], "relabeling": {"host": "Connection URL"}, } - - @property - def connection_extra_lower(self) -> dict: - """ - ``connection.extra_dejson`` but where keys are converted to lower case. - - This is used internally for case-insensitive access of jdbc params. - """ - conn = self.get_connection(getattr(self, self.conn_name_attr)) - return {k.lower(): v for k, v in conn.extra_dejson.items()} - @property def driver_path(self) -> str | None: from airflow.configuration import conf diff --git a/airflow/providers/odbc/hooks/odbc.py b/airflow/providers/odbc/hooks/odbc.py index 423eb8a0df3d..bc20c26d21db 100644 --- a/airflow/providers/odbc/hooks/odbc.py +++ b/airflow/providers/odbc/hooks/odbc.py @@ -99,15 +99,6 @@ def sqlalchemy_scheme(self) -> str: raise RuntimeError("sqlalchemy_scheme in connection extra should not contain : or / characters") return self._sqlalchemy_scheme or extra_scheme or self.DEFAULT_SQLALCHEMY_SCHEME - @property - def connection_extra_lower(self) -> dict: - """ - ``connection.extra_dejson`` but where keys are converted to lower case. - - This is used internally for case-insensitive access of odbc params. - """ - return {k.lower(): v for k, v in self.connection.extra_dejson.items()} - @property def driver(self) -> str | None: """Driver from init param if given; else try to find one in connection extra.""" @@ -167,7 +158,7 @@ def odbc_connection_string(self): extra_exclude = {"driver", "dsn", "connect_kwargs", "sqlalchemy_scheme", "placeholder"} extra_params = { - k: v for k, v in self.connection.extra_dejson.items() if k.lower() not in extra_exclude + k: v for k, v in self.connection_extra_lower if k not in extra_exclude } for k, v in extra_params.items(): conn_str += f"{k}={v};" From 25a122a0e5aa5102e2ec023613e43518b56c67fc Mon Sep 17 00:00:00 2001 From: David Blain Date: Fri, 12 Jul 2024 18:57:24 +0200 Subject: [PATCH 02/32] refactor: Fixed static checks --- airflow/providers/jdbc/hooks/jdbc.py | 1 + airflow/providers/odbc/hooks/odbc.py | 4 +--- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/airflow/providers/jdbc/hooks/jdbc.py b/airflow/providers/jdbc/hooks/jdbc.py index 61d69be5c103..cd1c4c4a7bcf 100644 --- a/airflow/providers/jdbc/hooks/jdbc.py +++ b/airflow/providers/jdbc/hooks/jdbc.py @@ -98,6 +98,7 @@ def get_ui_field_behaviour(cls) -> dict[str, Any]: "hidden_fields": ["port", "schema"], "relabeling": {"host": "Connection URL"}, } + @property def driver_path(self) -> str | None: from airflow.configuration import conf diff --git a/airflow/providers/odbc/hooks/odbc.py b/airflow/providers/odbc/hooks/odbc.py index bc20c26d21db..2887e4ba46a0 100644 --- a/airflow/providers/odbc/hooks/odbc.py +++ b/airflow/providers/odbc/hooks/odbc.py @@ -157,9 +157,7 @@ def odbc_connection_string(self): conn_str += f"PORT={self.connection.port};" extra_exclude = {"driver", "dsn", "connect_kwargs", "sqlalchemy_scheme", "placeholder"} - extra_params = { - k: v for k, v in self.connection_extra_lower if k not in extra_exclude - } + extra_params = {k: v for k, v in self.connection_extra_lower if k not in extra_exclude} for k, v in extra_params.items(): conn_str += f"{k}={v};" From 076dac54b482ab5efd50ac8d92a3296b48218f3d Mon Sep 17 00:00:00 2001 From: David Blain Date: Fri, 12 Jul 2024 19:08:24 +0200 Subject: [PATCH 03/32] refactor: Removed connection_extra_lower from MsSqlHook --- airflow/providers/microsoft/mssql/hooks/mssql.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/airflow/providers/microsoft/mssql/hooks/mssql.py b/airflow/providers/microsoft/mssql/hooks/mssql.py index a231590ae2ce..7732aad828d9 100644 --- a/airflow/providers/microsoft/mssql/hooks/mssql.py +++ b/airflow/providers/microsoft/mssql/hooks/mssql.py @@ -67,15 +67,6 @@ def connection(self) -> Connection: """ return self.get_connection(getattr(self, self.conn_name_attr)) - @property - def connection_extra_lower(self) -> dict: - """ - ``connection.extra_dejson`` but where keys are converted to lower case. - - This is used internally for case-insensitive access of mssql params. - """ - return {k.lower(): v for k, v in self.connection.extra_dejson.items()} - @property def sqlalchemy_scheme(self) -> str: """Sqlalchemy scheme either from constructor, connection extras or default.""" From 5748c70bd9630134de8bd16db318c6b4e9b1f4ec Mon Sep 17 00:00:00 2001 From: David Blain Date: Fri, 12 Jul 2024 19:18:06 +0200 Subject: [PATCH 04/32] refactor: Added cached connection property in DbApiHook --- airflow/providers/common/sql/hooks/sql.py | 15 ++++++++++++--- airflow/providers/common/sql/hooks/sql.pyi | 3 +++ airflow/providers/microsoft/mssql/hooks/mssql.py | 15 +-------------- airflow/providers/odbc/hooks/odbc.py | 7 ------- 4 files changed, 16 insertions(+), 24 deletions(-) diff --git a/airflow/providers/common/sql/hooks/sql.py b/airflow/providers/common/sql/hooks/sql.py index d2d2da96909d..73b53d9d98c5 100644 --- a/airflow/providers/common/sql/hooks/sql.py +++ b/airflow/providers/common/sql/hooks/sql.py @@ -52,6 +52,7 @@ from pandas import DataFrame from sqlalchemy.engine import URL + from airflow.models import Connection from airflow.providers.openlineage.extractors import OperatorLineage from airflow.providers.openlineage.sqlparser import DatabaseInfo @@ -202,9 +203,17 @@ def placeholder(self): return self._placeholder @cached_property + def connection(self) -> Connection: + """ + Get the airflow connection object. + + :return: The connection object. + """ + return self.get_connection(self.get_conn_id()) + + @property def connection_extra(self) -> dict: - conn = self.get_connection(self.get_conn_id()) - return conn.extra_dejson + return self.connection.extra_dejson @cached_property def connection_extra_lower(self) -> dict: @@ -217,7 +226,7 @@ def connection_extra_lower(self) -> dict: def get_conn(self): """Return a connection object.""" - db = self.get_connection(self.get_conn_id()) + db = self.connection return self.connector.connect(host=db.host, port=db.port, username=db.login, schema=db.schema) def get_uri(self) -> str: diff --git a/airflow/providers/common/sql/hooks/sql.pyi b/airflow/providers/common/sql/hooks/sql.pyi index dae8235a2403..e4cb8cae01e5 100644 --- a/airflow/providers/common/sql/hooks/sql.pyi +++ b/airflow/providers/common/sql/hooks/sql.pyi @@ -38,6 +38,7 @@ from airflow.exceptions import ( AirflowProviderDeprecationWarning as AirflowProviderDeprecationWarning, ) from airflow.hooks.base import BaseHook as BaseHook +from airflow.models import Connection from airflow.providers.openlineage.extractors import OperatorLineage as OperatorLineage from airflow.providers.openlineage.sqlparser import DatabaseInfo as DatabaseInfo from functools import cached_property as cached_property @@ -69,6 +70,8 @@ class DbApiHook(BaseHook): @cached_property def placeholder(self): ... @cached_property + def connection(self) -> Connection: ... + @property def connection_extra(self) -> dict: ... @cached_property def connection_extra_lower(self) -> dict: ... diff --git a/airflow/providers/microsoft/mssql/hooks/mssql.py b/airflow/providers/microsoft/mssql/hooks/mssql.py index 7732aad828d9..6ef64c745855 100644 --- a/airflow/providers/microsoft/mssql/hooks/mssql.py +++ b/airflow/providers/microsoft/mssql/hooks/mssql.py @@ -19,17 +19,13 @@ from __future__ import annotations -from functools import cached_property -from typing import TYPE_CHECKING, Any +from typing import Any import pymssql from pymssql import Connection as PymssqlConnection from airflow.providers.common.sql.hooks.sql import DbApiHook -if TYPE_CHECKING: - from airflow.models import Connection - class MsSqlHook(DbApiHook): """ @@ -58,15 +54,6 @@ def __init__( self.schema = kwargs.pop("schema", None) self._sqlalchemy_scheme = sqlalchemy_scheme - @cached_property - def connection(self) -> Connection: - """ - Get the airflow connection object. - - :return: The connection object. - """ - return self.get_connection(getattr(self, self.conn_name_attr)) - @property def sqlalchemy_scheme(self) -> str: """Sqlalchemy scheme either from constructor, connection extras or default.""" diff --git a/airflow/providers/odbc/hooks/odbc.py b/airflow/providers/odbc/hooks/odbc.py index 2887e4ba46a0..cf1e6349c51d 100644 --- a/airflow/providers/odbc/hooks/odbc.py +++ b/airflow/providers/odbc/hooks/odbc.py @@ -79,13 +79,6 @@ def __init__( self._connection = None self._connect_kwargs = connect_kwargs - @property - def connection(self): - """The Connection object with ID ``odbc_conn_id``.""" - if not self._connection: - self._connection = self.get_connection(getattr(self, self.conn_name_attr)) - return self._connection - @property def database(self) -> str | None: """Database provided in init if exists; otherwise, ``schema`` from ``Connection`` object.""" From 8a7d7520da6cfda02b094eea06f015dbc16e1967 Mon Sep 17 00:00:00 2001 From: David Blain Date: Fri, 12 Jul 2024 19:20:42 +0200 Subject: [PATCH 05/32] refactor: Fixed odbc_connection_string in OdbcHook --- airflow/providers/odbc/hooks/odbc.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/airflow/providers/odbc/hooks/odbc.py b/airflow/providers/odbc/hooks/odbc.py index cf1e6349c51d..548324e865f9 100644 --- a/airflow/providers/odbc/hooks/odbc.py +++ b/airflow/providers/odbc/hooks/odbc.py @@ -150,7 +150,9 @@ def odbc_connection_string(self): conn_str += f"PORT={self.connection.port};" extra_exclude = {"driver", "dsn", "connect_kwargs", "sqlalchemy_scheme", "placeholder"} - extra_params = {k: v for k, v in self.connection_extra_lower if k not in extra_exclude} + extra_params = { + k: v for k, v in self.connection_extra.items() if k.lower() not in extra_exclude + } for k, v in extra_params.items(): conn_str += f"{k}={v};" From 93ef22b7e55e4e7d4f71d5a3689543d4ececfe21 Mon Sep 17 00:00:00 2001 From: David Blain Date: Fri, 12 Jul 2024 19:37:27 +0200 Subject: [PATCH 06/32] refactor: Fixed static check --- airflow/providers/odbc/hooks/odbc.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/airflow/providers/odbc/hooks/odbc.py b/airflow/providers/odbc/hooks/odbc.py index 548324e865f9..48dada49f88b 100644 --- a/airflow/providers/odbc/hooks/odbc.py +++ b/airflow/providers/odbc/hooks/odbc.py @@ -150,9 +150,7 @@ def odbc_connection_string(self): conn_str += f"PORT={self.connection.port};" extra_exclude = {"driver", "dsn", "connect_kwargs", "sqlalchemy_scheme", "placeholder"} - extra_params = { - k: v for k, v in self.connection_extra.items() if k.lower() not in extra_exclude - } + extra_params = {k: v for k, v in self.connection_extra.items() if k.lower() not in extra_exclude} for k, v in extra_params.items(): conn_str += f"{k}={v};" From f5bab94e206b434b6053abb59a39fa16fbfa28ea Mon Sep 17 00:00:00 2001 From: David Blain Date: Fri, 12 Jul 2024 21:58:22 +0200 Subject: [PATCH 07/32] refactor: Fixed import of Connection --- airflow/providers/common/sql/hooks/sql.pyi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airflow/providers/common/sql/hooks/sql.pyi b/airflow/providers/common/sql/hooks/sql.pyi index e4cb8cae01e5..7e8774a2f002 100644 --- a/airflow/providers/common/sql/hooks/sql.pyi +++ b/airflow/providers/common/sql/hooks/sql.pyi @@ -38,7 +38,7 @@ from airflow.exceptions import ( AirflowProviderDeprecationWarning as AirflowProviderDeprecationWarning, ) from airflow.hooks.base import BaseHook as BaseHook -from airflow.models import Connection +from airflow.models import Connection as Connection from airflow.providers.openlineage.extractors import OperatorLineage as OperatorLineage from airflow.providers.openlineage.sqlparser import DatabaseInfo as DatabaseInfo from functools import cached_property as cached_property From d99353f41371acbebfd663dd479fcf352179d08c Mon Sep 17 00:00:00 2001 From: David Blain Date: Sat, 13 Jul 2024 15:12:19 +0200 Subject: [PATCH 08/32] refactor: Refactored connection on PostgresHook --- airflow/providers/postgres/hooks/postgres.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/airflow/providers/postgres/hooks/postgres.py b/airflow/providers/postgres/hooks/postgres.py index 4b7074ec75df..d9150475b647 100644 --- a/airflow/providers/postgres/hooks/postgres.py +++ b/airflow/providers/postgres/hooks/postgres.py @@ -88,7 +88,7 @@ def __init__(self, *args, options: str | None = None, **kwargs) -> None: ) kwargs["database"] = kwargs["schema"] super().__init__(*args, **kwargs) - self.connection: Connection | None = kwargs.pop("connection", None) + self._connection: Connection | None = kwargs.pop("connection", None) self.conn: connection = None self.database: str | None = kwargs.pop("database", None) self.options = options @@ -140,10 +140,13 @@ def _get_cursor(self, raw_cursor: str) -> CursorType: valid_cursors = ", ".join(cursor_types.keys()) raise ValueError(f"Invalid cursor passed {_cursor}. Valid options are: {valid_cursors}") + @property + def connection(self) -> Connection: + return deepcopy(self._connection or self.get_connection(self.get_conn_id())) + def get_conn(self) -> connection: """Establish a connection to a postgres database.""" - conn_id = getattr(self, self.conn_name_attr) - conn = deepcopy(self.connection or self.get_connection(conn_id)) + conn = self.connection # check for authentication via AWS IAM if conn.extra_dejson.get("iam", False): From dd6c74f7625abe262a05173a981f2dc9472720fe Mon Sep 17 00:00:00 2001 From: David Blain Date: Sat, 13 Jul 2024 15:27:36 +0200 Subject: [PATCH 09/32] refactor: Refactored Connection property so logic is generic for PostgresHook and ElasticSearchHook and added _placeholder and _connection property in sql.pyi --- airflow/providers/common/sql/hooks/sql.py | 13 ++++++------- airflow/providers/common/sql/hooks/sql.pyi | 19 +++++++++---------- .../elasticsearch/hooks/elasticsearch.py | 10 ++++------ airflow/providers/postgres/hooks/postgres.py | 6 ------ 4 files changed, 19 insertions(+), 29 deletions(-) diff --git a/airflow/providers/common/sql/hooks/sql.py b/airflow/providers/common/sql/hooks/sql.py index 73b53d9d98c5..fa3a8db2fcb6 100644 --- a/airflow/providers/common/sql/hooks/sql.py +++ b/airflow/providers/common/sql/hooks/sql.py @@ -19,6 +19,7 @@ import contextlib import warnings from contextlib import closing, contextmanager +from copy import deepcopy from datetime import datetime from functools import cached_property from typing import ( @@ -183,6 +184,7 @@ def __init__(self, *args, schema: str | None = None, log_sql: bool = True, **kwa self._replace_statement_format: str = kwargs.get( "replace_statement_format", "REPLACE INTO {} {} VALUES ({})" ) + self._connection: Connection | None = kwargs.pop("connection", None) def get_conn_id(self) -> str: return getattr(self, self.conn_name_attr) @@ -202,14 +204,11 @@ def placeholder(self): ) return self._placeholder - @cached_property + @property def connection(self) -> Connection: - """ - Get the airflow connection object. - - :return: The connection object. - """ - return self.get_connection(self.get_conn_id()) + if self._connection is None: + self._connection = self.get_connection(self.get_conn_id()) + return deepcopy(self._connection) @property def connection_extra(self) -> dict: diff --git a/airflow/providers/common/sql/hooks/sql.pyi b/airflow/providers/common/sql/hooks/sql.pyi index 7e8774a2f002..9abc98e4af8a 100644 --- a/airflow/providers/common/sql/hooks/sql.pyi +++ b/airflow/providers/common/sql/hooks/sql.pyi @@ -32,19 +32,16 @@ Definition of the public interface for airflow.providers.common.sql.hooks.sql isort:skip_file """ from _typeshed import Incomplete -from airflow.exceptions import ( - AirflowException as AirflowException, - AirflowOptionalProviderFeatureException as AirflowOptionalProviderFeatureException, - AirflowProviderDeprecationWarning as AirflowProviderDeprecationWarning, -) +from functools import cached_property as cached_property +from typing import Any, Callable, Generator, Iterable, Mapping, Protocol, Sequence, TypeVar, overload + +from pandas import DataFrame as DataFrame +from sqlalchemy.engine import URL as URL + from airflow.hooks.base import BaseHook as BaseHook from airflow.models import Connection as Connection from airflow.providers.openlineage.extractors import OperatorLineage as OperatorLineage from airflow.providers.openlineage.sqlparser import DatabaseInfo as DatabaseInfo -from functools import cached_property as cached_property -from pandas import DataFrame as DataFrame -from sqlalchemy.engine import URL as URL -from typing import Any, Callable, Generator, Iterable, Mapping, Protocol, Sequence, TypeVar, overload T = TypeVar("T") SQL_PLACEHOLDERS: Incomplete @@ -64,12 +61,14 @@ class DbApiHook(BaseHook): connector: ConnectorProtocol | None log_sql: Incomplete descriptions: Incomplete + _placeholder: str + _connection: Connection | None def __init__(self, *args, schema: str | None = None, log_sql: bool = True, **kwargs) -> None: ... def get_conn_id(self) -> str: ... @cached_property def placeholder(self): ... - @cached_property + @property def connection(self) -> Connection: ... @property def connection_extra(self) -> dict: ... diff --git a/airflow/providers/elasticsearch/hooks/elasticsearch.py b/airflow/providers/elasticsearch/hooks/elasticsearch.py index 75230b1f6db5..6a9aed38717c 100644 --- a/airflow/providers/elasticsearch/hooks/elasticsearch.py +++ b/airflow/providers/elasticsearch/hooks/elasticsearch.py @@ -22,11 +22,11 @@ from urllib import parse from deprecated import deprecated -from elasticsearch import Elasticsearch from airflow.exceptions import AirflowProviderDeprecationWarning from airflow.hooks.base import BaseHook from airflow.providers.common.sql.hooks.sql import DbApiHook +from elasticsearch import Elasticsearch if TYPE_CHECKING: from airflow.models.connection import Connection as AirflowConnection @@ -87,12 +87,11 @@ class ElasticsearchSQLHook(DbApiHook): def __init__(self, schema: str = "http", connection: AirflowConnection | None = None, *args, **kwargs): super().__init__(*args, **kwargs) self.schema = schema - self.connection = connection + self._connection = connection def get_conn(self) -> ESConnection: """Return an elasticsearch connection object.""" - conn_id = getattr(self, self.conn_name_attr) - conn = self.connection or self.get_connection(conn_id) + conn = self.connection conn_args = { "host": conn.host, @@ -111,8 +110,7 @@ def get_conn(self) -> ESConnection: return connect(**conn_args) def get_uri(self) -> str: - conn_id = getattr(self, self.conn_name_attr) - conn = self.connection or self.get_connection(conn_id) + conn = self.connection login = "" if conn.login: diff --git a/airflow/providers/postgres/hooks/postgres.py b/airflow/providers/postgres/hooks/postgres.py index d9150475b647..cf50eb245843 100644 --- a/airflow/providers/postgres/hooks/postgres.py +++ b/airflow/providers/postgres/hooks/postgres.py @@ -20,7 +20,6 @@ import os import warnings from contextlib import closing -from copy import deepcopy from typing import TYPE_CHECKING, Any, Iterable, Union import psycopg2 @@ -88,7 +87,6 @@ def __init__(self, *args, options: str | None = None, **kwargs) -> None: ) kwargs["database"] = kwargs["schema"] super().__init__(*args, **kwargs) - self._connection: Connection | None = kwargs.pop("connection", None) self.conn: connection = None self.database: str | None = kwargs.pop("database", None) self.options = options @@ -140,10 +138,6 @@ def _get_cursor(self, raw_cursor: str) -> CursorType: valid_cursors = ", ".join(cursor_types.keys()) raise ValueError(f"Invalid cursor passed {_cursor}. Valid options are: {valid_cursors}") - @property - def connection(self) -> Connection: - return deepcopy(self._connection or self.get_connection(self.get_conn_id())) - def get_conn(self) -> connection: """Establish a connection to a postgres database.""" conn = self.connection From 3de038c471159bbe8b50e60e925038ac87a81930 Mon Sep 17 00:00:00 2001 From: David Blain Date: Sat, 13 Jul 2024 15:43:07 +0200 Subject: [PATCH 10/32] refactor: Removed assignment of connection in MySqlHook --- airflow/providers/mysql/hooks/mysql.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/airflow/providers/mysql/hooks/mysql.py b/airflow/providers/mysql/hooks/mysql.py index ebf88c6557e2..59de5ca1fe7c 100644 --- a/airflow/providers/mysql/hooks/mysql.py +++ b/airflow/providers/mysql/hooks/mysql.py @@ -35,7 +35,6 @@ from mysql.connector.abstracts import MySQLConnectionAbstract except ModuleNotFoundError: logger.warning("The package 'mysql-connector-python' is not installed. Import skipped") - from MySQLdb.connections import Connection as MySQLdbConnection MySQLConnectionTypes = Union["MySQLdbConnection", "MySQLConnectionAbstract"] @@ -72,7 +71,6 @@ class MySqlHook(DbApiHook): def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) self.schema = kwargs.pop("schema", None) - self.connection = kwargs.pop("connection", None) self.local_infile = kwargs.pop("local_infile", False) self.init_command = kwargs.pop("init_command", None) From 3a69726da3d58b85455f1b6694e00bf5ec37aa66 Mon Sep 17 00:00:00 2001 From: David Blain Date: Sat, 13 Jul 2024 16:21:21 +0200 Subject: [PATCH 11/32] refactor: Fixed import elasticsearch --- airflow/providers/elasticsearch/hooks/elasticsearch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airflow/providers/elasticsearch/hooks/elasticsearch.py b/airflow/providers/elasticsearch/hooks/elasticsearch.py index 6a9aed38717c..9800da251c82 100644 --- a/airflow/providers/elasticsearch/hooks/elasticsearch.py +++ b/airflow/providers/elasticsearch/hooks/elasticsearch.py @@ -22,11 +22,11 @@ from urllib import parse from deprecated import deprecated +from elasticsearch import Elasticsearch from airflow.exceptions import AirflowProviderDeprecationWarning from airflow.hooks.base import BaseHook from airflow.providers.common.sql.hooks.sql import DbApiHook -from elasticsearch import Elasticsearch if TYPE_CHECKING: from airflow.models.connection import Connection as AirflowConnection From 9700377cca6df1d068b7d8e1fe079ddcc1eac207 Mon Sep 17 00:00:00 2001 From: David Blain Date: Sat, 13 Jul 2024 16:22:34 +0200 Subject: [PATCH 12/32] refactor: Removed _placeholder and _connection from sql.pyi --- airflow/providers/common/sql/hooks/sql.pyi | 3 --- 1 file changed, 3 deletions(-) diff --git a/airflow/providers/common/sql/hooks/sql.pyi b/airflow/providers/common/sql/hooks/sql.pyi index 9abc98e4af8a..580c60b4bffa 100644 --- a/airflow/providers/common/sql/hooks/sql.pyi +++ b/airflow/providers/common/sql/hooks/sql.pyi @@ -61,10 +61,7 @@ class DbApiHook(BaseHook): connector: ConnectorProtocol | None log_sql: Incomplete descriptions: Incomplete - _placeholder: str - _connection: Connection | None def __init__(self, *args, schema: str | None = None, log_sql: bool = True, **kwargs) -> None: ... - def get_conn_id(self) -> str: ... @cached_property def placeholder(self): ... From 3e255e2d4451a89fbcb0c69b113cc190567fbc02 Mon Sep 17 00:00:00 2001 From: David Blain Date: Sat, 13 Jul 2024 16:24:13 +0200 Subject: [PATCH 13/32] refactor: Reorganized imports in sql.pyi --- airflow/providers/common/sql/hooks/sql.pyi | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/airflow/providers/common/sql/hooks/sql.pyi b/airflow/providers/common/sql/hooks/sql.pyi index 580c60b4bffa..26cbb687810e 100644 --- a/airflow/providers/common/sql/hooks/sql.pyi +++ b/airflow/providers/common/sql/hooks/sql.pyi @@ -32,16 +32,19 @@ Definition of the public interface for airflow.providers.common.sql.hooks.sql isort:skip_file """ from _typeshed import Incomplete -from functools import cached_property as cached_property -from typing import Any, Callable, Generator, Iterable, Mapping, Protocol, Sequence, TypeVar, overload - -from pandas import DataFrame as DataFrame -from sqlalchemy.engine import URL as URL - +from airflow.exceptions import ( + AirflowException as AirflowException, + AirflowOptionalProviderFeatureException as AirflowOptionalProviderFeatureException, + AirflowProviderDeprecationWarning as AirflowProviderDeprecationWarning, +) from airflow.hooks.base import BaseHook as BaseHook from airflow.models import Connection as Connection from airflow.providers.openlineage.extractors import OperatorLineage as OperatorLineage from airflow.providers.openlineage.sqlparser import DatabaseInfo as DatabaseInfo +from functools import cached_property as cached_property +from pandas import DataFrame as DataFrame +from sqlalchemy.engine import URL as URL +from typing import Any, Callable, Generator, Iterable, Mapping, Protocol, Sequence, TypeVar, overload T = TypeVar("T") SQL_PLACEHOLDERS: Incomplete From 392d66d60e4bafadb8063c0c1dbcd3fb79b72663 Mon Sep 17 00:00:00 2001 From: David Blain Date: Mon, 15 Jul 2024 08:31:08 +0200 Subject: [PATCH 14/32] refactor: Re-added import of MySQLdbConnection in MySQLHook --- airflow/providers/mysql/hooks/mysql.py | 1 + 1 file changed, 1 insertion(+) diff --git a/airflow/providers/mysql/hooks/mysql.py b/airflow/providers/mysql/hooks/mysql.py index 59de5ca1fe7c..afd9e2f15a3e 100644 --- a/airflow/providers/mysql/hooks/mysql.py +++ b/airflow/providers/mysql/hooks/mysql.py @@ -35,6 +35,7 @@ from mysql.connector.abstracts import MySQLConnectionAbstract except ModuleNotFoundError: logger.warning("The package 'mysql-connector-python' is not installed. Import skipped") + from MySQLdb.connections import Connection as MySQLdbConnection MySQLConnectionTypes = Union["MySQLdbConnection", "MySQLConnectionAbstract"] From 01c5710e1b94bba60eb56471382b819c8f04741c Mon Sep 17 00:00:00 2001 From: David Blain Date: Mon, 15 Jul 2024 20:18:35 +0200 Subject: [PATCH 15/32] refactor: Try to fix AttributeError: type object 'SkipDBTestsSession' has no attribute 'get_bind' --- airflow/settings.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/airflow/settings.py b/airflow/settings.py index dbe68bcb1356..16bb66bd44b4 100644 --- a/airflow/settings.py +++ b/airflow/settings.py @@ -256,6 +256,16 @@ def __init__(self): def remove(*args, **kwargs): pass + def get_bind( + self, + mapper=None, + clause=None, + bind=None, + _sa_skip_events=None, + _sa_skip_for_implicit_returning=False, + ): + pass + class TracebackSession: """ From 63a8e752f25a7ea6176d8a74b630f0265ed3c5c7 Mon Sep 17 00:00:00 2001 From: David Blain Date: Tue, 30 Jul 2024 08:25:17 +0200 Subject: [PATCH 16/32] refactor: Updated dependency to common sql 1.14.2 --- airflow/providers/elasticsearch/provider.yaml | 2 +- airflow/providers/jdbc/provider.yaml | 2 +- airflow/providers/microsoft/mssql/provider.yaml | 2 +- airflow/providers/odbc/provider.yaml | 2 +- airflow/providers/postgres/provider.yaml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/airflow/providers/elasticsearch/provider.yaml b/airflow/providers/elasticsearch/provider.yaml index a7c2d52fecba..2b19c50463bb 100644 --- a/airflow/providers/elasticsearch/provider.yaml +++ b/airflow/providers/elasticsearch/provider.yaml @@ -66,7 +66,7 @@ versions: dependencies: - apache-airflow>=2.7.0 - - apache-airflow-providers-common-sql>=1.14.1 + - apache-airflow-providers-common-sql>=1.14.2 - elasticsearch>=8.10,<9 integrations: diff --git a/airflow/providers/jdbc/provider.yaml b/airflow/providers/jdbc/provider.yaml index bf0b74608c20..4efbfd22bdfe 100644 --- a/airflow/providers/jdbc/provider.yaml +++ b/airflow/providers/jdbc/provider.yaml @@ -51,7 +51,7 @@ versions: dependencies: - apache-airflow>=2.7.0 - - apache-airflow-providers-common-sql>=1.14.1 + - apache-airflow-providers-common-sql>=1.14.2 - jaydebeapi>=1.1.1 integrations: diff --git a/airflow/providers/microsoft/mssql/provider.yaml b/airflow/providers/microsoft/mssql/provider.yaml index b8f2550cfca2..4c6c99627013 100644 --- a/airflow/providers/microsoft/mssql/provider.yaml +++ b/airflow/providers/microsoft/mssql/provider.yaml @@ -53,7 +53,7 @@ versions: dependencies: - apache-airflow>=2.7.0 - - apache-airflow-providers-common-sql>=1.14.1 + - apache-airflow-providers-common-sql>=1.14.2 - pymssql>=2.3.0 integrations: diff --git a/airflow/providers/odbc/provider.yaml b/airflow/providers/odbc/provider.yaml index 708c5db903e7..618a87aad4a7 100644 --- a/airflow/providers/odbc/provider.yaml +++ b/airflow/providers/odbc/provider.yaml @@ -52,7 +52,7 @@ versions: dependencies: - apache-airflow>=2.7.0 - - apache-airflow-providers-common-sql>=1.14.1 + - apache-airflow-providers-common-sql>=1.14.2 - pyodbc>=5.0.0 integrations: diff --git a/airflow/providers/postgres/provider.yaml b/airflow/providers/postgres/provider.yaml index 1e976fb42f14..dbc3096ed33d 100644 --- a/airflow/providers/postgres/provider.yaml +++ b/airflow/providers/postgres/provider.yaml @@ -63,7 +63,7 @@ versions: dependencies: - apache-airflow>=2.7.0 - - apache-airflow-providers-common-sql>=1.14.1 + - apache-airflow-providers-common-sql>=1.14.2 - psycopg2-binary>=2.9.4 additional-extras: From 10246c8aeff1035b5b6259d7317a66277996b512 Mon Sep 17 00:00:00 2001 From: dabla Date: Tue, 30 Jul 2024 14:23:37 +0200 Subject: [PATCH 17/32] refactor: Updated provider dependencies --- generated/provider_dependencies.json | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/generated/provider_dependencies.json b/generated/provider_dependencies.json index 85dbd405e8f8..d89eb0b32695 100644 --- a/generated/provider_dependencies.json +++ b/generated/provider_dependencies.json @@ -519,7 +519,7 @@ }, "elasticsearch": { "deps": [ - "apache-airflow-providers-common-sql>=1.14.1", + "apache-airflow-providers-common-sql>=1.14.2", "apache-airflow>=2.7.0", "elasticsearch>=8.10,<9" ], @@ -754,7 +754,7 @@ }, "jdbc": { "deps": [ - "apache-airflow-providers-common-sql>=1.14.1", + "apache-airflow-providers-common-sql>=1.14.2", "apache-airflow>=2.7.0", "jaydebeapi>=1.1.1" ], @@ -818,7 +818,7 @@ }, "microsoft.mssql": { "deps": [ - "apache-airflow-providers-common-sql>=1.14.1", + "apache-airflow-providers-common-sql>=1.14.2", "apache-airflow>=2.7.0", "pymssql>=2.3.0" ], @@ -899,7 +899,7 @@ }, "odbc": { "deps": [ - "apache-airflow-providers-common-sql>=1.14.1", + "apache-airflow-providers-common-sql>=1.14.2", "apache-airflow>=2.7.0", "pyodbc>=5.0.0" ], @@ -1045,7 +1045,7 @@ }, "postgres": { "deps": [ - "apache-airflow-providers-common-sql>=1.14.1", + "apache-airflow-providers-common-sql>=1.14.2", "apache-airflow>=2.7.0", "psycopg2-binary>=2.9.4" ], From 3cf60173f1c6ff7f21f71844cbacdedb1085be51 Mon Sep 17 00:00:00 2001 From: David Blain Date: Wed, 31 Jul 2024 08:32:42 +0200 Subject: [PATCH 18/32] refactor: Fixed assertion of common sql pacakge version testing --- dev/breeze/tests/test_packages.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/breeze/tests/test_packages.py b/dev/breeze/tests/test_packages.py index 228a1ca0dc5e..7e7269f66e09 100644 --- a/dev/breeze/tests/test_packages.py +++ b/dev/breeze/tests/test_packages.py @@ -204,7 +204,7 @@ def test_get_documentation_package_path(): "postgres", "beta0", """ - "apache-airflow-providers-common-sql>=1.14.1b0", + "apache-airflow-providers-common-sql>=1.14.2b0", "apache-airflow>=2.7.0b0", "psycopg2-binary>=2.9.4", """, @@ -214,7 +214,7 @@ def test_get_documentation_package_path(): "postgres", "", """ - "apache-airflow-providers-common-sql>=1.14.1", + "apache-airflow-providers-common-sql>=1.14.2", "apache-airflow>=2.7.0", "psycopg2-binary>=2.9.4", """, From 0ae65407db009dedcb71e757684a2cd5194f6e4c Mon Sep 17 00:00:00 2001 From: David Blain Date: Sun, 4 Aug 2024 10:20:43 +0200 Subject: [PATCH 19/32] refactor: Updated sql common version in providers --- airflow/providers/elasticsearch/provider.yaml | 2 +- airflow/providers/jdbc/provider.yaml | 2 +- .../providers/microsoft/mssql/provider.yaml | 2 +- airflow/providers/mysql/provider.yaml | 2 +- airflow/providers/odbc/provider.yaml | 2 +- airflow/providers/oracle/hooks/oracle.py | 26 +++++++++---------- airflow/providers/oracle/provider.yaml | 2 +- airflow/providers/postgres/provider.yaml | 2 +- dev/breeze/tests/test_packages.py | 4 +-- 9 files changed, 22 insertions(+), 22 deletions(-) diff --git a/airflow/providers/elasticsearch/provider.yaml b/airflow/providers/elasticsearch/provider.yaml index 6cd3d16cceda..2d5946dd7823 100644 --- a/airflow/providers/elasticsearch/provider.yaml +++ b/airflow/providers/elasticsearch/provider.yaml @@ -67,7 +67,7 @@ versions: dependencies: - apache-airflow>=2.7.0 - - apache-airflow-providers-common-sql>=1.14.2 + - apache-airflow-providers-common-sql>=1.15.0 - elasticsearch>=8.10,<9 integrations: diff --git a/airflow/providers/jdbc/provider.yaml b/airflow/providers/jdbc/provider.yaml index d43f5312bf7e..623289b5c11a 100644 --- a/airflow/providers/jdbc/provider.yaml +++ b/airflow/providers/jdbc/provider.yaml @@ -52,7 +52,7 @@ versions: dependencies: - apache-airflow>=2.7.0 - - apache-airflow-providers-common-sql>=1.14.2 + - apache-airflow-providers-common-sql>=1.15.0 - jaydebeapi>=1.1.1 integrations: diff --git a/airflow/providers/microsoft/mssql/provider.yaml b/airflow/providers/microsoft/mssql/provider.yaml index 76408680e500..e301ca151f45 100644 --- a/airflow/providers/microsoft/mssql/provider.yaml +++ b/airflow/providers/microsoft/mssql/provider.yaml @@ -54,7 +54,7 @@ versions: dependencies: - apache-airflow>=2.7.0 - - apache-airflow-providers-common-sql>=1.14.2 + - apache-airflow-providers-common-sql>=1.15.0 - pymssql>=2.3.0 integrations: diff --git a/airflow/providers/mysql/provider.yaml b/airflow/providers/mysql/provider.yaml index ffb7909e75f3..79c23c980030 100644 --- a/airflow/providers/mysql/provider.yaml +++ b/airflow/providers/mysql/provider.yaml @@ -65,7 +65,7 @@ versions: dependencies: - apache-airflow>=2.7.0 - - apache-airflow-providers-common-sql>=1.14.1 + - apache-airflow-providers-common-sql>=1.15.0 - mysqlclient>=1.4.0 - mysql-connector-python>=8.0.29 diff --git a/airflow/providers/odbc/provider.yaml b/airflow/providers/odbc/provider.yaml index 7885cfc6e022..0a2719897e3d 100644 --- a/airflow/providers/odbc/provider.yaml +++ b/airflow/providers/odbc/provider.yaml @@ -53,7 +53,7 @@ versions: dependencies: - apache-airflow>=2.7.0 - - apache-airflow-providers-common-sql>=1.14.2 + - apache-airflow-providers-common-sql>=1.15.0 - pyodbc>=5.0.0 integrations: diff --git a/airflow/providers/oracle/hooks/oracle.py b/airflow/providers/oracle/hooks/oracle.py index a252a7599cd3..f6a45679546b 100644 --- a/airflow/providers/oracle/hooks/oracle.py +++ b/airflow/providers/oracle/hooks/oracle.py @@ -145,24 +145,24 @@ def get_conn(self) -> oracledb.Connection: """ conn = self.get_connection(self.oracle_conn_id) # type: ignore[attr-defined] conn_config = {"user": conn.login, "password": conn.password} - sid = conn.extra_dejson.get("sid") - mod = conn.extra_dejson.get("module") + sid = self.connection_extra_lower.get("sid") + mod = self.connection_extra_lower.get("module") schema = conn.schema # Enable oracledb thick mode if thick_mode is set to True # Parameters take precedence over connection config extra # Defaults to use thin mode if not provided in params or connection config extra - thick_mode = _get_first_bool(self.thick_mode, conn.extra_dejson.get("thick_mode")) + thick_mode = _get_first_bool(self.thick_mode, self.connection_extra_lower.get("thick_mode")) if thick_mode is True: if self.thick_mode_lib_dir is None: - self.thick_mode_lib_dir = conn.extra_dejson.get("thick_mode_lib_dir") + self.thick_mode_lib_dir = self.connection_extra_lower.get("thick_mode_lib_dir") if not isinstance(self.thick_mode_lib_dir, (str, type(None))): raise TypeError( f"thick_mode_lib_dir expected str or None, " f"got {type(self.thick_mode_lib_dir).__name__}" ) if self.thick_mode_config_dir is None: - self.thick_mode_config_dir = conn.extra_dejson.get("thick_mode_config_dir") + self.thick_mode_config_dir = self.connection_extra_lower.get("thick_mode_config_dir") if not isinstance(self.thick_mode_config_dir, (str, type(None))): raise TypeError( f"thick_mode_config_dir expected str or None, " @@ -174,23 +174,23 @@ def get_conn(self) -> oracledb.Connection: # Set oracledb Defaults Attributes if provided # (https://python-oracledb.readthedocs.io/en/latest/api_manual/defaults.html) - fetch_decimals = _get_first_bool(self.fetch_decimals, conn.extra_dejson.get("fetch_decimals")) + fetch_decimals = _get_first_bool(self.fetch_decimals, self.connection_extra_lower.get("fetch_decimals")) if isinstance(fetch_decimals, bool): oracledb.defaults.fetch_decimals = fetch_decimals - fetch_lobs = _get_first_bool(self.fetch_lobs, conn.extra_dejson.get("fetch_lobs")) + fetch_lobs = _get_first_bool(self.fetch_lobs, self.connection_extra_lower.get("fetch_lobs")) if isinstance(fetch_lobs, bool): oracledb.defaults.fetch_lobs = fetch_lobs # Set up DSN - service_name = conn.extra_dejson.get("service_name") + service_name = self.connection_extra_lower.get("service_name") port = conn.port if conn.port else 1521 if conn.host and sid and not service_name: conn_config["dsn"] = oracledb.makedsn(conn.host, port, sid) elif conn.host and service_name and not sid: conn_config["dsn"] = oracledb.makedsn(conn.host, port, service_name=service_name) else: - dsn = conn.extra_dejson.get("dsn") + dsn = self.connection_extra_lower.get("dsn") if dsn is None: dsn = conn.host if conn.port is not None: @@ -207,10 +207,10 @@ def get_conn(self) -> oracledb.Connection: dsn += f"/{conn.schema}" conn_config["dsn"] = dsn - if "events" in conn.extra_dejson: - conn_config["events"] = conn.extra_dejson.get("events") + if "events" in self.connection_extra_lower: + conn_config["events"] = self.connection_extra_lower.get("events") - mode = conn.extra_dejson.get("mode", "").lower() + mode = self.connection_extra_lower.get("mode", "").lower() if mode == "sysdba": conn_config["mode"] = oracledb.AUTH_MODE_SYSDBA elif mode == "sysasm": @@ -226,7 +226,7 @@ def get_conn(self) -> oracledb.Connection: elif mode == "sysrac": conn_config["mode"] = oracledb.AUTH_MODE_SYSRAC - purity = conn.extra_dejson.get("purity", "").lower() + purity = self.connection_extra_lower.get("purity", "").lower() if purity == "new": conn_config["purity"] = oracledb.PURITY_NEW elif purity == "self": diff --git a/airflow/providers/oracle/provider.yaml b/airflow/providers/oracle/provider.yaml index d2a702d34821..9ac78384f183 100644 --- a/airflow/providers/oracle/provider.yaml +++ b/airflow/providers/oracle/provider.yaml @@ -59,7 +59,7 @@ versions: dependencies: - apache-airflow>=2.7.0 - - apache-airflow-providers-common-sql>=1.3.1 + - apache-airflow-providers-common-sql>=1.15.0 - oracledb>=2.0.0 additional-extras: diff --git a/airflow/providers/postgres/provider.yaml b/airflow/providers/postgres/provider.yaml index 1b593717505e..777fe23530ec 100644 --- a/airflow/providers/postgres/provider.yaml +++ b/airflow/providers/postgres/provider.yaml @@ -64,7 +64,7 @@ versions: dependencies: - apache-airflow>=2.7.0 - - apache-airflow-providers-common-sql>=1.14.2 + - apache-airflow-providers-common-sql>=1.15.0 - psycopg2-binary>=2.9.4 additional-extras: diff --git a/dev/breeze/tests/test_packages.py b/dev/breeze/tests/test_packages.py index 7e7269f66e09..2e502e042192 100644 --- a/dev/breeze/tests/test_packages.py +++ b/dev/breeze/tests/test_packages.py @@ -204,7 +204,7 @@ def test_get_documentation_package_path(): "postgres", "beta0", """ - "apache-airflow-providers-common-sql>=1.14.2b0", + "apache-airflow-providers-common-sql>=1.15.0b0", "apache-airflow>=2.7.0b0", "psycopg2-binary>=2.9.4", """, @@ -214,7 +214,7 @@ def test_get_documentation_package_path(): "postgres", "", """ - "apache-airflow-providers-common-sql>=1.14.2", + "apache-airflow-providers-common-sql>=1.15.0", "apache-airflow>=2.7.0", "psycopg2-binary>=2.9.4", """, From 20bf5136b561ac166bb43eafcb30664b6d7e13d5 Mon Sep 17 00:00:00 2001 From: dabla Date: Sun, 4 Aug 2024 12:16:30 +0200 Subject: [PATCH 20/32] refactor: updated provider dependencies --- generated/provider_dependencies.json | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/generated/provider_dependencies.json b/generated/provider_dependencies.json index 3b8bc4fb7272..6cb338318b6f 100644 --- a/generated/provider_dependencies.json +++ b/generated/provider_dependencies.json @@ -519,7 +519,7 @@ }, "elasticsearch": { "deps": [ - "apache-airflow-providers-common-sql>=1.14.2", + "apache-airflow-providers-common-sql>=1.15.0", "apache-airflow>=2.7.0", "elasticsearch>=8.10,<9" ], @@ -754,7 +754,7 @@ }, "jdbc": { "deps": [ - "apache-airflow-providers-common-sql>=1.14.2", + "apache-airflow-providers-common-sql>=1.15.0", "apache-airflow>=2.7.0", "jaydebeapi>=1.1.1" ], @@ -818,7 +818,7 @@ }, "microsoft.mssql": { "deps": [ - "apache-airflow-providers-common-sql>=1.14.2", + "apache-airflow-providers-common-sql>=1.15.0", "apache-airflow>=2.7.0", "pymssql>=2.3.0" ], @@ -868,7 +868,7 @@ }, "mysql": { "deps": [ - "apache-airflow-providers-common-sql>=1.14.1", + "apache-airflow-providers-common-sql>=1.15.0", "apache-airflow>=2.7.0", "mysql-connector-python>=8.0.29", "mysqlclient>=1.4.0" @@ -899,7 +899,7 @@ }, "odbc": { "deps": [ - "apache-airflow-providers-common-sql>=1.14.2", + "apache-airflow-providers-common-sql>=1.15.0", "apache-airflow>=2.7.0", "pyodbc>=5.0.0" ], @@ -977,7 +977,7 @@ }, "oracle": { "deps": [ - "apache-airflow-providers-common-sql>=1.3.1", + "apache-airflow-providers-common-sql>=1.15.0", "apache-airflow>=2.7.0", "oracledb>=2.0.0" ], @@ -1045,7 +1045,7 @@ }, "postgres": { "deps": [ - "apache-airflow-providers-common-sql>=1.14.2", + "apache-airflow-providers-common-sql>=1.15.0", "apache-airflow>=2.7.0", "psycopg2-binary>=2.9.4" ], From b1b16daaebd04bf0775ab07615c0748d46b931d2 Mon Sep 17 00:00:00 2001 From: David Blain Date: Sun, 4 Aug 2024 15:33:13 +0200 Subject: [PATCH 21/32] refactor: Use cached connection property in OracleHook --- airflow/providers/oracle/hooks/oracle.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airflow/providers/oracle/hooks/oracle.py b/airflow/providers/oracle/hooks/oracle.py index f6a45679546b..ee3d5aac2f06 100644 --- a/airflow/providers/oracle/hooks/oracle.py +++ b/airflow/providers/oracle/hooks/oracle.py @@ -143,7 +143,7 @@ def get_conn(self) -> oracledb.Connection: """ - conn = self.get_connection(self.oracle_conn_id) # type: ignore[attr-defined] + conn = self.connection conn_config = {"user": conn.login, "password": conn.password} sid = self.connection_extra_lower.get("sid") mod = self.connection_extra_lower.get("module") From 0ce1fec6ace2d57b1b0de77439ba5f3a9dd06764 Mon Sep 17 00:00:00 2001 From: David Blain Date: Sun, 4 Aug 2024 15:33:56 +0200 Subject: [PATCH 22/32] refactor: Removed deepcopy from connection --- airflow/providers/common/sql/hooks/sql.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/airflow/providers/common/sql/hooks/sql.py b/airflow/providers/common/sql/hooks/sql.py index 6bd4f9e8a6b3..1eb2a3fa38c7 100644 --- a/airflow/providers/common/sql/hooks/sql.py +++ b/airflow/providers/common/sql/hooks/sql.py @@ -19,7 +19,6 @@ import contextlib import warnings from contextlib import closing, contextmanager -from copy import deepcopy from datetime import datetime from functools import cached_property from typing import ( @@ -209,7 +208,7 @@ def placeholder(self): def connection(self) -> Connection: if self._connection is None: self._connection = self.get_connection(self.get_conn_id()) - return deepcopy(self._connection) + return self._connection @property def connection_extra(self) -> dict: From 28e0774a6ebd21adba8d5d790e46b19785cfdd01 Mon Sep 17 00:00:00 2001 From: David Blain Date: Sun, 4 Aug 2024 15:42:59 +0200 Subject: [PATCH 23/32] refactor: Updated docstring connection_extra_lower --- airflow/providers/common/sql/hooks/sql.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airflow/providers/common/sql/hooks/sql.py b/airflow/providers/common/sql/hooks/sql.py index 1eb2a3fa38c7..33d1cdf87996 100644 --- a/airflow/providers/common/sql/hooks/sql.py +++ b/airflow/providers/common/sql/hooks/sql.py @@ -219,7 +219,7 @@ def connection_extra_lower(self) -> dict: """ ``connection.extra_dejson`` but where keys are converted to lower case. - This is used internally for case-insensitive access of jdbc params. + This is used internally for case-insensitive access of extra params. """ return {k.lower(): v for k, v in self.connection_extra.items()} From 91ead4a95695465406899bf1d0ec1b53bd52d43a Mon Sep 17 00:00:00 2001 From: David Blain Date: Mon, 5 Aug 2024 09:45:58 +0200 Subject: [PATCH 24/32] refactor: Reformatted OracleHook --- airflow/providers/oracle/hooks/oracle.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/airflow/providers/oracle/hooks/oracle.py b/airflow/providers/oracle/hooks/oracle.py index ee3d5aac2f06..23f43f6314b8 100644 --- a/airflow/providers/oracle/hooks/oracle.py +++ b/airflow/providers/oracle/hooks/oracle.py @@ -174,7 +174,9 @@ def get_conn(self) -> oracledb.Connection: # Set oracledb Defaults Attributes if provided # (https://python-oracledb.readthedocs.io/en/latest/api_manual/defaults.html) - fetch_decimals = _get_first_bool(self.fetch_decimals, self.connection_extra_lower.get("fetch_decimals")) + fetch_decimals = _get_first_bool( + self.fetch_decimals, self.connection_extra_lower.get("fetch_decimals") + ) if isinstance(fetch_decimals, bool): oracledb.defaults.fetch_decimals = fetch_decimals From 67e9b7466bbcbc09b62b355365e364a8fee2bdc3 Mon Sep 17 00:00:00 2001 From: David Blain Date: Tue, 13 Aug 2024 15:36:11 +0200 Subject: [PATCH 25/32] refactor: Connection in PostgresHook must be deep copied otherwise tests will fail --- airflow/providers/postgres/hooks/postgres.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/airflow/providers/postgres/hooks/postgres.py b/airflow/providers/postgres/hooks/postgres.py index 2584f477d4e2..4790575565fc 100644 --- a/airflow/providers/postgres/hooks/postgres.py +++ b/airflow/providers/postgres/hooks/postgres.py @@ -20,6 +20,7 @@ import os import warnings from contextlib import closing +from copy import deepcopy from typing import TYPE_CHECKING, Any, Iterable, Union import psycopg2 @@ -140,7 +141,7 @@ def _get_cursor(self, raw_cursor: str) -> CursorType: def get_conn(self) -> connection: """Establish a connection to a postgres database.""" - conn = self.connection + conn = deepcopy(self.connection) # check for authentication via AWS IAM if conn.extra_dejson.get("iam", False): From 7f097328c082aaed343a1b703a7e880f4e427e5b Mon Sep 17 00:00:00 2001 From: David Blain Date: Tue, 13 Aug 2024 16:10:50 +0200 Subject: [PATCH 26/32] refactor: Refactored OracleHook as before --- airflow/providers/oracle/hooks/oracle.py | 30 +++++++++++------------- airflow/providers/oracle/provider.yaml | 2 +- 2 files changed, 15 insertions(+), 17 deletions(-) diff --git a/airflow/providers/oracle/hooks/oracle.py b/airflow/providers/oracle/hooks/oracle.py index 23f43f6314b8..a252a7599cd3 100644 --- a/airflow/providers/oracle/hooks/oracle.py +++ b/airflow/providers/oracle/hooks/oracle.py @@ -143,26 +143,26 @@ def get_conn(self) -> oracledb.Connection: """ - conn = self.connection + conn = self.get_connection(self.oracle_conn_id) # type: ignore[attr-defined] conn_config = {"user": conn.login, "password": conn.password} - sid = self.connection_extra_lower.get("sid") - mod = self.connection_extra_lower.get("module") + sid = conn.extra_dejson.get("sid") + mod = conn.extra_dejson.get("module") schema = conn.schema # Enable oracledb thick mode if thick_mode is set to True # Parameters take precedence over connection config extra # Defaults to use thin mode if not provided in params or connection config extra - thick_mode = _get_first_bool(self.thick_mode, self.connection_extra_lower.get("thick_mode")) + thick_mode = _get_first_bool(self.thick_mode, conn.extra_dejson.get("thick_mode")) if thick_mode is True: if self.thick_mode_lib_dir is None: - self.thick_mode_lib_dir = self.connection_extra_lower.get("thick_mode_lib_dir") + self.thick_mode_lib_dir = conn.extra_dejson.get("thick_mode_lib_dir") if not isinstance(self.thick_mode_lib_dir, (str, type(None))): raise TypeError( f"thick_mode_lib_dir expected str or None, " f"got {type(self.thick_mode_lib_dir).__name__}" ) if self.thick_mode_config_dir is None: - self.thick_mode_config_dir = self.connection_extra_lower.get("thick_mode_config_dir") + self.thick_mode_config_dir = conn.extra_dejson.get("thick_mode_config_dir") if not isinstance(self.thick_mode_config_dir, (str, type(None))): raise TypeError( f"thick_mode_config_dir expected str or None, " @@ -174,25 +174,23 @@ def get_conn(self) -> oracledb.Connection: # Set oracledb Defaults Attributes if provided # (https://python-oracledb.readthedocs.io/en/latest/api_manual/defaults.html) - fetch_decimals = _get_first_bool( - self.fetch_decimals, self.connection_extra_lower.get("fetch_decimals") - ) + fetch_decimals = _get_first_bool(self.fetch_decimals, conn.extra_dejson.get("fetch_decimals")) if isinstance(fetch_decimals, bool): oracledb.defaults.fetch_decimals = fetch_decimals - fetch_lobs = _get_first_bool(self.fetch_lobs, self.connection_extra_lower.get("fetch_lobs")) + fetch_lobs = _get_first_bool(self.fetch_lobs, conn.extra_dejson.get("fetch_lobs")) if isinstance(fetch_lobs, bool): oracledb.defaults.fetch_lobs = fetch_lobs # Set up DSN - service_name = self.connection_extra_lower.get("service_name") + service_name = conn.extra_dejson.get("service_name") port = conn.port if conn.port else 1521 if conn.host and sid and not service_name: conn_config["dsn"] = oracledb.makedsn(conn.host, port, sid) elif conn.host and service_name and not sid: conn_config["dsn"] = oracledb.makedsn(conn.host, port, service_name=service_name) else: - dsn = self.connection_extra_lower.get("dsn") + dsn = conn.extra_dejson.get("dsn") if dsn is None: dsn = conn.host if conn.port is not None: @@ -209,10 +207,10 @@ def get_conn(self) -> oracledb.Connection: dsn += f"/{conn.schema}" conn_config["dsn"] = dsn - if "events" in self.connection_extra_lower: - conn_config["events"] = self.connection_extra_lower.get("events") + if "events" in conn.extra_dejson: + conn_config["events"] = conn.extra_dejson.get("events") - mode = self.connection_extra_lower.get("mode", "").lower() + mode = conn.extra_dejson.get("mode", "").lower() if mode == "sysdba": conn_config["mode"] = oracledb.AUTH_MODE_SYSDBA elif mode == "sysasm": @@ -228,7 +226,7 @@ def get_conn(self) -> oracledb.Connection: elif mode == "sysrac": conn_config["mode"] = oracledb.AUTH_MODE_SYSRAC - purity = self.connection_extra_lower.get("purity", "").lower() + purity = conn.extra_dejson.get("purity", "").lower() if purity == "new": conn_config["purity"] = oracledb.PURITY_NEW elif purity == "self": diff --git a/airflow/providers/oracle/provider.yaml b/airflow/providers/oracle/provider.yaml index 7647260d48b3..c08b55a675c4 100644 --- a/airflow/providers/oracle/provider.yaml +++ b/airflow/providers/oracle/provider.yaml @@ -59,7 +59,7 @@ versions: dependencies: - apache-airflow>=2.8.0 - - apache-airflow-providers-common-sql>=1.15.0 + - apache-airflow-providers-common-sql>=1.3.1 - oracledb>=2.0.0 additional-extras: From d6ba693b17a4ec864f6fb3ccbc9af535717c033b Mon Sep 17 00:00:00 2001 From: dabla Date: Wed, 14 Aug 2024 09:34:15 +0200 Subject: [PATCH 27/32] refactor: updated provider dependencies --- generated/provider_dependencies.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/generated/provider_dependencies.json b/generated/provider_dependencies.json index d527ec7fd710..dfce4dd59526 100644 --- a/generated/provider_dependencies.json +++ b/generated/provider_dependencies.json @@ -978,7 +978,7 @@ }, "oracle": { "deps": [ - "apache-airflow-providers-common-sql>=1.15.0", + "apache-airflow-providers-common-sql>=1.3.1", "apache-airflow>=2.8.0", "oracledb>=2.0.0" ], From b5bc0ced223456391558b55866bb5fa1f75636eb Mon Sep 17 00:00:00 2001 From: David Blain Date: Mon, 2 Sep 2024 10:21:51 +0200 Subject: [PATCH 28/32] refactor: Updated sql common provider version --- airflow/providers/elasticsearch/provider.yaml | 2 +- airflow/providers/jdbc/provider.yaml | 2 +- airflow/providers/microsoft/mssql/provider.yaml | 2 +- airflow/providers/mysql/provider.yaml | 2 +- airflow/providers/odbc/provider.yaml | 2 +- airflow/providers/postgres/provider.yaml | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/airflow/providers/elasticsearch/provider.yaml b/airflow/providers/elasticsearch/provider.yaml index eac1dce4e916..102c0ede7e02 100644 --- a/airflow/providers/elasticsearch/provider.yaml +++ b/airflow/providers/elasticsearch/provider.yaml @@ -68,7 +68,7 @@ versions: dependencies: - apache-airflow>=2.8.0 - - apache-airflow-providers-common-sql>=1.15.0 + - apache-airflow-providers-common-sql>=1.17.0 - elasticsearch>=8.10,<9 integrations: diff --git a/airflow/providers/jdbc/provider.yaml b/airflow/providers/jdbc/provider.yaml index 8c647869f02a..2178b5dea91f 100644 --- a/airflow/providers/jdbc/provider.yaml +++ b/airflow/providers/jdbc/provider.yaml @@ -53,7 +53,7 @@ versions: dependencies: - apache-airflow>=2.8.0 - - apache-airflow-providers-common-sql>=1.15.0 + - apache-airflow-providers-common-sql>=1.17.0 - jaydebeapi>=1.1.1 integrations: diff --git a/airflow/providers/microsoft/mssql/provider.yaml b/airflow/providers/microsoft/mssql/provider.yaml index 257491ce6af4..7eab3a208dd4 100644 --- a/airflow/providers/microsoft/mssql/provider.yaml +++ b/airflow/providers/microsoft/mssql/provider.yaml @@ -55,7 +55,7 @@ versions: dependencies: - apache-airflow>=2.8.0 - - apache-airflow-providers-common-sql>=1.15.0 + - apache-airflow-providers-common-sql>=1.17.0 - pymssql>=2.3.0 # The methodtools dependency can be removed with min airflow version >=2.9.1 # as it was added in https://github.com/apache/airflow/pull/37757 diff --git a/airflow/providers/mysql/provider.yaml b/airflow/providers/mysql/provider.yaml index 048c3a1e71c3..355d23f946da 100644 --- a/airflow/providers/mysql/provider.yaml +++ b/airflow/providers/mysql/provider.yaml @@ -66,7 +66,7 @@ versions: dependencies: - apache-airflow>=2.8.0 - - apache-airflow-providers-common-sql>=1.15.0 + - apache-airflow-providers-common-sql>=1.17.0 - mysqlclient>=1.4.0 - mysql-connector-python>=8.0.29 diff --git a/airflow/providers/odbc/provider.yaml b/airflow/providers/odbc/provider.yaml index cf8fca609445..1a0dd5d7d3f7 100644 --- a/airflow/providers/odbc/provider.yaml +++ b/airflow/providers/odbc/provider.yaml @@ -54,7 +54,7 @@ versions: dependencies: - apache-airflow>=2.8.0 - - apache-airflow-providers-common-sql>=1.15.0 + - apache-airflow-providers-common-sql>=1.17.0 - pyodbc>=5.0.0 integrations: diff --git a/airflow/providers/postgres/provider.yaml b/airflow/providers/postgres/provider.yaml index 405309a45f3b..8a464340712a 100644 --- a/airflow/providers/postgres/provider.yaml +++ b/airflow/providers/postgres/provider.yaml @@ -65,7 +65,7 @@ versions: dependencies: - apache-airflow>=2.8.0 - - apache-airflow-providers-common-sql>=1.15.0 + - apache-airflow-providers-common-sql>=1.17.0 - psycopg2-binary>=2.9.4 additional-extras: From c9413f3b97010d3348d2e5a878d59568c8555be5 Mon Sep 17 00:00:00 2001 From: dabla Date: Mon, 2 Sep 2024 16:40:26 +0200 Subject: [PATCH 29/32] refactor: updated provider dependencies json file --- generated/provider_dependencies.json | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/generated/provider_dependencies.json b/generated/provider_dependencies.json index bb294db53a6a..7f8566689221 100644 --- a/generated/provider_dependencies.json +++ b/generated/provider_dependencies.json @@ -518,7 +518,7 @@ }, "elasticsearch": { "deps": [ - "apache-airflow-providers-common-sql>=1.15.0", + "apache-airflow-providers-common-sql>=1.17.0", "apache-airflow>=2.8.0", "elasticsearch>=8.10,<9" ], @@ -756,7 +756,7 @@ }, "jdbc": { "deps": [ - "apache-airflow-providers-common-sql>=1.15.0", + "apache-airflow-providers-common-sql>=1.17.0", "apache-airflow>=2.8.0", "jaydebeapi>=1.1.1" ], @@ -820,7 +820,7 @@ }, "microsoft.mssql": { "deps": [ - "apache-airflow-providers-common-sql>=1.15.0", + "apache-airflow-providers-common-sql>=1.17.0", "apache-airflow>=2.8.0", "methodtools>=0.4.7", "pymssql>=2.3.0" @@ -871,7 +871,7 @@ }, "mysql": { "deps": [ - "apache-airflow-providers-common-sql>=1.15.0", + "apache-airflow-providers-common-sql>=1.17.0", "apache-airflow>=2.8.0", "mysql-connector-python>=8.0.29", "mysqlclient>=1.4.0" @@ -902,7 +902,7 @@ }, "odbc": { "deps": [ - "apache-airflow-providers-common-sql>=1.15.0", + "apache-airflow-providers-common-sql>=1.17.0", "apache-airflow>=2.8.0", "pyodbc>=5.0.0" ], @@ -1047,7 +1047,7 @@ }, "postgres": { "deps": [ - "apache-airflow-providers-common-sql>=1.15.0", + "apache-airflow-providers-common-sql>=1.17.0", "apache-airflow>=2.8.0", "psycopg2-binary>=2.9.4" ], From 868cf4e10674d8da16878f6b89c90931ee658e55 Mon Sep 17 00:00:00 2001 From: David Blain Date: Mon, 2 Sep 2024 17:01:44 +0200 Subject: [PATCH 30/32] refactor: Updated common sql provider version to be tested in breeze --- dev/breeze/tests/test_packages.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/breeze/tests/test_packages.py b/dev/breeze/tests/test_packages.py index c90f837039b6..9556ae695be8 100644 --- a/dev/breeze/tests/test_packages.py +++ b/dev/breeze/tests/test_packages.py @@ -204,7 +204,7 @@ def test_get_documentation_package_path(): "postgres", "beta0", """ - "apache-airflow-providers-common-sql>=1.15.0b0", + "apache-airflow-providers-common-sql>=1.17.0b0", "apache-airflow>=2.8.0b0", "psycopg2-binary>=2.9.4", """, @@ -214,7 +214,7 @@ def test_get_documentation_package_path(): "postgres", "", """ - "apache-airflow-providers-common-sql>=1.15.0", + "apache-airflow-providers-common-sql>=1.17.0", "apache-airflow>=2.8.0", "psycopg2-binary>=2.9.4", """, From adaca5e3f96460df803d74dfc0c2425de5f359e8 Mon Sep 17 00:00:00 2001 From: David Blain Date: Tue, 3 Sep 2024 10:57:13 +0200 Subject: [PATCH 31/32] refactor: Bumped version to 1.17.0 of common sql provider --- airflow/providers/common/sql/provider.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/airflow/providers/common/sql/provider.yaml b/airflow/providers/common/sql/provider.yaml index 06c43af1d8db..f600acc1fa44 100644 --- a/airflow/providers/common/sql/provider.yaml +++ b/airflow/providers/common/sql/provider.yaml @@ -25,6 +25,7 @@ state: ready source-date-epoch: 1723970051 # note that those versions are maintained by release manager - do not update them manually versions: + - 1.17.0 - 1.16.0 - 1.15.0 - 1.14.2 From 2c2ed77b19ae792832b9df974f7617d181ed1e08 Mon Sep 17 00:00:00 2001 From: David Blain Date: Tue, 3 Sep 2024 10:57:52 +0200 Subject: [PATCH 32/32] refactor: Added explanation what has changed in common sql provider for 1.17.0 within CHANGELOG.rst --- airflow/providers/common/sql/CHANGELOG.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/airflow/providers/common/sql/CHANGELOG.rst b/airflow/providers/common/sql/CHANGELOG.rst index 8cd190482c23..6938a7d838a7 100644 --- a/airflow/providers/common/sql/CHANGELOG.rst +++ b/airflow/providers/common/sql/CHANGELOG.rst @@ -25,6 +25,14 @@ Changelog --------- +1.17.0 +...... + +Features +~~~~~~~~ + +* ``Connection in DB Hook is now cached to avoid multiple lookups when properties from extras have to be resolved`` + 1.16.0 ......