From abb2fa7a02e18f0883b4c9ca73ffb14ce0af9333 Mon Sep 17 00:00:00 2001 From: Vincent <97131062+vincbeck@users.noreply.github.com> Date: Fri, 6 Dec 2024 15:22:51 -0500 Subject: [PATCH 1/5] Fix main (#44747) --- airflow/ui/src/layouts/Nav/Nav.tsx | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/airflow/ui/src/layouts/Nav/Nav.tsx b/airflow/ui/src/layouts/Nav/Nav.tsx index c17e57609061f..f716d612e8af4 100644 --- a/airflow/ui/src/layouts/Nav/Nav.tsx +++ b/airflow/ui/src/layouts/Nav/Nav.tsx @@ -20,7 +20,6 @@ import { Box, Flex, VStack, Link } from "@chakra-ui/react"; import { FiCornerUpLeft, FiDatabase, FiHome, FiSettings } from "react-icons/fi"; import { useVersionServiceGetVersion } from "openapi/queries"; - import { AirflowPin } from "src/assets/AirflowPin"; import { DagIcon } from "src/assets/DagIcon"; @@ -89,4 +88,4 @@ export const Nav = () => { ); -} +}; From 696cb29f3bb5f0a1a947d183688cbd4f30bfeb3f Mon Sep 17 00:00:00 2001 From: Pratiksha <128999446+Prab-27@users.noreply.github.com> Date: Sat, 7 Dec 2024 02:52:37 +0530 Subject: [PATCH 2/5] Removed deprecated code from hashicorp provider (#44598) * Remove deprecated code from HashiCorp provider * Remove additional deprecated code from hashicorp provider * remove test code for deprecated method * fix if clause for approle authentication * modified changelog --------- Co-authored-by: pratiksha rajendrabhai badheka --- .../airflow/providers/hashicorp/CHANGELOG.rst | 12 + .../providers/hashicorp/hooks/vault.py | 25 +- .../providers/hashicorp/secrets/vault.py | 19 -- .../tests/hashicorp/secrets/test_vault.py | 216 ------------------ 4 files changed, 15 insertions(+), 257 deletions(-) diff --git a/providers/src/airflow/providers/hashicorp/CHANGELOG.rst b/providers/src/airflow/providers/hashicorp/CHANGELOG.rst index 032f9d565c85a..0c80950db363f 100644 --- a/providers/src/airflow/providers/hashicorp/CHANGELOG.rst +++ b/providers/src/airflow/providers/hashicorp/CHANGELOG.rst @@ -27,6 +27,18 @@ Changelog --------- +main +..... + +.. warning:: + All deprecated classes, parameters and features have been removed from the hashicorp provider package. + The following breaking changes were introduced: + + * The usage of role_id for AppRole authentication has been deprecated from airflow.providers.hashicorp.hook.vault .Please use connection login + * The usage of role_id in connection extra for AppRole authentication has been deprecated from airflow.providers.hashicorp.hook.vault. Please use connection login + * Removed role_id from get_connection_form_widgets + * Removed deprecated method ``VaultBackend.get_conn_uri`` from airflow.providers.hashicorp.secrets.vault + 3.8.0 ..... diff --git a/providers/src/airflow/providers/hashicorp/hooks/vault.py b/providers/src/airflow/providers/hashicorp/hooks/vault.py index ccee598d6cc14..37e409f388af2 100644 --- a/providers/src/airflow/providers/hashicorp/hooks/vault.py +++ b/providers/src/airflow/providers/hashicorp/hooks/vault.py @@ -19,12 +19,10 @@ from __future__ import annotations import json -import warnings from typing import TYPE_CHECKING, Any from hvac.exceptions import VaultError -from airflow.exceptions import AirflowProviderDeprecationWarning from airflow.hooks.base import BaseHook from airflow.providers.hashicorp._internal_client.vault_client import ( DEFAULT_KUBERNETES_JWT_PATH, @@ -70,7 +68,7 @@ class VaultHook(BaseHook): Login/Password are used as credentials: - * approle: login -> role_id, password -> secret_id + * approle: login -> connection.login * github: password -> token * token: password -> token * aws_iam: login -> key_id, password -> secret_id @@ -147,24 +145,8 @@ def __init__( if kwargs: client_kwargs = merge_dicts(client_kwargs, kwargs) - if auth_type == "approle": - if role_id: - warnings.warn( - """The usage of role_id for AppRole authentication has been deprecated. - Please use connection login.""", - AirflowProviderDeprecationWarning, - stacklevel=2, - ) - elif self.connection.extra_dejson.get("role_id"): - role_id = self.connection.extra_dejson.get("role_id") - warnings.warn( - """The usage of role_id in connection extra for AppRole authentication has been - deprecated. Please use connection login.""", - AirflowProviderDeprecationWarning, - stacklevel=2, - ) - elif self.connection.login: - role_id = self.connection.login + if auth_type == "approle" and self.connection.login: + role_id = self.connection.login if auth_type == "aws_iam": if not role_id: @@ -385,7 +367,6 @@ def get_connection_form_widgets(cls) -> dict[str, Any]: description="Must be 1 or 2.", default=DEFAULT_KV_ENGINE_VERSION, ), - "role_id": StringField(lazy_gettext("Role ID (deprecated)"), widget=BS3TextFieldWidget()), "kubernetes_role": StringField(lazy_gettext("Kubernetes role"), widget=BS3TextFieldWidget()), "kubernetes_jwt_path": StringField( lazy_gettext("Kubernetes jwt path"), widget=BS3TextFieldWidget() diff --git a/providers/src/airflow/providers/hashicorp/secrets/vault.py b/providers/src/airflow/providers/hashicorp/secrets/vault.py index 2591c77652e26..275c0dc79baa0 100644 --- a/providers/src/airflow/providers/hashicorp/secrets/vault.py +++ b/providers/src/airflow/providers/hashicorp/secrets/vault.py @@ -21,9 +21,6 @@ from typing import TYPE_CHECKING -from deprecated import deprecated - -from airflow.exceptions import AirflowProviderDeprecationWarning from airflow.providers.hashicorp._internal_client.vault_client import _VaultClient from airflow.secrets import BaseSecretsBackend from airflow.utils.log.logging_mixin import LoggingMixin @@ -191,22 +188,6 @@ def get_response(self, conn_id: str) -> dict | None: secret_path=(mount_point + "/" if mount_point else "") + secret_path ) - @deprecated( - reason="Method `VaultBackend.get_conn_uri` is deprecated and will be removed in a future release.", - category=AirflowProviderDeprecationWarning, - ) - def get_conn_uri(self, conn_id: str) -> str | None: - """ - Get serialized representation of connection. - - :param conn_id: The connection id - :return: The connection uri retrieved from the secret - """ - # Since VaultBackend implements `get_connection`, `get_conn_uri` is not used. So we - # don't need to implement (or direct users to use) method `get_conn_value` instead - response = self.get_response(conn_id) - return response.get("conn_uri") if response else None - # Make sure connection is imported this way for type checking, otherwise when importing # the backend it will get a circular dependency and fail if TYPE_CHECKING: diff --git a/providers/tests/hashicorp/secrets/test_vault.py b/providers/tests/hashicorp/secrets/test_vault.py index 14834e2cbd07a..8b5e1b3cfc4df 100644 --- a/providers/tests/hashicorp/secrets/test_vault.py +++ b/providers/tests/hashicorp/secrets/test_vault.py @@ -21,89 +21,10 @@ import pytest from hvac.exceptions import InvalidPath, VaultError -from airflow.exceptions import AirflowProviderDeprecationWarning from airflow.providers.hashicorp.secrets.vault import VaultBackend class TestVaultSecrets: - @mock.patch("airflow.providers.hashicorp._internal_client.vault_client.hvac") - def test_get_conn_uri(self, mock_hvac): - mock_client = mock.MagicMock() - mock_hvac.Client.return_value = mock_client - mock_client.secrets.kv.v2.read_secret_version.return_value = { - "request_id": "94011e25-f8dc-ec29-221b-1f9c1d9ad2ae", - "lease_id": "", - "renewable": False, - "lease_duration": 0, - "data": { - "data": {"conn_uri": "postgresql://airflow:airflow@host:5432/airflow"}, - "metadata": { - "created_time": "2020-03-16T21:01:43.331126Z", - "deletion_time": "", - "destroyed": False, - "version": 1, - }, - }, - "wrap_info": None, - "warnings": None, - "auth": None, - } - - kwargs = { - "connections_path": "connections", - "mount_point": "airflow", - "auth_type": "token", - "url": "http://127.0.0.1:8200", - "token": "s.7AU0I51yv1Q1lxOIg1F3ZRAS", - } - - test_client = VaultBackend(**kwargs) - with pytest.warns( - AirflowProviderDeprecationWarning, - match="Method `VaultBackend.get_conn_uri` is deprecated and will be removed in a future release.", - ): - returned_uri = test_client.get_conn_uri(conn_id="test_postgres") - assert returned_uri == "postgresql://airflow:airflow@host:5432/airflow" - - @mock.patch("airflow.providers.hashicorp._internal_client.vault_client.hvac") - def test_get_conn_uri_without_predefined_mount_point(self, mock_hvac): - mock_client = mock.MagicMock() - mock_hvac.Client.return_value = mock_client - mock_client.secrets.kv.v2.read_secret_version.return_value = { - "request_id": "94011e25-f8dc-ec29-221b-1f9c1d9ad2ae", - "lease_id": "", - "renewable": False, - "lease_duration": 0, - "data": { - "data": {"conn_uri": "postgresql://airflow:airflow@host:5432/airflow"}, - "metadata": { - "created_time": "2020-03-16T21:01:43.331126Z", - "deletion_time": "", - "destroyed": False, - "version": 1, - }, - }, - "wrap_info": None, - "warnings": None, - "auth": None, - } - - kwargs = { - "connections_path": "connections", - "mount_point": None, - "auth_type": "token", - "url": "http://127.0.0.1:8200", - "token": "s.7AU0I51yv1Q1lxOIg1F3ZRAS", - } - - test_client = VaultBackend(**kwargs) - with pytest.warns( - AirflowProviderDeprecationWarning, - match="Method `VaultBackend.get_conn_uri` is deprecated and will be removed in a future release.", - ): - returned_uri = test_client.get_conn_uri(conn_id="airflow/test_postgres") - assert returned_uri == "postgresql://airflow:airflow@host:5432/airflow" - @mock.patch("airflow.providers.hashicorp._internal_client.vault_client.hvac") def test_get_connection(self, mock_hvac): mock_client = mock.MagicMock() @@ -190,143 +111,6 @@ def test_get_connection_without_predefined_mount_point(self, mock_hvac): connection = test_client.get_connection(conn_id="airflow/test_postgres") assert connection.get_uri() == "postgresql://airflow:airflow@host:5432/airflow?foo=bar&baz=taz" - @pytest.mark.parametrize( - "mount_point, connections_path, conn_id, expected_args", - [ - ( - "airflow", - "connections", - "test_postgres", - {"mount_point": "airflow", "path": "connections/test_postgres"}, - ), - ( - "airflow", - "", - "path/to/connections/test_postgres", - {"mount_point": "airflow", "path": "path/to/connections/test_postgres"}, - ), - ( - None, - "connections", - "airflow/test_postgres", - {"mount_point": "airflow", "path": "connections/test_postgres"}, - ), - ( - None, - "", - "airflow/path/to/connections/test_postgres", - {"mount_point": "airflow", "path": "path/to/connections/test_postgres"}, - ), - ], - ) - @mock.patch("airflow.providers.hashicorp._internal_client.vault_client.hvac") - def test_get_conn_uri_engine_version_1( - self, mock_hvac, mount_point, connections_path, conn_id, expected_args - ): - mock_client = mock.MagicMock() - mock_hvac.Client.return_value = mock_client - mock_client.secrets.kv.v1.read_secret.return_value = { - "request_id": "182d0673-618c-9889-4cba-4e1f4cfe4b4b", - "lease_id": "", - "renewable": False, - "lease_duration": 2764800, - "data": {"conn_uri": "postgresql://airflow:airflow@host:5432/airflow"}, - "wrap_info": None, - "warnings": None, - "auth": None, - } - - kwargs = { - "connections_path": connections_path, - "mount_point": mount_point, - "auth_type": "token", - "url": "http://127.0.0.1:8200", - "token": "s.7AU0I51yv1Q1lxOIg1F3ZRAS", - "kv_engine_version": 1, - } - - test_client = VaultBackend(**kwargs) - with pytest.warns( - AirflowProviderDeprecationWarning, - match="Method `VaultBackend.get_conn_uri` is deprecated and will be removed in a future release.", - ): - returned_uri = test_client.get_conn_uri(conn_id=conn_id) - mock_client.secrets.kv.v1.read_secret.assert_called_once_with(**expected_args) - assert returned_uri == "postgresql://airflow:airflow@host:5432/airflow" - - @mock.patch("airflow.providers.hashicorp._internal_client.vault_client.hvac") - def test_get_conn_uri_engine_version_1_custom_auth_mount_point(self, mock_hvac): - mock_client = mock.MagicMock() - mock_hvac.Client.return_value = mock_client - mock_client.secrets.kv.v1.read_secret.return_value = { - "request_id": "182d0673-618c-9889-4cba-4e1f4cfe4b4b", - "lease_id": "", - "renewable": False, - "lease_duration": 2764800, - "data": {"conn_uri": "postgresql://airflow:airflow@host:5432/airflow"}, - "wrap_info": None, - "warnings": None, - "auth": None, - } - - kwargs = { - "connections_path": "connections", - "mount_point": "airflow", - "auth_mount_point": "custom", - "auth_type": "token", - "url": "http://127.0.0.1:8200", - "token": "s.7AU0I51yv1Q1lxOIg1F3ZRAS", - "kv_engine_version": 1, - } - - test_client = VaultBackend(**kwargs) - assert test_client.vault_client.auth_mount_point == "custom" - with pytest.warns( - AirflowProviderDeprecationWarning, - match="Method `VaultBackend.get_conn_uri` is deprecated and will be removed in a future release.", - ): - returned_uri = test_client.get_conn_uri(conn_id="test_postgres") - mock_client.secrets.kv.v1.read_secret.assert_called_once_with( - mount_point="airflow", path="connections/test_postgres" - ) - assert returned_uri == "postgresql://airflow:airflow@host:5432/airflow" - - @mock.patch.dict( - "os.environ", - { - "AIRFLOW_CONN_TEST_MYSQL": "mysql://airflow:airflow@host:5432/airflow", - }, - ) - @mock.patch("airflow.providers.hashicorp._internal_client.vault_client.hvac") - def test_get_conn_uri_non_existent_key(self, mock_hvac): - """ - Test that if the key with connection ID is not present in Vault, _VaultClient.get_connection - should return None - """ - mock_client = mock.MagicMock() - mock_hvac.Client.return_value = mock_client - # Response does not contain the requested key - mock_client.secrets.kv.v2.read_secret_version.side_effect = InvalidPath() - - kwargs = { - "connections_path": "connections", - "mount_point": "airflow", - "auth_type": "token", - "url": "http://127.0.0.1:8200", - "token": "s.7AU0I51yv1Q1lxOIg1F3ZRAS", - } - - test_client = VaultBackend(**kwargs) - with pytest.warns( - AirflowProviderDeprecationWarning, - match="Method `VaultBackend.get_conn_uri` is deprecated and will be removed in a future release.", - ): - assert test_client.get_conn_uri(conn_id="test_mysql") is None - mock_client.secrets.kv.v2.read_secret_version.assert_called_once_with( - mount_point="airflow", path="connections/test_mysql", version=None, raise_on_deleted_version=True - ) - assert test_client.get_connection(conn_id="test_mysql") is None - @mock.patch("airflow.providers.hashicorp._internal_client.vault_client.hvac") def test_get_variable_value(self, mock_hvac): mock_client = mock.MagicMock() From 1ff9fe80103c454e5ba94402aceaf9a1727e345b Mon Sep 17 00:00:00 2001 From: Jens Scheffler <95105677+jscheffl@users.noreply.github.com> Date: Fri, 6 Dec 2024 22:50:16 +0100 Subject: [PATCH 3/5] Remove Provider Deprecations in Oracle (#44704) * Remove Provider Deprecations in Oracle * Remove docs from removed operator and examples --- .../operators/index.rst | 14 ------- .../airflow/providers/oracle/CHANGELOG.rst | 15 +++++++ .../oracle/example_dags/example_oracle.py | 9 ----- .../airflow/providers/oracle/hooks/oracle.py | 9 ----- .../providers/oracle/operators/oracle.py | 40 +------------------ .../tests/oracle/operators/test_oracle.py | 32 +-------------- 6 files changed, 17 insertions(+), 102 deletions(-) diff --git a/docs/apache-airflow-providers-oracle/operators/index.rst b/docs/apache-airflow-providers-oracle/operators/index.rst index 990cb2e6503b2..0a09991085e33 100644 --- a/docs/apache-airflow-providers-oracle/operators/index.rst +++ b/docs/apache-airflow-providers-oracle/operators/index.rst @@ -22,20 +22,6 @@ Oracle Operators ================ The Oracle connection type provides connection to a Oracle database. -Execute SQL in an Oracle database ---------------------------------- - -To execute arbitrary SQL in an Oracle database, use the -:class:`~airflow.providers.oracle.operators.oracle.OracleOperator`. - -An example of executing a simple query is as follows: - -.. exampleinclude:: /../../providers/src/airflow/providers/oracle/example_dags/example_oracle.py - :language: python - :start-after: [START howto_oracle_operator] - :end-before: [END howto_oracle_operator] - - Execute a Stored Procedure in an Oracle database ------------------------------------------------ diff --git a/providers/src/airflow/providers/oracle/CHANGELOG.rst b/providers/src/airflow/providers/oracle/CHANGELOG.rst index ac2306379a700..394b8af9c14d6 100644 --- a/providers/src/airflow/providers/oracle/CHANGELOG.rst +++ b/providers/src/airflow/providers/oracle/CHANGELOG.rst @@ -27,6 +27,21 @@ Changelog --------- +main +.... + +Breaking changes +~~~~~~~~~~~~~~~~ + +.. warning:: + All deprecated classes, parameters and features have been removed from the Oracle provider package. + The following breaking changes were introduced: + + * Hooks + * Remove deprecated support setting the Oracle Service Name using ``conn.schema``. Please use ``conn.extra.service_name`` instead. + * Operators + * Remove ``airflow.providers.oracle.operators.oracle.OracleOperator``. Please use ``airflow.providers.common.sql.operators.sql.SQLExecuteQueryOperator`` instead. + 3.12.1 ...... diff --git a/providers/src/airflow/providers/oracle/example_dags/example_oracle.py b/providers/src/airflow/providers/oracle/example_dags/example_oracle.py index a68c0be6c882d..5501831e946ee 100644 --- a/providers/src/airflow/providers/oracle/example_dags/example_oracle.py +++ b/providers/src/airflow/providers/oracle/example_dags/example_oracle.py @@ -19,7 +19,6 @@ from datetime import datetime from airflow import DAG -from airflow.providers.common.sql.operators.sql import SQLExecuteQueryOperator from airflow.providers.oracle.operators.oracle import OracleStoredProcedureOperator with DAG( @@ -29,14 +28,6 @@ start_date=datetime(2023, 1, 1), dag_id="example_oracle", ) as dag: - # [START howto_oracle_operator] - - opr_sql = SQLExecuteQueryOperator( - task_id="task_sql", conn_id="oracle", sql="SELECT 1 FROM DUAL", autocommit=True - ) - - # [END howto_oracle_operator] - # [START howto_oracle_stored_procedure_operator_with_list_inout] opr_stored_procedure_with_list_input_output = OracleStoredProcedureOperator( diff --git a/providers/src/airflow/providers/oracle/hooks/oracle.py b/providers/src/airflow/providers/oracle/hooks/oracle.py index 192a2da4b1a64..f09cbc0d479e1 100644 --- a/providers/src/airflow/providers/oracle/hooks/oracle.py +++ b/providers/src/airflow/providers/oracle/hooks/oracle.py @@ -23,7 +23,6 @@ import oracledb -from airflow.exceptions import AirflowProviderDeprecationWarning from airflow.providers.common.sql.hooks.sql import DbApiHook PARAM_TYPES = {bool, float, int, str} @@ -197,14 +196,6 @@ def get_conn(self) -> oracledb.Connection: dsn += f":{conn.port}" if service_name: dsn += f"/{service_name}" - elif conn.schema: - warnings.warn( - """Using conn.schema to pass the Oracle Service Name is deprecated. - Please use conn.extra.service_name instead.""", - AirflowProviderDeprecationWarning, - stacklevel=2, - ) - dsn += f"/{conn.schema}" conn_config["dsn"] = dsn if "events" in conn.extra_dejson: diff --git a/providers/src/airflow/providers/oracle/operators/oracle.py b/providers/src/airflow/providers/oracle/operators/oracle.py index 60da9fcb70dad..9e1247262ffaf 100644 --- a/providers/src/airflow/providers/oracle/operators/oracle.py +++ b/providers/src/airflow/providers/oracle/operators/oracle.py @@ -19,55 +19,17 @@ import re from collections.abc import Sequence -from typing import TYPE_CHECKING, ClassVar +from typing import TYPE_CHECKING import oracledb -from deprecated import deprecated -from airflow.exceptions import AirflowProviderDeprecationWarning from airflow.models import BaseOperator -from airflow.providers.common.sql.operators.sql import SQLExecuteQueryOperator from airflow.providers.oracle.hooks.oracle import OracleHook if TYPE_CHECKING: from airflow.utils.context import Context -@deprecated( - reason="Please use `airflow.providers.common.sql.operators.sql.SQLExecuteQueryOperator`.", - category=AirflowProviderDeprecationWarning, -) -class OracleOperator(SQLExecuteQueryOperator): - """ - Executes sql code in a specific Oracle database. - - This class is deprecated. - - Please use :class:`airflow.providers.common.sql.operators.sql.SQLExecuteQueryOperator`. - - :param sql: the sql code to be executed. Can receive a str representing a sql statement, - a list of str (sql statements), or reference to a template file. - Template reference are recognized by str ending in '.sql' - (templated) - :param oracle_conn_id: The :ref:`Oracle connection id ` - reference to a specific Oracle database. - :param parameters: (optional, templated) the parameters to render the SQL query with. - :param autocommit: if True, each command is automatically committed. - (default value: False) - """ - - template_fields: Sequence[str] = ( - "parameters", - "sql", - ) - template_ext: Sequence[str] = (".sql",) - template_fields_renderers: ClassVar[dict] = {"sql": "sql"} - ui_color = "#ededed" - - def __init__(self, *, oracle_conn_id: str = "oracle_default", **kwargs) -> None: - super().__init__(conn_id=oracle_conn_id, **kwargs) - - class OracleStoredProcedureOperator(BaseOperator): """ Executes stored procedure in a specific Oracle database. diff --git a/providers/tests/oracle/operators/test_oracle.py b/providers/tests/oracle/operators/test_oracle.py index 623e214efc3d0..2f06e1513472a 100644 --- a/providers/tests/oracle/operators/test_oracle.py +++ b/providers/tests/oracle/operators/test_oracle.py @@ -23,39 +23,9 @@ import oracledb import pytest -from airflow.exceptions import AirflowProviderDeprecationWarning from airflow.models import TaskInstance -from airflow.providers.common.sql.hooks.sql import fetch_all_handler from airflow.providers.oracle.hooks.oracle import OracleHook -from airflow.providers.oracle.operators.oracle import OracleOperator, OracleStoredProcedureOperator - - -class TestOracleOperator: - @mock.patch("airflow.providers.common.sql.operators.sql.SQLExecuteQueryOperator.get_db_hook") - def test_execute(self, mock_get_db_hook): - sql = "SELECT * FROM test_table" - oracle_conn_id = "oracle_default" - parameters = {"parameter": "value"} - autocommit = False - context = "test_context" - task_id = "test_task_id" - - with pytest.warns(AirflowProviderDeprecationWarning, match="Call to deprecated class *"): - operator = OracleOperator( - sql=sql, - oracle_conn_id=oracle_conn_id, - parameters=parameters, - autocommit=autocommit, - task_id=task_id, - ) - operator.execute(context=context) - mock_get_db_hook.return_value.run.assert_called_once_with( - sql=sql, - autocommit=autocommit, - parameters=parameters, - handler=fetch_all_handler, - return_last=True, - ) +from airflow.providers.oracle.operators.oracle import OracleStoredProcedureOperator class TestOracleStoredProcedureOperator: From ee6f6fbbb1a04ed38f9269193cfdfb2b36866e72 Mon Sep 17 00:00:00 2001 From: LIU ZHE YOU <68415893+jason810496@users.noreply.github.com> Date: Sat, 7 Dec 2024 06:16:13 +0800 Subject: [PATCH 4/5] Remove Provider Deprecations in Redis (#44633) * Remove Provider Deprecations in Redis * Fix mock_get_connection in test_redis --- .../src/airflow/providers/redis/CHANGELOG.rst | 11 +++ .../airflow/providers/redis/hooks/redis.py | 14 +--- providers/tests/redis/hooks/test_redis.py | 74 ++++--------------- 3 files changed, 28 insertions(+), 71 deletions(-) diff --git a/providers/src/airflow/providers/redis/CHANGELOG.rst b/providers/src/airflow/providers/redis/CHANGELOG.rst index 2f53fb18b0c32..715553e757f25 100644 --- a/providers/src/airflow/providers/redis/CHANGELOG.rst +++ b/providers/src/airflow/providers/redis/CHANGELOG.rst @@ -27,6 +27,17 @@ Changelog --------- +main +..... + +.. warning:: + All deprecated classes, parameters and features have been removed from the Redis provider package. + The following breaking changes were introduced: + + * Hooks + + * Removed ``ssl_cert_file`` parameter from ``RedisHook``. Use ``ssl_certfile`` instead + 3.8.0 ..... diff --git a/providers/src/airflow/providers/redis/hooks/redis.py b/providers/src/airflow/providers/redis/hooks/redis.py index 10890f36b0eb1..2f956a5cb137e 100644 --- a/providers/src/airflow/providers/redis/hooks/redis.py +++ b/providers/src/airflow/providers/redis/hooks/redis.py @@ -19,12 +19,10 @@ from __future__ import annotations -import warnings from typing import Any from redis import Redis -from airflow.exceptions import AirflowProviderDeprecationWarning from airflow.hooks.base import BaseHook DEFAULT_SSL_CERT_REQS = "required" @@ -37,7 +35,7 @@ class RedisHook(BaseHook): You can set your db in the extra field of your connection as ``{"db": 3}``. Also you can set ssl parameters as: - ``{"ssl": true, "ssl_cert_reqs": "require", "ssl_cert_file": "/path/to/cert.pem", etc}``. + ``{"ssl": true, "ssl_cert_reqs": "require", "ssl_certfile": "/path/to/cert.pem", etc}``. """ conn_name_attr = "redis_conn_id" @@ -81,16 +79,6 @@ def get_conn(self): ] ssl_args = {name: val for name, val in conn.extra_dejson.items() if name in ssl_arg_names} - # This logic is for backward compatibility only - if "ssl_cert_file" in conn.extra_dejson and "ssl_certfile" not in conn.extra_dejson: - warnings.warn( - "Extra parameter `ssl_cert_file` deprecated and will be removed " - "in a future release. Please use `ssl_certfile` instead.", - AirflowProviderDeprecationWarning, - stacklevel=2, - ) - ssl_args["ssl_certfile"] = conn.extra_dejson.get("ssl_cert_file") - if not self.redis: self.log.debug( 'Initializing redis object for conn_id "%s" on %s:%s:%s', diff --git a/providers/tests/redis/hooks/test_redis.py b/providers/tests/redis/hooks/test_redis.py index 1216dc6b64131..b64ffb909d241 100644 --- a/providers/tests/redis/hooks/test_redis.py +++ b/providers/tests/redis/hooks/test_redis.py @@ -21,7 +21,6 @@ import pytest -from airflow.exceptions import AirflowProviderDeprecationWarning from airflow.models import Connection from airflow.providers.redis.hooks.redis import RedisHook @@ -29,11 +28,6 @@ class TestRedisHook: - deprecation_message = ( - "Extra parameter `ssl_cert_file` deprecated and will be removed " - "in a future release. Please use `ssl_certfile` instead." - ) - def test_get_conn(self): hook = RedisHook(redis_conn_id="redis_default") assert hook.redis is None @@ -47,24 +41,26 @@ def test_get_conn(self): @mock.patch("airflow.providers.redis.hooks.redis.Redis") @mock.patch( "airflow.providers.redis.hooks.redis.RedisHook.get_connection", - return_value=Connection( + ) + def test_get_conn_with_extra_config(self, mock_get_connection, mock_redis): + connection = Connection( login="user", password="password", host="remote_host", port=1234, - extra="""{ - "db": 2, - "ssl": true, - "ssl_cert_reqs": "required", - "ssl_ca_certs": "/path/to/custom/ca-cert", - "ssl_keyfile": "/path/to/key-file", - "ssl_certfile": "/path/to/cert-file", - "ssl_check_hostname": true - }""", - ), - ) - def test_get_conn_with_extra_config(self, mock_get_connection, mock_redis): - connection = mock_get_connection.return_value + ) + connection.set_extra( + """{ + "db": 2, + "ssl": true, + "ssl_cert_reqs": "required", + "ssl_ca_certs": "/path/to/custom/ca-cert", + "ssl_keyfile": "/path/to/key-file", + "ssl_certfile": "/path/to/cert-file", + "ssl_check_hostname": true + }""" + ) + mock_get_connection.return_value = connection hook = RedisHook() hook.get_conn() @@ -82,44 +78,6 @@ def test_get_conn_with_extra_config(self, mock_get_connection, mock_redis): ssl_check_hostname=connection.extra_dejson["ssl_check_hostname"], ) - @mock.patch("airflow.providers.redis.hooks.redis.Redis") - @mock.patch( - "airflow.providers.redis.hooks.redis.RedisHook.get_connection", - return_value=Connection( - password="password", - host="remote_host", - port=1234, - extra="""{ - "db": 2, - "ssl": true, - "ssl_cert_reqs": "required", - "ssl_ca_certs": "/path/to/custom/ca-cert", - "ssl_keyfile": "/path/to/key-file", - "ssl_cert_file": "/path/to/cert-file", - "ssl_check_hostname": true - }""", - ), - ) - def test_get_conn_with_deprecated_extra_config(self, mock_get_connection, mock_redis): - connection = mock_get_connection.return_value - hook = RedisHook() - - with pytest.warns(AirflowProviderDeprecationWarning, match=self.deprecation_message): - hook.get_conn() - mock_redis.assert_called_once_with( - host=connection.host, - password=connection.password, - username=None, - port=connection.port, - db=connection.extra_dejson["db"], - ssl=connection.extra_dejson["ssl"], - ssl_cert_reqs=connection.extra_dejson["ssl_cert_reqs"], - ssl_ca_certs=connection.extra_dejson["ssl_ca_certs"], - ssl_keyfile=connection.extra_dejson["ssl_keyfile"], - ssl_certfile=connection.extra_dejson["ssl_cert_file"], - ssl_check_hostname=connection.extra_dejson["ssl_check_hostname"], - ) - def test_get_conn_password_stays_none(self): hook = RedisHook(redis_conn_id="redis_default") hook.get_conn() From 8bbba50d4a87f8c944465cded15f537b588076ac Mon Sep 17 00:00:00 2001 From: vatsrahul1001 <43964496+vatsrahul1001@users.noreply.github.com> Date: Sat, 7 Dec 2024 09:32:58 +0530 Subject: [PATCH 5/5] Remove deprecations from Apache hive Provider (#44715) * remove deprecations * Update providers/src/airflow/providers/apache/hive/CHANGELOG.rst Co-authored-by: Wei Lee --------- Co-authored-by: Wei Lee --- providers/src/airflow/providers/apache/hive/CHANGELOG.rst | 8 ++++++++ providers/src/airflow/providers/apache/hive/hooks/hive.py | 8 -------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/providers/src/airflow/providers/apache/hive/CHANGELOG.rst b/providers/src/airflow/providers/apache/hive/CHANGELOG.rst index 1213e305cf813..b55a36bd82cb4 100644 --- a/providers/src/airflow/providers/apache/hive/CHANGELOG.rst +++ b/providers/src/airflow/providers/apache/hive/CHANGELOG.rst @@ -26,6 +26,14 @@ Changelog --------- +main +.... + +.. warning:: + All deprecated classes, parameters and features have been removed from the {provider_name} provider package. + The following breaking changes were introduced: + + * Removed deprecated ``GSSAPI`` for ``auth_mechanism.`` Use ``KERBEROS`` instead. 8.2.1 ..... diff --git a/providers/src/airflow/providers/apache/hive/hooks/hive.py b/providers/src/airflow/providers/apache/hive/hooks/hive.py index d768743cd459b..dde421e01e69b 100644 --- a/providers/src/airflow/providers/apache/hive/hooks/hive.py +++ b/providers/src/airflow/providers/apache/hive/hooks/hive.py @@ -873,14 +873,6 @@ def get_conn(self, schema: str | None = None) -> Any: auth_mechanism = db.extra_dejson.get("auth_mechanism", "KERBEROS") kerberos_service_name = db.extra_dejson.get("kerberos_service_name", "hive") - # pyhive uses GSSAPI instead of KERBEROS as a auth_mechanism identifier - if auth_mechanism == "GSSAPI": - self.log.warning( - "Detected deprecated 'GSSAPI' for auth_mechanism for %s. Please use 'KERBEROS' instead", - self.hiveserver2_conn_id, # type: ignore - ) - auth_mechanism = "KERBEROS" - # Password should be set if and only if in LDAP or CUSTOM mode if auth_mechanism in ("LDAP", "CUSTOM"): password = db.password