From 08ac9e63db12018692430ceadc2e7476b51bf41f Mon Sep 17 00:00:00 2001 From: dataengineervishal Date: Sat, 16 Aug 2025 00:09:38 +0530 Subject: [PATCH 1/4] Add support for custom api_endpoint in Google Sheets hook/operator - Added 'api_endpoint' param to 'GSheetsHook' and 'GoogleSheetsCreateSpreadsheetOperator' - Allows using regional or private API endpoints (e.g: private.googleapis.com) - Updated docstrings and unit tests for the new parameter --- providers/google/provider.yaml | 1 + providers/google/pyproject.toml | 6 +++--- .../src/airflow/providers/google/__init__.py | 2 +- .../providers/google/suite/hooks/sheets.py | 16 +++++++++++++- .../google/suite/operators/sheets.py | 5 +++++ .../unit/google/suite/hooks/test_sheets.py | 21 ++++++++++++++++--- 6 files changed, 43 insertions(+), 8 deletions(-) diff --git a/providers/google/provider.yaml b/providers/google/provider.yaml index 734d2433d2417..178dfc04f299f 100644 --- a/providers/google/provider.yaml +++ b/providers/google/provider.yaml @@ -35,6 +35,7 @@ source-date-epoch: 1754503289 # In such case adding >= NEW_VERSION and bumping to NEW_VERSION in a provider have # to be done in the same PR versions: + - 17.1.1 - 17.1.0 - 17.0.0 - 16.1.0 diff --git a/providers/google/pyproject.toml b/providers/google/pyproject.toml index 99afb40615d9e..9a194c07762bc 100644 --- a/providers/google/pyproject.toml +++ b/providers/google/pyproject.toml @@ -25,7 +25,7 @@ build-backend = "flit_core.buildapi" [project] name = "apache-airflow-providers-google" -version = "17.1.0" +version = "17.1.1" description = "Provider package apache-airflow-providers-google for Apache Airflow" readme = "README.rst" authors = [ @@ -259,8 +259,8 @@ apache-airflow-providers-common-sql = {workspace = true} apache-airflow-providers-standard = {workspace = true} [project.urls] -"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-google/17.1.0" -"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-google/17.1.0/changelog.html" +"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-google/17.1.1" +"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-google/17.1.1/changelog.html" "Bug Tracker" = "https://github.com/apache/airflow/issues" "Source Code" = "https://github.com/apache/airflow" "Slack Chat" = "https://s.apache.org/airflow-slack" diff --git a/providers/google/src/airflow/providers/google/__init__.py b/providers/google/src/airflow/providers/google/__init__.py index 040800881b18e..0186d0638e323 100644 --- a/providers/google/src/airflow/providers/google/__init__.py +++ b/providers/google/src/airflow/providers/google/__init__.py @@ -29,7 +29,7 @@ __all__ = ["__version__"] -__version__ = "17.1.0" +__version__ = "17.1.1" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( "2.10.0" diff --git a/providers/google/src/airflow/providers/google/suite/hooks/sheets.py b/providers/google/src/airflow/providers/google/suite/hooks/sheets.py index 88ae6bb0d031a..de09e66a4ac37 100644 --- a/providers/google/src/airflow/providers/google/suite/hooks/sheets.py +++ b/providers/google/src/airflow/providers/google/suite/hooks/sheets.py @@ -22,6 +22,7 @@ from collections.abc import Sequence from typing import Any +from google.api_core.client_options import ClientOptions from googleapiclient.discovery import build from airflow.exceptions import AirflowException @@ -44,6 +45,8 @@ class GSheetsHook(GoogleBaseHook): If set as a sequence, the identities from the list must grant Service Account Token Creator IAM role to the directly preceding identity, with first account from the list granting this role to the originating account. + :param api_endpoint: Optional. Custom API endpoint, i.e: regional or private endpoint. + This can be used to target private VPC or restricted access endpoints. """ def __init__( @@ -51,6 +54,7 @@ def __init__( gcp_conn_id: str = "google_cloud_default", api_version: str = "v4", impersonation_chain: str | Sequence[str] | None = None, + api_endpoint: str | None = None, ) -> None: super().__init__( gcp_conn_id=gcp_conn_id, @@ -58,6 +62,7 @@ def __init__( ) self.gcp_conn_id = gcp_conn_id self.api_version = api_version + self.api_endpoint = api_endpoint self._conn = None def get_conn(self) -> Any: @@ -68,7 +73,16 @@ def get_conn(self) -> Any: """ if not self._conn: http_authorized = self._authorize() - self._conn = build("sheets", self.api_version, http=http_authorized, cache_discovery=False) + client_options = None + if self.api_endpoint: + client_options = ClientOptions(api_endpoint=self.api_endpoint) + self._conn = build( + "sheets", + self.api_version, + http=http_authorized, + cache_discovery=False, + client_options=client_options, + ) return self._conn diff --git a/providers/google/src/airflow/providers/google/suite/operators/sheets.py b/providers/google/src/airflow/providers/google/suite/operators/sheets.py index 2cf942218c675..d19a05a755707 100644 --- a/providers/google/src/airflow/providers/google/suite/operators/sheets.py +++ b/providers/google/src/airflow/providers/google/suite/operators/sheets.py @@ -42,6 +42,8 @@ class GoogleSheetsCreateSpreadsheetOperator(BaseOperator): If set as a sequence, the identities from the list must grant Service Account Token Creator IAM role to the directly preceding identity, with first account from the list granting this role to the originating account (templated). + :param api_endpoint: Optional. Custom API endpoint, e.g: private.googleapis.com. + This can be used to target private VPC or restricted access endpoints. """ template_fields: Sequence[str] = ( @@ -55,17 +57,20 @@ def __init__( spreadsheet: dict[str, Any], gcp_conn_id: str = "google_cloud_default", impersonation_chain: str | Sequence[str] | None = None, + api_endpoint: str | None = None, **kwargs, ) -> None: super().__init__(**kwargs) self.gcp_conn_id = gcp_conn_id self.spreadsheet = spreadsheet self.impersonation_chain = impersonation_chain + self.api_endpoint = api_endpoint def execute(self, context: Any) -> dict[str, Any]: hook = GSheetsHook( gcp_conn_id=self.gcp_conn_id, impersonation_chain=self.impersonation_chain, + api_endpoint=self.api_endpoint, ) spreadsheet = hook.create_spreadsheet(spreadsheet=self.spreadsheet) context["task_instance"].xcom_push(key="spreadsheet_id", value=spreadsheet["spreadsheetId"]) diff --git a/providers/google/tests/unit/google/suite/hooks/test_sheets.py b/providers/google/tests/unit/google/suite/hooks/test_sheets.py index 968dcc2877b40..fe8d8b183583a 100644 --- a/providers/google/tests/unit/google/suite/hooks/test_sheets.py +++ b/providers/google/tests/unit/google/suite/hooks/test_sheets.py @@ -58,11 +58,26 @@ def setup_method(self): @mock.patch("airflow.providers.google.suite.hooks.sheets.build") def test_gsheets_client_creation(self, mock_build, mock_authorize): result = self.hook.get_conn() - mock_build.assert_called_once_with( - "sheets", "v4", http=mock_authorize.return_value, cache_discovery=False - ) + mock_build.assert_called_once() + args, kwargs = mock_build.call_args + assert kwargs["http"] == mock_authorize.return_value + assert kwargs["cache_discovery"] is False assert mock_build.return_value == result + @mock.patch("airflow.providers.google.suite.hooks.sheets.GSheetsHook._authorize") + @mock.patch("airflow.providers.google.suite.hooks.sheets.build") + def test_gsheets_hook_custom_endpoint(self, mock_build, mock_authorize): + self.hook.api_endpoint = "https://private.googleapis.com" + mock_build.assert_called_once() + _, kwargs = mock_build.call_args + client_options = kwargs.get("client_options") + if client_options is None: + api_endpoint = None + else: + api_endpoint = getattr(client_options, "api_endpoint", None) + + assert api_endpoint == "https://private.googleapis.com" + @mock.patch("airflow.providers.google.suite.hooks.sheets.GSheetsHook.get_conn") def test_get_values(self, get_conn): get_method = get_conn.return_value.spreadsheets.return_value.values.return_value.get From f061a56d5391ffa173a6f29790487cff85d16c5e Mon Sep 17 00:00:00 2001 From: dataengineervishal Date: Mon, 18 Aug 2025 01:29:33 +0530 Subject: [PATCH 2/4] Fix gsheet hook test case --- providers/google/tests/unit/google/suite/hooks/test_sheets.py | 1 + 1 file changed, 1 insertion(+) diff --git a/providers/google/tests/unit/google/suite/hooks/test_sheets.py b/providers/google/tests/unit/google/suite/hooks/test_sheets.py index fe8d8b183583a..8cd3dd22ed4c3 100644 --- a/providers/google/tests/unit/google/suite/hooks/test_sheets.py +++ b/providers/google/tests/unit/google/suite/hooks/test_sheets.py @@ -68,6 +68,7 @@ def test_gsheets_client_creation(self, mock_build, mock_authorize): @mock.patch("airflow.providers.google.suite.hooks.sheets.build") def test_gsheets_hook_custom_endpoint(self, mock_build, mock_authorize): self.hook.api_endpoint = "https://private.googleapis.com" + self.hook.get_conn() mock_build.assert_called_once() _, kwargs = mock_build.call_args client_options = kwargs.get("client_options") From d3e7c1471331a84542715bafa724bdbc375eff01 Mon Sep 17 00:00:00 2001 From: dataengineervishal Date: Mon, 18 Aug 2025 13:35:24 +0530 Subject: [PATCH 3/4] Fix version --- dev/breeze/README.md | 2 +- providers/google/provider.yaml | 1 - providers/google/pyproject.toml | 6 +++--- providers/google/src/airflow/providers/google/__init__.py | 2 +- 4 files changed, 5 insertions(+), 6 deletions(-) diff --git a/dev/breeze/README.md b/dev/breeze/README.md index 3eef940fdf620..b2cd297ca3b41 100644 --- a/dev/breeze/README.md +++ b/dev/breeze/README.md @@ -135,6 +135,6 @@ PLEASE DO NOT MODIFY THE HASH BELOW! IT IS AUTOMATICALLY UPDATED BY PREK. --------------------------------------------------------------------------------------------------------- -Package config hash: 4c796051a27cfdb7f49a758f6c306f75e606c6bbe66d0db79c8be74a88fd5636255abe6930cb26af4b48bfbe81b4bed303738070825cf5b456c9bd843c92f03c +Package config hash: 2eb62991633b5245c32453bdb816370494b74f7699a69928a40a034140c228f5113a9e6ab7e3c449908ba9b9c6af954587782baa5ee3b48564296d29549d2006 --------------------------------------------------------------------------------------------------------- diff --git a/providers/google/provider.yaml b/providers/google/provider.yaml index 178dfc04f299f..734d2433d2417 100644 --- a/providers/google/provider.yaml +++ b/providers/google/provider.yaml @@ -35,7 +35,6 @@ source-date-epoch: 1754503289 # In such case adding >= NEW_VERSION and bumping to NEW_VERSION in a provider have # to be done in the same PR versions: - - 17.1.1 - 17.1.0 - 17.0.0 - 16.1.0 diff --git a/providers/google/pyproject.toml b/providers/google/pyproject.toml index 9a194c07762bc..99afb40615d9e 100644 --- a/providers/google/pyproject.toml +++ b/providers/google/pyproject.toml @@ -25,7 +25,7 @@ build-backend = "flit_core.buildapi" [project] name = "apache-airflow-providers-google" -version = "17.1.1" +version = "17.1.0" description = "Provider package apache-airflow-providers-google for Apache Airflow" readme = "README.rst" authors = [ @@ -259,8 +259,8 @@ apache-airflow-providers-common-sql = {workspace = true} apache-airflow-providers-standard = {workspace = true} [project.urls] -"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-google/17.1.1" -"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-google/17.1.1/changelog.html" +"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-google/17.1.0" +"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-google/17.1.0/changelog.html" "Bug Tracker" = "https://github.com/apache/airflow/issues" "Source Code" = "https://github.com/apache/airflow" "Slack Chat" = "https://s.apache.org/airflow-slack" diff --git a/providers/google/src/airflow/providers/google/__init__.py b/providers/google/src/airflow/providers/google/__init__.py index 0186d0638e323..040800881b18e 100644 --- a/providers/google/src/airflow/providers/google/__init__.py +++ b/providers/google/src/airflow/providers/google/__init__.py @@ -29,7 +29,7 @@ __all__ = ["__version__"] -__version__ = "17.1.1" +__version__ = "17.1.0" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( "2.10.0" From 93d99b33a82c48d947c77704c0ddef3022640014 Mon Sep 17 00:00:00 2001 From: Vishal <69051998+dataengineervishal@users.noreply.github.com> Date: Mon, 18 Aug 2025 16:11:42 +0530 Subject: [PATCH 4/4] Update README.md --- dev/breeze/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dev/breeze/README.md b/dev/breeze/README.md index d18c762fe932f..ae1f4e637544d 100644 --- a/dev/breeze/README.md +++ b/dev/breeze/README.md @@ -136,3 +136,5 @@ PLEASE DO NOT MODIFY THE HASH BELOW! IT IS AUTOMATICALLY UPDATED BY PREK. --------------------------------------------------------------------------------------------------------- Package config hash: d7decc43e975d5ece61c85d03ef369732d3c3c82609ad791f694687f6353dd110a37ca9e5aad7b2b99b0a7fcf985bc4a1095421bfa41f5ed102f267bc6f50333 + +---------------------------------------------------------------------------------------------------------