feat(source-google-sheets): add integration tests (#48835)

Co-authored-by: maxi297 <maxime@airbyte.io>

airbyte-integrations/connectors/source-google-sheets/metadata.yaml

@@ -10,7 +10,7 @@ data:
   connectorSubtype: file
   connectorType: source
   definitionId: 71607ba1-c0ac-4799-8049-7f4b90dd50f7
-  dockerImageTag: 0.7.4
+  dockerImageTag: 0.8.4
   dockerRepository: airbyte/source-google-sheets
   documentationUrl: https://docs.airbyte.com/integrations/sources/google-sheets
   githubIssueLabel: source-google-sheets
@@ -35,8 +35,8 @@ data:
   connectorTestSuitesOptions:
     - suite: liveTests
       testConnections:
-        - name: google-sheets_service_config_dev_null
-          id: 758d197a-864f-45d3-9d5d-0a52fd9c0a22
+        - name: temp-google-sheets_service_config_dev_null
+          id: 035f390f-591f-4bb5-b6d5-52145627befb
     - suite: unitTests
     - suite: acceptanceTests
       testSecrets:

airbyte-integrations/connectors/source-google-sheets/pyproject.toml

@@ -3,7 +3,7 @@ requires = [ "poetry-core>=1.0.0",]
 build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
-version = "0.7.4"
+version = "0.8.4"
 name = "source-google-sheets"
 description = "Source implementation for Google Sheets."
 authors = [ "Airbyte <contact@airbyte.io>",]

airbyte-integrations/connectors/source-google-sheets/source_google_sheets/client.py

@@ -33,24 +33,16 @@ def give_up(error):
     def __init__(self, credentials: Dict[str, str], scopes: List[str] = SCOPES):
         self.client = Helpers.get_authenticated_sheets_client(credentials, scopes)
 
-    def _create_range(self, sheet, row_cursor):
-        range = f"{sheet}!{row_cursor}:{row_cursor + self.Backoff.row_batch_size}"
-        return range
-
     @backoff.on_exception(backoff.expo, errors.HttpError, max_time=120, giveup=Backoff.give_up, on_backoff=Backoff.increase_row_batch_size)
     def get(self, **kwargs):
         return self.client.get(**kwargs).execute()
 
-    @backoff.on_exception(backoff.expo, errors.HttpError, max_time=120, giveup=Backoff.give_up, on_backoff=Backoff.increase_row_batch_size)
-    def create(self, **kwargs):
-        return self.client.create(**kwargs).execute()
-
     @backoff.on_exception(backoff.expo, errors.HttpError, max_time=120, giveup=Backoff.give_up, on_backoff=Backoff.increase_row_batch_size)
     def get_values(self, **kwargs):
         range = self._create_range(kwargs.pop("sheet"), kwargs.pop("row_cursor"))
         logger.info(f"Fetching range {range}")
         return self.client.values().batchGet(ranges=range, **kwargs).execute()
 
-    @backoff.on_exception(backoff.expo, errors.HttpError, max_time=120, giveup=Backoff.give_up, on_backoff=Backoff.increase_row_batch_size)
-    def update_values(self, **kwargs):
-        return self.client.values().batchUpdate(**kwargs).execute()
+    def _create_range(self, sheet, row_cursor):
+        range = f"{sheet}!{row_cursor}:{row_cursor + self.Backoff.row_batch_size}"
+        return range

airbyte-integrations/connectors/source-google-sheets/unit_tests/integration/custom_http_mocker.py

@@ -0,0 +1,73 @@
+# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
+
+
+from functools import wraps
+from typing import Dict
+from unittest.mock import patch
+from urllib.parse import parse_qsl, unquote, urlencode, urlunparse
+
+from airbyte_cdk.test.mock_http import HttpResponse
+from airbyte_cdk.test.mock_http.request import HttpRequest
+from httplib2 import Response
+
+
+def parse_and_transform(parse_result_str: str):
+    """
+    Parse the input string representation of a HttpRequest transform it into the URL.
+    """
+    parse_result_part = parse_result_str.split("ParseResult(", 1)[1].split(")", 1)[0]
+
+    # Convert the ParseResult string into a dictionary
+    components = eval(f"dict({parse_result_part})")
+
+    url = urlunparse((
+        components["scheme"],
+        components["netloc"],
+        components["path"],
+        components["params"],
+        components["query"],
+        components["fragment"],
+    ))
+
+    return url
+
+
+class CustomHttpMocker:
+    """
+    This is a limited mocker for usage with httplib2.Http.request
+    It has a similar interface to airbyte HttpMocker such than when we move this connector to low-code only with
+    a http retriever we will be able to substitute CustomHttpMocker => HttpMocker in out integration testing with minimal changes.
+
+    Note: there is only support for get and post method and url matching ignoring the body but this is enough for the current test set.
+    """
+    requests_mapper: Dict = {}
+
+    def post(self, request: HttpRequest, response: HttpResponse):
+        custom_response = (Response({"status": response.status_code}), response.body.encode("utf-8"))
+        uri = parse_and_transform(str(request))
+        decoded_url = unquote(uri)
+        self.requests_mapper[("POST", decoded_url)] = custom_response
+
+    def get(self, request: HttpRequest, response: HttpResponse):
+        custom_response = (Response({"status": response.status_code}), response.body.encode("utf-8"))
+        uri = parse_and_transform(str(request))
+        decoded_url = unquote(uri)
+        self.requests_mapper[("GET", decoded_url)] = custom_response
+
+    def mock_request(self, uri, method="GET", body=None, headers=None, **kwargs):
+        decoded_url = unquote(uri)
+        mocked_response = self.requests_mapper.get((method, decoded_url))
+        if not mocked_response:
+            raise Exception(f"Mock response not found {uri} {method}")
+        return mocked_response
+
+    # trying to type that using callables provides the error `incompatible with return type "_F" in supertype "ContextDecorator"`
+    def __call__(self, test_func): # type: ignore
+        @wraps(test_func)
+        def wrapper(*args, **kwargs): # type: ignore  # this is a very generic wrapper that does not need to be typed
+            kwargs["http_mocker"] = self
+
+            with patch("httplib2.Http.request", side_effect=self.mock_request):
+                return test_func(*args, **kwargs)
+
+        return wrapper

airbyte-integrations/connectors/source-google-sheets/unit_tests/integration/request_builder.py

@@ -0,0 +1,72 @@
+# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
+
+from __future__ import annotations
+
+from airbyte_cdk.test.mock_http.request import HttpRequest
+
+#  todo: this should be picked from manifest in the future
+GOOGLE_SHEETS_BASE_URL = "https://sheets.googleapis.com/v4/spreadsheets"
+OAUTH_AUTHORIZATION_ENDPOINT = "https://oauth2.googleapis.com"
+
+class RequestBuilder:
+    @classmethod
+    def get_account_endpoint(cls) -> RequestBuilder:
+        return cls(resource="values:batchGet")
+
+
+    def __init__(self, resource:str=None) -> None:
+        self._spreadsheet_id = None
+        self._query_params = {}
+        self._body = None
+        self.resource = resource
+
+    def with_include_grid_data(self, include_grid_data: bool) -> RequestBuilder:
+        self._query_params["includeGridData"] = "true" if include_grid_data else "false"
+        return self
+
+    def with_alt(self, alt: str) -> RequestBuilder:
+        self._query_params["alt"] = alt
+        return self
+
+    def with_ranges(self, ranges: str) -> RequestBuilder:
+        self._query_params["ranges"] = ranges
+        return self
+
+    def with_major_dimension(self, dimension: str) -> RequestBuilder:
+        self._query_params["majorDimension"] = dimension
+        return self
+
+    def with_spreadsheet_id(self, spreadsheet_id: str) -> RequestBuilder:
+        self._spreadsheet_id = spreadsheet_id
+        return self
+
+    def build(self) -> HttpRequest:
+        endpoint = f"/{self.resource}" if self.resource else ""
+        return HttpRequest(
+            url=f"{GOOGLE_SHEETS_BASE_URL}/{self._spreadsheet_id}{endpoint}",
+            query_params=self._query_params,
+            body=self._body,
+        )
+
+
+class AuthBuilder:
+    @classmethod
+    def get_token_endpoint(cls) -> AuthBuilder:
+        return cls(resource="token")
+
+    def __init__(self, resource):
+        self._body = ""
+        self._resource = resource
+        self._query_params = ""
+
+    def with_body(self, body: str):
+        self._body = body
+        return self
+
+    def build(self) -> HttpRequest:
+        endpoint = f"/{self._resource}" if self._resource else ""
+        return HttpRequest(
+            url=f"{OAUTH_AUTHORIZATION_ENDPOINT}{endpoint}",
+            query_params=self._query_params,
+            body=self._body,
+        )

airbyte-integrations/connectors/source-google-sheets/unit_tests/integration/test_credentials.py

@@ -0,0 +1,63 @@
+# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
+import json
+
+# this key was generated with rsa library from cryptography
+test_private_key = """
+-----BEGIN PRIVATE KEY-----
+MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCjM57/r+LuNv9z
+Bbe7+sMCJJm6t1MafRfxYVEz9c52/mLa5iEnpJfE+NRFgyp8hE8t493Btt/bJk94
+2bMthZqMh2n9dIJZOUYBzd5MHLOc6vCUHFZT9fzHX/yTrz/bxa31BQs6c1p2HOiY
+Kr3r0Dyj5jXsKo0mgt+iVKGgcSZ6NCzMRL8N9M++13k2RUPwaGksuyNqzAqgNhHd
+wQcuS42AEFEOT1sfa4xG5mMY6+tPKDP92/ISHNUMD9NpzIA8A+tFX/w/L5VQKR3r
+fTfrSTtkG6qF+3ARdeKqpxrW4ZPHuzNH8Y2I1uBuVaDvmZMvi+BLKgwwhWuEGjB1
+j6Tv4TgXAgMBAAECggEAJTXLXlPdg1/hzXlzw3XwyYfLz0EmPwdfkqcUKysz2Hi2
+1F8dFxtViVEMoQ6/fKV0Iivur1DBaIerHgxQ6KOqMblcRrAuWiaPWjD0qtjucOw2
+TybI3hrbeB/gCFIwVq0TNSbhwQF1EjIULEGujNotQVdnWwH2rd2wHKR8N4ck9T6b
+SKz8+u21RY2cBprneS6wxh+dvba8+7cpHn4cB+TB6UMeUow01LF4ye+hYVDNx6j9
+VcdWXlH9fCy/GUTF4um+ABunlMCm6D5DAUVeiugd+ChSKzqOlV/H17EK1MF4HAjh
+Alo2FJrKd8/ZwX1Xmngi9Y2Dlggmfiw4HzeNZNFFPQKBgQDmeQxDBvZapjdJntOM
+DccoQGJyZMznd277xefKTfZLetcWWtgantW7IAxEEbwZOfrQFnBsNIlnreGPoZ7n
+DL2jv/oVeEcr29FxlbDR2R1/h8Mp7d41Qi1Sd9RAhGcVtYZhLCCMMxt4DN7/v77M
+2lc3B0NhgC3kxOJCC8kN2gqSTQKBgQC1RyENp1UTOSUbFsx1WbAzy2K974CRnL54
+uK7efoLQ7fLh6OsRlBoy60aUpHv2tCb5ac+xdMELoAQu3PQKJMDJITstwgM4p6AP
+x32lAHzOYnhG9/3P7kc29OaY8tlkDAn0ckv05DLAGLbKAemcGKW3/u08EITQtdW7
+dEH75Ow98wKBgQDb5tV3QrZeKcgI251XPXIwCrakFV+Y3tErM1qFIbwFqtB8yPL2
++2RM5jgt3ooNu89/KlncNIiCP1s/k2Mta2+qRStVvuyRgWympsAOic1meGATqp1h
+TaI21JTVdj9xbEEqiFMJ0l28PvOrLAXeKdobbDezWPzxEZYclGgiak+55QKBgBVU
+6W7R4hEBCHzHkge9Jh7yMAxpwpdf+on6MZm9CWfMmGg9IGxRIUQcq5GSSYQebveq
+m+Yl9xGHIvbgyVboPEduwagAzKA+GXfB4ecox4cBz2WKiTOOtpKg/wHAkhRT1lgN
+myKWN+KjBd9/mh3kSJv+Q6xtxTNKMnx8kccyiRpBAoGBAJV9AAXj4icaDiPKoQw5
+UERTGuVoEpWbc3yi/PXJ99fQxHZIHQa7a7VyyTHsDplqWu/qfHFHj+IJops8+l1F
+U7PQBfXvIpubb55EhNCaID1VaRauGjW2x8PGA/27KQ3mB1uxEZUO8crcDYvPsZJf
+jHfASOY3OsGgYW95pkyx5TH7
+-----END PRIVATE KEY-----
+
+"""
+
+service_account_info = {
+    "type": "service_account",
+    "project_id": "test-project-id",
+    "private_key_id": "test-private-key-id",
+    "private_key": test_private_key,
+    "client_email": "test-service-account@test-project.iam.gserviceaccount.com",
+    "client_id": "1234567890",
+    "auth_uri": "https://accounts.google.com/o/oauth2/auth",
+    "token_uri": "https://oauth2.googleapis.com/token",
+    "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
+    "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/test-service-account%40test-project.iam.gserviceaccount.com",
+}
+
+service_account_info_encoded = json.dumps(service_account_info).encode("utf-8")
+
+service_account_credentials = {
+    "auth_type": "Service",
+    "service_account_info": service_account_info_encoded,
+}
+
+
+oauth_credentials = {
+    "auth_type": "Client",
+    "client_id": "43987534895734985.apps.googleusercontent.com",
+    "client_secret": "2347586435987643598",
+    "refresh_token": "1//4398574389537495437983457985437"
+  }