🎉 Source Google Directory: support oauth

.github/workflows/publish-command.yml

@@ -107,6 +107,7 @@ jobs:
           GOOGLE_ANALYTICS_V4_TEST_CREDS_OLD: ${{ secrets.GOOGLE_ANALYTICS_V4_TEST_CREDS_OLD }}
           GOOGLE_CLOUD_STORAGE_TEST_CREDS: ${{ secrets.GOOGLE_CLOUD_STORAGE_TEST_CREDS }}
           GOOGLE_DIRECTORY_TEST_CREDS: ${{ secrets.GOOGLE_DIRECTORY_TEST_CREDS }}
+          GOOGLE_DIRECTORY_TEST_CREDS_OAUTH: ${{ secrets.GOOGLE_DIRECTORY_TEST_CREDS_OAUTH }}
           GOOGLE_SEARCH_CONSOLE_CDK_TEST_CREDS: ${{ secrets.GOOGLE_SEARCH_CONSOLE_CDK_TEST_CREDS }}
           GOOGLE_SEARCH_CONSOLE_CDK_TEST_CREDS_SRV_ACC: ${{ secrets.GOOGLE_SEARCH_CONSOLE_CDK_TEST_CREDS_SRV_ACC }}
           GOOGLE_SHEETS_TESTS_CREDS: ${{ secrets.GOOGLE_SHEETS_TESTS_CREDS }}

.github/workflows/test-command.yml

@@ -102,6 +102,7 @@ jobs:
           GOOGLE_ANALYTICS_V4_TEST_CREDS_OLD: ${{ secrets.GOOGLE_ANALYTICS_V4_TEST_CREDS_OLD }}
           GOOGLE_CLOUD_STORAGE_TEST_CREDS: ${{ secrets.GOOGLE_CLOUD_STORAGE_TEST_CREDS }}
           GOOGLE_DIRECTORY_TEST_CREDS: ${{ secrets.GOOGLE_DIRECTORY_TEST_CREDS }}
+          GOOGLE_DIRECTORY_TEST_CREDS_OAUTH: ${{ secrets.GOOGLE_DIRECTORY_TEST_CREDS_OAUTH }}
           GOOGLE_SEARCH_CONSOLE_CDK_TEST_CREDS: ${{ secrets.GOOGLE_SEARCH_CONSOLE_CDK_TEST_CREDS }}
           GOOGLE_SEARCH_CONSOLE_CDK_TEST_CREDS_SRV_ACC: ${{ secrets.GOOGLE_SEARCH_CONSOLE_CDK_TEST_CREDS_SRV_ACC }}
           GOOGLE_SHEETS_TESTS_CREDS: ${{ secrets.GOOGLE_SHEETS_TESTS_CREDS }}

airbyte-integrations/connectors/source-google-directory/source_google_directory/api.py

@@ -6,10 +6,12 @@
 import json
 from abc import ABC, abstractmethod
 from functools import partial
-from typing import Callable, Dict, Iterator, Sequence
+from typing import Any, Callable, Dict, Iterator, Mapping, Sequence
 
 import backoff
+from google.auth.transport.requests import Request
 from google.oauth2 import service_account
+from google.oauth2.credentials import Credentials
 from googleapiclient.discovery import Resource, build
 from googleapiclient.errors import HttpError as GoogleApiHttpError
 
@@ -19,19 +21,40 @@
 
 
 class API:
-    def __init__(self, credentials_json: str, email: str):
+    def __init__(self, credentials: Mapping[str, Any]):
         self._creds = None
-        self._credentials_json = credentials_json
-        self._admin_email = email
+        self._raw_credentials = credentials
 
-    def _load_account_info(self) -> Dict:
-        account_info = json.loads(self._credentials_json)
+    @staticmethod
+    def _load_account_info(credentials_json: str) -> Dict:
+        account_info = json.loads(credentials_json)
         return account_info
 
-    def _obtain_creds(self) -> service_account.Credentials:
-        account_info = self._load_account_info()
+    def _obtain_service_account_creds(self) -> service_account.Credentials:
+        """Obtaining creds based on Service account scenario"""
+        credentials_json = self._raw_credentials.get("credentials_json")
+        admin_email = self._raw_credentials.get("email")
+        account_info = self._load_account_info(credentials_json)
         creds = service_account.Credentials.from_service_account_info(account_info, scopes=SCOPES)
-        self._creds = creds.with_subject(self._admin_email)
+        self._creds = creds.with_subject(admin_email)
+
+    def _obtain_web_app_creds(self) -> Credentials:
+        """Obtaining creds based on Web server application scenario"""
+        info = {
+            "client_id": self._raw_credentials.get("client_id"),
+            "client_secret": self._raw_credentials.get("client_secret"),
+            "refresh_token": self._raw_credentials.get("refresh_token"),
+        }
+        creds = Credentials.from_authorized_user_info(info)
+        if creds.expired:
+            creds.refresh(Request())
+        self._creds = creds
+
+    def _obtain_creds(self):
+        if "credentials_json" in self._raw_credentials:
+            self._obtain_service_account_creds()
+        elif "client_id" and "client_secret" in self._raw_credentials:
+            self._obtain_web_app_creds()
 
     def _construct_resource(self) -> Resource:
         if not self._creds:

airbyte-integrations/connectors/source-google-directory/source_google_directory/client.py

@@ -11,8 +11,11 @@
 
 
 class Client(BaseClient):
-    def __init__(self, credentials_json: str, email: str):
-        self._api = API(credentials_json, email)
+    def __init__(self, credentials: Mapping[str, Any] = None, credentials_json: str = None, email: str = None):
+        # supporting old config format
+        if not credentials:
+            credentials = {"credentials_json": credentials_json, "email": email}
+        self._api = API(credentials)
         self._apis = {"users": UsersAPI(self._api), "groups": GroupsAPI(self._api), "group_members": GroupMembersAPI(self._api)}
         super().__init__()
 

airbyte-integrations/connectors/source-google-directory/source_google_directory/spec.json

@@ -4,18 +4,87 @@
     "$schema": "http://json-schema.org/draft-07/schema#",
     "title": "Google Directory Spec",
     "type": "object",
-    "required": ["credentials_json", "email"],
-    "additionalProperties": false,
+    "required": [],
+    "additionalProperties": true,
     "properties": {
-      "credentials_json": {
-        "type": "string",
-        "description": "The contents of the JSON service account key. See the <a href=\"https://developers.google.com/admin-sdk/directory/v1/guides/delegation\">docs</a> for more information on how to generate this key.",
-        "airbyte_secret": true
-      },
-      "email": {
-        "type": "string",
-        "description": "The email of the user, which has permissions to access the Google Workspace Admin APIs."
+      "credentials": {
+        "title": "Google Credentials",
+        "description": "Google APIs use the OAuth 2.0 protocol for authentication and authorization. The Source supports <a href=\"https://developers.google.com/identity/protocols/oauth2#webserver\" target=\"_blank\">Web server application</a> and <a href=\"https://developers.google.com/identity/protocols/oauth2#serviceaccount\" target=\"_blank\">Service accounts</a> scenarios",
+        "type": "object",
+        "oneOf": [
+          {
+            "title": "Web server application",
+            "description": "For these scenario user only needs to give permission to read Google Directory data",
+            "type": "object",
+            "required": ["client_id", "client_secret", "refresh_token"],
+            "properties": {
+              "credentials_title": {
+                "type": "string",
+                "title": "Credentials title",
+                "description": "Authentication scenario",
+                "const": "Web server app",
+                "enum": ["Web server app"],
+                "default": "Web server app",
+                "order": 0
+              },
+              "client_id": {
+                "title": "Client ID",
+                "type": "string",
+                "description": "The client ID of developer application",
+                "airbyte_secret": true
+              },
+              "client_secret": {
+                "title": "Client secret",
+                "type": "string",
+                "description": "The client secret of developer application",
+                "airbyte_secret": true
+              },
+              "refresh_token": {
+                "title": "Refresh Token",
+                "type": "string",
+                "description": "The token for obtaining new access token",
+                "airbyte_secret": true
+              }
+            }
+          },
+          {
+            "title": "Service accounts",
+            "description": "For these scenario user should obtain service account's credentials from the Google API Console and provide delegated email",
+            "type": "object",
+            "required": ["credentials_json", "email"],
+            "properties": {
+              "credentials_title": {
+                "type": "string",
+                "title": "Credentials title",
+                "description": "Authentication scenario",
+                "const": "Service accounts",
+                "enum": ["Service accounts"],
+                "default": "Service accounts",
+                "order": 0
+              },
+              "credentials_json": {
+                "type": "string",
+                "title": "Credentials JSON",
+                "description": "The contents of the JSON service account key. See the <a href=\"https://developers.google.com/admin-sdk/directory/v1/guides/delegation\">docs</a> for more information on how to generate this key.",
+                "airbyte_secret": true
+              },
+              "email": {
+                "type": "string",
+                "title": "Email",
+                "description": "The email of the user, which has permissions to access the Google Workspace Admin APIs."
+              }
+            }
+          }
+        ]
       }
     }
+  },
+  "authSpecification": {
+    "auth_type": "oauth2.0",
+    "oauth2Specification": {
+      "rootObject": ["credentials", 0],
+      "oauthFlowInitParameters": [["client_id"], ["client_secret"]],
+      "oauthFlowOutputParameters": [["refresh_token"]]
+    }
   }
 }

docs/integrations/sources/google-directory.md

@@ -37,7 +37,9 @@ This connector attempts to back off gracefully when it hits Directory API's rate
 
 ## Getting started
 
-### Requirements
+Google APIs use the OAuth 2.0 protocol for authentication and authorization. The Source supports [Web server application](https://developers.google.com/identity/protocols/oauth2#webserver) and [Service accounts](https://developers.google.com/identity/protocols/oauth2#serviceaccount) scenarios.
+
+### Requirements Service accounts scenario
 
 * Credentials to a Google Service Account with delegated Domain Wide Authority
 * Email address of the workspace admin which created the Service Account

tools/bin/ci_credentials.sh

@@ -82,6 +82,7 @@ write_standard_creds source-google-analytics-v4 "$GOOGLE_ANALYTICS_V4_TEST_CREDS
 write_standard_creds source-google-analytics-v4 "$GOOGLE_ANALYTICS_V4_TEST_CREDS_SRV_ACC" "service_config.json"
 write_standard_creds source-google-analytics-v4 "$GOOGLE_ANALYTICS_V4_TEST_CREDS_OLD" "old_config.json"
 write_standard_creds source-google-directory "$GOOGLE_DIRECTORY_TEST_CREDS"
+write_standard_creds source-google-directory "$GOOGLE_DIRECTORY_TEST_CREDS_OAUTH" "config_oauth.json"
 write_standard_creds source-google-search-console "$GOOGLE_SEARCH_CONSOLE_CDK_TEST_CREDS"
 write_standard_creds source-google-search-console "$GOOGLE_SEARCH_CONSOLE_CDK_TEST_CREDS_SRV_ACC" "service_account_config.json"
 write_standard_creds source-google-sheets "$GOOGLE_SHEETS_TESTS_CREDS"