From 237a4b3373bef4e7e2b09689f0c8fdf64d727379 Mon Sep 17 00:00:00 2001 From: Vitalii Vdovenko Date: Wed, 27 Oct 2021 16:07:34 +0300 Subject: [PATCH] Source Google Directory #6265 - update credentials --- .github/workflows/publish-command.yml | 1 + .github/workflows/test-command.yml | 1 + .../source_google_directory/api.py | 6 ++++-- .../source_google_directory/spec.json | 12 +++++++----- docs/integrations/sources/google-directory.md | 4 +++- tools/bin/ci_credentials.sh | 1 + 6 files changed, 17 insertions(+), 8 deletions(-) diff --git a/.github/workflows/publish-command.yml b/.github/workflows/publish-command.yml index c79a899b755f..50da87bbbcf7 100644 --- a/.github/workflows/publish-command.yml +++ b/.github/workflows/publish-command.yml @@ -107,6 +107,7 @@ jobs: GOOGLE_ANALYTICS_V4_TEST_CREDS_OLD: ${{ secrets.GOOGLE_ANALYTICS_V4_TEST_CREDS_OLD }} GOOGLE_CLOUD_STORAGE_TEST_CREDS: ${{ secrets.GOOGLE_CLOUD_STORAGE_TEST_CREDS }} GOOGLE_DIRECTORY_TEST_CREDS: ${{ secrets.GOOGLE_DIRECTORY_TEST_CREDS }} + GOOGLE_DIRECTORY_TEST_CREDS_OAUTH: ${{ secrets.GOOGLE_DIRECTORY_TEST_CREDS_OAUTH }} GOOGLE_SEARCH_CONSOLE_CDK_TEST_CREDS: ${{ secrets.GOOGLE_SEARCH_CONSOLE_CDK_TEST_CREDS }} GOOGLE_SEARCH_CONSOLE_CDK_TEST_CREDS_SRV_ACC: ${{ secrets.GOOGLE_SEARCH_CONSOLE_CDK_TEST_CREDS_SRV_ACC }} GOOGLE_SHEETS_TESTS_CREDS: ${{ secrets.GOOGLE_SHEETS_TESTS_CREDS }} diff --git a/.github/workflows/test-command.yml b/.github/workflows/test-command.yml index 18808933d48f..294640793b3c 100644 --- a/.github/workflows/test-command.yml +++ b/.github/workflows/test-command.yml @@ -102,6 +102,7 @@ jobs: GOOGLE_ANALYTICS_V4_TEST_CREDS_OLD: ${{ secrets.GOOGLE_ANALYTICS_V4_TEST_CREDS_OLD }} GOOGLE_CLOUD_STORAGE_TEST_CREDS: ${{ secrets.GOOGLE_CLOUD_STORAGE_TEST_CREDS }} GOOGLE_DIRECTORY_TEST_CREDS: ${{ secrets.GOOGLE_DIRECTORY_TEST_CREDS }} + GOOGLE_DIRECTORY_TEST_CREDS_OAUTH: ${{ secrets.GOOGLE_DIRECTORY_TEST_CREDS_OAUTH }} GOOGLE_SEARCH_CONSOLE_CDK_TEST_CREDS: ${{ secrets.GOOGLE_SEARCH_CONSOLE_CDK_TEST_CREDS }} GOOGLE_SEARCH_CONSOLE_CDK_TEST_CREDS_SRV_ACC: ${{ secrets.GOOGLE_SEARCH_CONSOLE_CDK_TEST_CREDS_SRV_ACC }} GOOGLE_SHEETS_TESTS_CREDS: ${{ secrets.GOOGLE_SHEETS_TESTS_CREDS }} diff --git a/airbyte-integrations/connectors/source-google-directory/source_google_directory/api.py b/airbyte-integrations/connectors/source-google-directory/source_google_directory/api.py index 78b5ba447ab4..866b44bb206d 100644 --- a/airbyte-integrations/connectors/source-google-directory/source_google_directory/api.py +++ b/airbyte-integrations/connectors/source-google-directory/source_google_directory/api.py @@ -6,14 +6,14 @@ import json from abc import ABC, abstractmethod from functools import partial -from typing import Callable, Dict, Iterator, Sequence, Mapping, Any +from typing import Any, Callable, Dict, Iterator, Mapping, Sequence import backoff +from google.auth.transport.requests import Request from google.oauth2 import service_account from google.oauth2.credentials import Credentials from googleapiclient.discovery import Resource, build from googleapiclient.errors import HttpError as GoogleApiHttpError -from google.auth.transport.requests import Request from .utils import rate_limit_handling @@ -31,6 +31,7 @@ def _load_account_info(credentials_json: str) -> Dict: return account_info def _obtain_service_account_creds(self) -> service_account.Credentials: + """Obtaining creds based on Service account scenario""" credentials_json = self._raw_credentials.get("credentials_json") admin_email = self._raw_credentials.get("email") account_info = self._load_account_info(credentials_json) @@ -38,6 +39,7 @@ def _obtain_service_account_creds(self) -> service_account.Credentials: self._creds = creds.with_subject(admin_email) def _obtain_web_app_creds(self) -> Credentials: + """Obtaining creds based on Web server application scenario""" info = { "client_id": self._raw_credentials.get("client_id"), "client_secret": self._raw_credentials.get("client_secret"), diff --git a/airbyte-integrations/connectors/source-google-directory/source_google_directory/spec.json b/airbyte-integrations/connectors/source-google-directory/source_google_directory/spec.json index 632532bcc43b..42b837a06506 100644 --- a/airbyte-integrations/connectors/source-google-directory/source_google_directory/spec.json +++ b/airbyte-integrations/connectors/source-google-directory/source_google_directory/spec.json @@ -16,7 +16,12 @@ "title": "Web server application", "description": "For these scenario user only needs to give permission to read Google Directory data", "type": "object", - "required": ["client_id", "client_secret", "refresh_token", "scopes"], + "required": [ + "client_id", + "client_secret", + "refresh_token", + "scopes" + ], "properties": { "credentials_title": { "type": "string", @@ -88,10 +93,7 @@ "auth_type": "oauth2.0", "oauth2Specification": { "rootObject": ["credentials", 0], - "oauthFlowInitParameters": [ - ["client_id"], - ["client_secret"] - ], + "oauthFlowInitParameters": [["client_id"], ["client_secret"]], "oauthFlowOutputParameters": [["access_token"], ["refresh_token"]] } } diff --git a/docs/integrations/sources/google-directory.md b/docs/integrations/sources/google-directory.md index b82e6f06f2f3..0641c77e3470 100644 --- a/docs/integrations/sources/google-directory.md +++ b/docs/integrations/sources/google-directory.md @@ -37,7 +37,9 @@ This connector attempts to back off gracefully when it hits Directory API's rate ## Getting started -### Requirements +Google APIs use the OAuth 2.0 protocol for authentication and authorization. The Source supports [Web server application](https://developers.google.com/identity/protocols/oauth2#webserver) and [Service accounts](https://developers.google.com/identity/protocols/oauth2#serviceaccount) scenarios. + +### Requirements Service accounts scenario * Credentials to a Google Service Account with delegated Domain Wide Authority * Email address of the workspace admin which created the Service Account diff --git a/tools/bin/ci_credentials.sh b/tools/bin/ci_credentials.sh index 69147692d89f..6a782e064f2c 100755 --- a/tools/bin/ci_credentials.sh +++ b/tools/bin/ci_credentials.sh @@ -82,6 +82,7 @@ write_standard_creds source-google-analytics-v4 "$GOOGLE_ANALYTICS_V4_TEST_CREDS write_standard_creds source-google-analytics-v4 "$GOOGLE_ANALYTICS_V4_TEST_CREDS_SRV_ACC" "service_config.json" write_standard_creds source-google-analytics-v4 "$GOOGLE_ANALYTICS_V4_TEST_CREDS_OLD" "old_config.json" write_standard_creds source-google-directory "$GOOGLE_DIRECTORY_TEST_CREDS" +write_standard_creds source-google-directory "$GOOGLE_DIRECTORY_TEST_CREDS_OAUTH" "config_oauth.json" write_standard_creds source-google-search-console "$GOOGLE_SEARCH_CONSOLE_CDK_TEST_CREDS" write_standard_creds source-google-search-console "$GOOGLE_SEARCH_CONSOLE_CDK_TEST_CREDS_SRV_ACC" "service_account_config.json" write_standard_creds source-google-sheets "$GOOGLE_SHEETS_TESTS_CREDS"