From 3cb0ccb0d5b0bfc384dfee68a19e2e9d3c425a7c Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Fri, 2 Jun 2023 18:09:09 +0200 Subject: [PATCH 01/10] wip --- .../declarative_component_schema.yaml | 46 +++++++++++++++++++ .../models/declarative_component_schema.py | 37 +++++++++++++++ .../parsers/model_to_component_factory.py | 23 +++++++++- .../http/requests_native_auth/oauth.py | 39 ++++++++++------ .../source-xero/source_xero/source.py | 4 +- 5 files changed, 131 insertions(+), 18 deletions(-) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/declarative_component_schema.yaml b/airbyte-cdk/python/airbyte_cdk/sources/declarative/declarative_component_schema.yaml index e02533107d5c..f2df0a180047 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/declarative_component_schema.yaml +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/declarative_component_schema.yaml @@ -735,6 +735,52 @@ definitions: type: string examples: - "%Y-%m-%d %H:%M:%S.%f+00:00" + refresh_token_updater: + title: Token Updater + description: When the token updater is defined, new refresh tokens, access tokens and the access token expiry date are written back to the config object. This is important if the refresh token can only used once. + properties: + refresh_token_name: + title: Refresh Token Response Field Name + type: string + default: "refresh_token" + examples: + - "refresh_token" + access_token_config_path: + title: Config Path To Access Token + description: Config path to the access token. + type: array + items: + type: string + default: ["credentials", "access_token"] + examples: + - ["credentials", "access_token"] + - ["access_token"] + refresh_token_config_path: + title: Config Path To Refresh Token + description: Config path to the access token. + type: array + items: + type: string + default: ["credentials", "refresh_token"] + examples: + - ["credentials", "refresh_token"] + - ["refresh_token"] + token_expiry_date_config_path: + title: Config Path To Expiry Date + description: Config path to the expiry date. + type: array + items: + type: string + default: ["credentials", "token_expiry_date"] + examples: + - ["credentials", "token_expiry_date"] + access_token_name: + title: Access Token Response Field Name + description: The name of the field to extract the access token from in the token refresh response. + type: string + default: "access_token" + examples: + - "access_token" $parameters: type: object additionalProperties: true diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/models/declarative_component_schema.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/models/declarative_component_schema.py index e694981fb97f..57b1a2ff6fee 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/models/declarative_component_schema.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/models/declarative_component_schema.py @@ -260,6 +260,38 @@ class Config: parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") +class RefreshTokenUpdater(BaseModel): + refresh_token_name: Optional[str] = Field( + "refresh_token", + examples=["refresh_token"], + title="Refresh Token Response Field Name", + ) + access_token_config_path: Optional[List[str]] = Field( + ["credentials", "access_token"], + description="Config path to the access token.", + examples=[["credentials", "access_token"], ["access_token"]], + title="Config Path To Access Token", + ) + refresh_token_config_path: Optional[List[str]] = Field( + ["credentials", "refresh_token"], + description="Config path to the access token.", + examples=[["credentials", "refresh_token"], ["refresh_token"]], + title="Config Path To Refresh Token", + ) + token_expiry_date_config_path: Optional[List[str]] = Field( + ["credentials", "token_expiry_date"], + description="Config path to the expiry date.", + examples=[["credentials", "token_expiry_date"]], + title="Config Path To Expiry Date", + ) + access_token_name: Optional[str] = Field( + "access_token", + description="The name of the field to extract the access token from in the token refresh response.", + examples=["access_token"], + title="Access Token Response Field Name", + ) + + class OAuthAuthenticator(BaseModel): type: Literal["OAuthAuthenticator"] client_id: str = Field( @@ -340,6 +372,11 @@ class OAuthAuthenticator(BaseModel): examples=["%Y-%m-%d %H:%M:%S.%f+00:00"], title="Token Expiry Date Format", ) + refresh_token_updater: Optional[RefreshTokenUpdater] = Field( + None, + description="When the token updater is defined, new refresh tokens, access tokens and the access token expiry date are written back to the config object. This is important if the refresh token can only used once.", + title="Token Updater", + ) parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py index 02e84c555e72..26af76805d6b 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py @@ -7,6 +7,7 @@ import importlib import inspect import re +import dpath from typing import Any, Callable, List, Literal, Mapping, Optional, Type, Union, get_args, get_origin, get_type_hints from airbyte_cdk.sources.declarative.auth import DeclarativeOauth2Authenticator @@ -24,6 +25,7 @@ from airbyte_cdk.sources.declarative.extractors import DpathExtractor, RecordFilter, RecordSelector from airbyte_cdk.sources.declarative.incremental import DatetimeBasedCursor from airbyte_cdk.sources.declarative.interpolation import InterpolatedString +from airbyte_cdk.sources.declarative.interpolation.interpolated_mapping import InterpolatedMapping from airbyte_cdk.sources.declarative.models.declarative_component_schema import AddedFieldDefinition as AddedFieldDefinitionModel from airbyte_cdk.sources.declarative.models.declarative_component_schema import AddFields as AddFieldsModel from airbyte_cdk.sources.declarative.models.declarative_component_schema import ApiKeyAuthenticator as ApiKeyAuthenticatorModel @@ -659,6 +661,23 @@ def create_no_pagination(model: NoPaginationModel, config: Config, **kwargs) -> @staticmethod def create_oauth_authenticator(model: OAuthAuthenticatorModel, config: Config, **kwargs) -> DeclarativeOauth2Authenticator: + if model.refresh_token_updater: + return SingleUseRefreshTokenOauth2Authenticator( + config, + InterpolatedString.create(model.token_refresh_endpoint, parameters=model.parameters).eval(config), + access_token_name=InterpolatedString.create(model.access_token_name, parameters=model.parameters).eval(config), + refresh_token_name=model.refresh_token_updater.refresh_token_name, + expires_in_name=InterpolatedString.create(model.expires_in_name, parameters=model.parameters).eval(config), + client_id=InterpolatedString.create(model.client_id, parameters=model.parameters).eval(config), + client_secret=InterpolatedString.create(model.client_secret, parameters=model.parameters).eval(config), + access_token_config_path=model.refresh_token_updater.access_token_config_path, + refresh_token_config_path=model.refresh_token_updater.refresh_token_config_path, + token_expiry_date_config_path=model.refresh_token_updater.token_expiry_date_config_path, + grant_type=InterpolatedString.create(model.grant_type, parameters=model.parameters).eval(config), + refresh_request_body=InterpolatedMapping(model.refresh_request_body or {}, parameters=model.parameters).eval(config), + scopes=model.scopes, + token_expiry_date_format=model.token_expiry_date_format, + ) return DeclarativeOauth2Authenticator( access_token_name=model.access_token_name, client_id=model.client_id, @@ -685,8 +704,8 @@ def create_single_use_refresh_token_oauth_authenticator( access_token_name=model.access_token_name, refresh_token_name=model.refresh_token_name, expires_in_name=model.expires_in_name, - client_id_config_path=model.client_id_config_path, - client_secret_config_path=model.client_secret_config_path, + client_id=dpath.util.get(config, model.client_id_config_path), + client_secret=dpath.util.get(config, model.client_id_config_path), access_token_config_path=model.access_token_config_path, refresh_token_config_path=model.refresh_token_config_path, token_expiry_date_config_path=model.token_expiry_date_config_path, diff --git a/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py b/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py index b0262a94dc97..917c37c33209 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py @@ -2,7 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -from typing import Any, List, Mapping, Sequence, Tuple, Union +from typing import Any, List, Mapping, Optional, Sequence, Tuple, Union import dpath import pendulum @@ -109,11 +109,12 @@ def __init__( refresh_token_name: str = "refresh_token", refresh_request_body: Mapping[str, Any] = None, grant_type: str = "refresh_token", - client_id_config_path: Sequence[str] = ("credentials", "client_id"), - client_secret_config_path: Sequence[str] = ("credentials", "client_secret"), + client_id: Optional[str] = None, + client_secret: Optional[str] = None, access_token_config_path: Sequence[str] = ("credentials", "access_token"), refresh_token_config_path: Sequence[str] = ("credentials", "refresh_token"), token_expiry_date_config_path: Sequence[str] = ("credentials", "token_expiry_date"), + token_expiry_date_format: str = None, ): """ @@ -132,11 +133,14 @@ def __init__( refresh_token_config_path (Sequence[str]): Dpath to the refresh_token field in the connector configuration. Defaults to ("credentials", "refresh_token"). token_expiry_date_config_path (Sequence[str]): Dpath to the token_expiry_date field in the connector configuration. Defaults to ("credentials", "token_expiry_date"). """ - self._client_id_config_path = client_id_config_path - self._client_secret_config_path = client_secret_config_path + self._client_id = client_id if client_id is not None else dpath.util.get(connector_config, ("credentials", "client_id")) + self._client_secret = ( + client_secret if client_secret is not None else dpath.util.get(connector_config, ("credentials", "client_secret")) + ) self._access_token_config_path = access_token_config_path self._refresh_token_config_path = refresh_token_config_path self._token_expiry_date_config_path = token_expiry_date_config_path + self._token_expiry_date_format = token_expiry_date_format self._refresh_token_name = refresh_token_name self._connector_config = connector_config self._validate_connector_config() @@ -166,8 +170,6 @@ def _validate_connector_config(self): f"This authenticator expects a value under the {self._access_token_config_path} field path. Please check your configuration structure or change the access_token_config_path value at initialization of this authenticator." ) for field_path, getter, parameter_name in [ - (self._client_id_config_path, self.get_client_id, "client_id_config_path"), - (self._client_secret_config_path, self.get_client_secret, "client_secret_config_path"), (self._refresh_token_config_path, self.get_refresh_token, "refresh_token_config_path"), (self._token_expiry_date_config_path, self.get_token_expiry_date, "token_expiry_date_config_path"), ]: @@ -182,10 +184,10 @@ def get_refresh_token_name(self) -> str: return self._refresh_token_name def get_client_id(self) -> str: - return dpath.util.get(self._connector_config, self._client_id_config_path) + return self._client_id def get_client_secret(self) -> str: - return dpath.util.get(self._connector_config, self._client_secret_config_path) + return self._client_secret @property def access_token(self) -> str: @@ -212,8 +214,11 @@ def token_has_expired(self) -> bool: return pendulum.now("UTC") > self.get_token_expiry_date() @staticmethod - def get_new_token_expiry_date(access_token_expires_in: int): - return pendulum.now("UTC").add(seconds=access_token_expires_in) + def get_new_token_expiry_date(access_token_expires_in: str, token_expiry_date_format: str = None) -> pendulum.DateTime: + if token_expiry_date_format: + return pendulum.from_format(access_token_expires_in, token_expiry_date_format) + else: + return pendulum.now("UTC").add(seconds=int(access_token_expires_in)) def get_access_token(self) -> str: """Retrieve new access and refresh token if the access token has expired. @@ -222,18 +227,24 @@ def get_access_token(self) -> str: str: The current access_token, updated if it was previously expired. """ if self.token_has_expired(): + with open('/Users/joereuter/Clones/airbyte/debug.txt', 'a') as file: file.write('get a new refresh token\n') new_access_token, access_token_expires_in, new_refresh_token = self.refresh_access_token() - new_token_expiry_date = self.get_new_token_expiry_date(access_token_expires_in) + new_token_expiry_date = self.get_new_token_expiry_date(access_token_expires_in, self._token_expiry_date_format) self.access_token = new_access_token self.set_refresh_token(new_refresh_token) self.set_token_expiry_date(new_token_expiry_date) + with open('/Users/joereuter/Clones/airbyte/debug.txt', 'a') as file: file.write(f'new refresh token: {new_refresh_token}\n') + with open('/Users/joereuter/Clones/airbyte/debug.txt', 'a') as file: file.write(f'new access token: {new_access_token}\n') + with open('/Users/joereuter/Clones/airbyte/debug.txt', 'a') as file: file.write(f'new expiry date: {new_token_expiry_date}\n') emit_configuration_as_airbyte_control_message(self._connector_config) + else: + with open('/Users/joereuter/Clones/airbyte/debug.txt', 'a') as file: file.write('reuse access token\n') return self.access_token - def refresh_access_token(self) -> Tuple[str, int, str]: + def refresh_access_token(self) -> Tuple[str, str, str]: response_json = self._get_refresh_access_token_response() return ( response_json[self.get_access_token_name()], - int(response_json[self.get_expires_in_name()]), + response_json[self.get_expires_in_name()], response_json[self.get_refresh_token_name()], ) diff --git a/airbyte-integrations/connectors/source-xero/source_xero/source.py b/airbyte-integrations/connectors/source-xero/source_xero/source.py index b29cfaa27cbf..1e1ccf3f7727 100644 --- a/airbyte-integrations/connectors/source-xero/source_xero/source.py +++ b/airbyte-integrations/connectors/source-xero/source_xero/source.py @@ -83,8 +83,8 @@ def get_authenticator(config: Mapping[str, Any]) -> Mapping[str, Any]: return XeroSingleUseRefreshTokenOauth2Authenticator( connector_config=config, token_refresh_endpoint="https://identity.xero.com/connect/token", - client_id_config_path=["authentication", "client_id"], - client_secret_config_path=["authentication", "client_secret"], + client_id=config["authentication"]["client_id"], + client_secret=config["authentication"]["client_secret"], access_token_config_path=["authentication", "access_token"], refresh_token_config_path=["authentication", "refresh_token"], token_expiry_date_config_path=["authentication", "token_expiry_date"], From a754d09eb7e0cf8bb3938c0ca224228fc6bcffb0 Mon Sep 17 00:00:00 2001 From: flash1293 Date: Fri, 2 Jun 2023 16:18:14 +0000 Subject: [PATCH 02/10] Automated Commit - Formatting Changes --- .../parsers/model_to_component_factory.py | 2 +- .../streams/http/requests_native_auth/oauth.py | 15 ++++++++++----- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py index 26af76805d6b..d3575c15c870 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py @@ -7,9 +7,9 @@ import importlib import inspect import re -import dpath from typing import Any, Callable, List, Literal, Mapping, Optional, Type, Union, get_args, get_origin, get_type_hints +import dpath from airbyte_cdk.sources.declarative.auth import DeclarativeOauth2Authenticator from airbyte_cdk.sources.declarative.auth.declarative_authenticator import NoAuth from airbyte_cdk.sources.declarative.auth.token import ( diff --git a/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py b/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py index 917c37c33209..c0a0506d8263 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py @@ -227,18 +227,23 @@ def get_access_token(self) -> str: str: The current access_token, updated if it was previously expired. """ if self.token_has_expired(): - with open('/Users/joereuter/Clones/airbyte/debug.txt', 'a') as file: file.write('get a new refresh token\n') + with open("/Users/joereuter/Clones/airbyte/debug.txt", "a") as file: + file.write("get a new refresh token\n") new_access_token, access_token_expires_in, new_refresh_token = self.refresh_access_token() new_token_expiry_date = self.get_new_token_expiry_date(access_token_expires_in, self._token_expiry_date_format) self.access_token = new_access_token self.set_refresh_token(new_refresh_token) self.set_token_expiry_date(new_token_expiry_date) - with open('/Users/joereuter/Clones/airbyte/debug.txt', 'a') as file: file.write(f'new refresh token: {new_refresh_token}\n') - with open('/Users/joereuter/Clones/airbyte/debug.txt', 'a') as file: file.write(f'new access token: {new_access_token}\n') - with open('/Users/joereuter/Clones/airbyte/debug.txt', 'a') as file: file.write(f'new expiry date: {new_token_expiry_date}\n') + with open("/Users/joereuter/Clones/airbyte/debug.txt", "a") as file: + file.write(f"new refresh token: {new_refresh_token}\n") + with open("/Users/joereuter/Clones/airbyte/debug.txt", "a") as file: + file.write(f"new access token: {new_access_token}\n") + with open("/Users/joereuter/Clones/airbyte/debug.txt", "a") as file: + file.write(f"new expiry date: {new_token_expiry_date}\n") emit_configuration_as_airbyte_control_message(self._connector_config) else: - with open('/Users/joereuter/Clones/airbyte/debug.txt', 'a') as file: file.write('reuse access token\n') + with open("/Users/joereuter/Clones/airbyte/debug.txt", "a") as file: + file.write("reuse access token\n") return self.access_token def refresh_access_token(self) -> Tuple[str, str, str]: From f02205cecac90ea0d8530b35f62eb4743ddcb3b3 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Mon, 5 Jun 2023 10:07:08 +0200 Subject: [PATCH 03/10] add documentation --- .../sources/streams/http/requests_native_auth/oauth.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py b/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py index 917c37c33209..0e84450fb17d 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py @@ -127,11 +127,12 @@ def __init__( refresh_token_name (str, optional): Name of the name of the refresh token field, used to parse the refresh token response. Defaults to "refresh_token". refresh_request_body (Mapping[str, Any], optional): Custom key value pair that will be added to the refresh token request body. Defaults to None. grant_type (str, optional): OAuth grant type. Defaults to "refresh_token". - client_id_config_path (Sequence[str]): Dpath to the client_id field in the connector configuration. Defaults to ("credentials", "client_id"). - client_secret_config_path (Sequence[str]): Dpath to the client_secret field in the connector configuration. Defaults to ("credentials", "client_secret"). + client_id (Optional[str]): The client id to authenticate. If not specified, defaults to credentials.client_id in the config object. + client_secret (Optional[str]): The client secret to authenticate. If not specified, defaults to credentials.client_secret in the config object. access_token_config_path (Sequence[str]): Dpath to the access_token field in the connector configuration. Defaults to ("credentials", "access_token"). refresh_token_config_path (Sequence[str]): Dpath to the refresh_token field in the connector configuration. Defaults to ("credentials", "refresh_token"). token_expiry_date_config_path (Sequence[str]): Dpath to the token_expiry_date field in the connector configuration. Defaults to ("credentials", "token_expiry_date"). + token_expiry_date_format (Optional[str]): Date format of the token expiry date field (set by expires_in_name). If not specified the token expiry date is interpreted as number of seconds until expiration. """ self._client_id = client_id if client_id is not None else dpath.util.get(connector_config, ("credentials", "client_id")) self._client_secret = ( From cc3c4d86e68f5acc0db2c9ba50663ace02f6db68 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Mon, 5 Jun 2023 10:47:27 +0200 Subject: [PATCH 04/10] tests and fixes --- .../declarative_component_schema.yaml | 7 ---- .../models/declarative_component_schema.py | 6 --- .../http/requests_native_auth/oauth.py | 11 ----- .../test_model_to_component_factory.py | 40 +++++++++++++++++++ 4 files changed, 40 insertions(+), 24 deletions(-) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/declarative_component_schema.yaml b/airbyte-cdk/python/airbyte_cdk/sources/declarative/declarative_component_schema.yaml index f2df0a180047..28bc5f4fdee4 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/declarative_component_schema.yaml +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/declarative_component_schema.yaml @@ -774,13 +774,6 @@ definitions: default: ["credentials", "token_expiry_date"] examples: - ["credentials", "token_expiry_date"] - access_token_name: - title: Access Token Response Field Name - description: The name of the field to extract the access token from in the token refresh response. - type: string - default: "access_token" - examples: - - "access_token" $parameters: type: object additionalProperties: true diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/models/declarative_component_schema.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/models/declarative_component_schema.py index 57b1a2ff6fee..3aef40a02c13 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/models/declarative_component_schema.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/models/declarative_component_schema.py @@ -284,12 +284,6 @@ class RefreshTokenUpdater(BaseModel): examples=[["credentials", "token_expiry_date"]], title="Config Path To Expiry Date", ) - access_token_name: Optional[str] = Field( - "access_token", - description="The name of the field to extract the access token from in the token refresh response.", - examples=["access_token"], - title="Access Token Response Field Name", - ) class OAuthAuthenticator(BaseModel): diff --git a/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py b/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py index 4cad43f281bd..381052974d23 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py @@ -228,23 +228,12 @@ def get_access_token(self) -> str: str: The current access_token, updated if it was previously expired. """ if self.token_has_expired(): - with open("/Users/joereuter/Clones/airbyte/debug.txt", "a") as file: - file.write("get a new refresh token\n") new_access_token, access_token_expires_in, new_refresh_token = self.refresh_access_token() new_token_expiry_date = self.get_new_token_expiry_date(access_token_expires_in, self._token_expiry_date_format) self.access_token = new_access_token self.set_refresh_token(new_refresh_token) self.set_token_expiry_date(new_token_expiry_date) - with open("/Users/joereuter/Clones/airbyte/debug.txt", "a") as file: - file.write(f"new refresh token: {new_refresh_token}\n") - with open("/Users/joereuter/Clones/airbyte/debug.txt", "a") as file: - file.write(f"new access token: {new_access_token}\n") - with open("/Users/joereuter/Clones/airbyte/debug.txt", "a") as file: - file.write(f"new expiry date: {new_token_expiry_date}\n") emit_configuration_as_airbyte_control_message(self._connector_config) - else: - with open("/Users/joereuter/Clones/airbyte/debug.txt", "a") as file: - file.write("reuse access token\n") return self.access_token def refresh_access_token(self) -> Tuple[str, str, str]: diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py b/airbyte-cdk/python/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py index c268b75557c3..1d9deba2b81f 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py @@ -54,6 +54,7 @@ from airbyte_cdk.sources.declarative.transformations import AddFields, RemoveFields from airbyte_cdk.sources.declarative.transformations.add_fields import AddedFieldDefinition from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource +from airbyte_cdk.sources.streams.http.requests_native_auth.oauth import SingleUseRefreshTokenOauth2Authenticator from unit_tests.sources.declarative.parsers.testing_components import TestingCustomSubstreamPartitionRouter, TestingSomeComponent factory = ModelToComponentFactory() @@ -293,6 +294,45 @@ def test_interpolate_config(): assert authenticator.get_refresh_request_body() == {"body_field": "yoyoyo", "interpolated_body_field": "verysecrettoken"} +def test_single_use_oauth_branch(): + single_use_input_config = {"apikey": "verysecrettoken", "repos": ["airbyte", "airbyte-cloud"], "credentials": {"access_token": "access_token", "token_expiry_date": "1970-01-01"}} + + content = """ + authenticator: + type: OAuthAuthenticator + client_id: "some_client_id" + client_secret: "some_client_secret" + token_refresh_endpoint: "https://api.sendgrid.com/v3/auth" + refresh_token: "{{ config['apikey'] }}" + refresh_request_body: + body_field: "yoyoyo" + interpolated_body_field: "{{ config['apikey'] }}" + refresh_token_updater: + refresh_token_name: "the_refresh_token" + refresh_token_config_path: + - apikey + """ + parsed_manifest = YamlDeclarativeSource._parse(content) + resolved_manifest = resolver.preprocess_manifest(parsed_manifest) + authenticator_manifest = transformer.propagate_types_and_parameters("", resolved_manifest["authenticator"], {}) + + authenticator: SingleUseRefreshTokenOauth2Authenticator = factory.create_component( + model_type=OAuthAuthenticatorModel, component_definition=authenticator_manifest, config=single_use_input_config + ) + + assert isinstance(authenticator, SingleUseRefreshTokenOauth2Authenticator) + assert authenticator._client_id == "some_client_id" + assert authenticator._client_secret == "some_client_secret" + assert authenticator._token_refresh_endpoint == "https://api.sendgrid.com/v3/auth" + assert authenticator._refresh_token == "verysecrettoken" + assert authenticator._refresh_request_body == {"body_field": "yoyoyo", "interpolated_body_field": "verysecrettoken"} + assert authenticator._refresh_token_name == "the_refresh_token" + assert authenticator._refresh_token_config_path == ["apikey"] + # default values + assert authenticator._access_token_config_path == ["credentials", "access_token"] + assert authenticator._token_expiry_date_config_path == ["credentials", "token_expiry_date"] + + def test_list_based_stream_slicer_with_values_refd(): content = """ repositories: ["airbyte", "airbyte-cloud"] From d54d2a7f35bdec53a66fa05e085053052576b750 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Mon, 5 Jun 2023 11:09:48 +0200 Subject: [PATCH 05/10] fix tests --- .../http/requests_native_auth/oauth.py | 3 +- .../test_requests_native_auth.py | 37 +++++++++++-------- 2 files changed, 24 insertions(+), 16 deletions(-) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py b/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py index 381052974d23..a3340cb483bf 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py @@ -114,7 +114,7 @@ def __init__( access_token_config_path: Sequence[str] = ("credentials", "access_token"), refresh_token_config_path: Sequence[str] = ("credentials", "refresh_token"), token_expiry_date_config_path: Sequence[str] = ("credentials", "token_expiry_date"), - token_expiry_date_format: str = None, + token_expiry_date_format: Optional[str] = None, ): """ @@ -156,6 +156,7 @@ def __init__( expires_in_name=expires_in_name, refresh_request_body=refresh_request_body, grant_type=grant_type, + token_expiry_date_format=token_expiry_date_format, ) def _validate_connector_config(self): diff --git a/airbyte-cdk/python/unit_tests/sources/streams/http/requests_native_auth/test_requests_native_auth.py b/airbyte-cdk/python/unit_tests/sources/streams/http/requests_native_auth/test_requests_native_auth.py index 6724dd0343d6..fa31210e1ec0 100644 --- a/airbyte-cdk/python/unit_tests/sources/streams/http/requests_native_auth/test_requests_native_auth.py +++ b/airbyte-cdk/python/unit_tests/sources/streams/http/requests_native_auth/test_requests_native_auth.py @@ -227,6 +227,8 @@ def test_init(self, connector_config): authenticator = SingleUseRefreshTokenOauth2Authenticator( connector_config, token_refresh_endpoint="foobar", + client_id=connector_config["credentials"]["client_id"], + client_secret=connector_config["credentials"]["client_secret"], ) assert authenticator.access_token == connector_config["credentials"]["access_token"] assert authenticator.get_refresh_token() == connector_config["credentials"]["refresh_token"] @@ -237,15 +239,28 @@ def test_init_with_invalid_config(self, invalid_connector_config): SingleUseRefreshTokenOauth2Authenticator( invalid_connector_config, token_refresh_endpoint="foobar", + client_id="my_client_id", + client_secret="my_client_secret", ) @freezegun.freeze_time("2022-12-31") - def test_get_access_token(self, capsys, mocker, connector_config): + @pytest.mark.parametrize( + "test_name, expires_in_value, expiry_date_format, expected_expiry_date", + [ + ("number_of_seconds", 42, None, "2022-12-31T00:00:42+00:00"), + ("string_of_seconds", "42", None, "2022-12-31T00:00:42+00:00"), + ("date_format", "2023-04-04", "YYYY-MM-DD", "2023-04-04T00:00:00+00:00"), + ] + ) + def test_get_access_token(self, test_name, expires_in_value, expiry_date_format, expected_expiry_date, capsys, mocker, connector_config): authenticator = SingleUseRefreshTokenOauth2Authenticator( connector_config, token_refresh_endpoint="foobar", + client_id=connector_config["credentials"]["client_id"], + client_secret=connector_config["credentials"]["client_secret"], + token_expiry_date_format=expiry_date_format, ) - authenticator.refresh_access_token = mocker.Mock(return_value=("new_access_token", 42, "new_refresh_token")) + authenticator.refresh_access_token = mocker.Mock(return_value=("new_access_token", expires_in_value, "new_refresh_token")) authenticator.token_has_expired = mocker.Mock(return_value=True) access_token = authenticator.get_access_token() captured = capsys.readouterr() @@ -253,7 +268,7 @@ def test_get_access_token(self, capsys, mocker, connector_config): expected_new_config = connector_config.copy() expected_new_config["credentials"]["access_token"] = "new_access_token" expected_new_config["credentials"]["refresh_token"] = "new_refresh_token" - expected_new_config["credentials"]["token_expiry_date"] = "2022-12-31T00:00:42+00:00" + expected_new_config["credentials"]["token_expiry_date"] = expected_expiry_date assert airbyte_message["control"]["connectorConfig"]["config"] == expected_new_config assert authenticator.access_token == access_token == "new_access_token" assert authenticator.get_refresh_token() == "new_refresh_token" @@ -268,26 +283,18 @@ def test_refresh_access_token(self, mocker, connector_config): authenticator = SingleUseRefreshTokenOauth2Authenticator( connector_config, token_refresh_endpoint="foobar", + client_id=connector_config["credentials"]["client_id"], + client_secret=connector_config["credentials"]["client_secret"], ) authenticator._get_refresh_access_token_response = mocker.Mock( return_value={ authenticator.get_access_token_name(): "new_access_token", - authenticator.get_expires_in_name(): 42, + authenticator.get_expires_in_name(): "42", authenticator.get_refresh_token_name(): "new_refresh_token", } ) - assert authenticator.refresh_access_token() == ("new_access_token", 42, "new_refresh_token") - - # Test with expires_in as str - authenticator._get_refresh_access_token_response = mocker.Mock( - return_value={ - authenticator.get_access_token_name(): "new_access_token", - authenticator.get_expires_in_name(): "1000", - authenticator.get_refresh_token_name(): "new_refresh_token", - } - ) - assert authenticator.refresh_access_token() == ("new_access_token", 1000, "new_refresh_token") + assert authenticator.refresh_access_token() == ("new_access_token", "42", "new_refresh_token") def mock_request(method, url, data): From 8b8d5df594f914ee5652052d4eb85f53b4ac26ff Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Mon, 5 Jun 2023 11:14:34 +0200 Subject: [PATCH 06/10] more documentation --- .../sources/declarative/declarative_component_schema.yaml | 8 ++++---- .../declarative/models/declarative_component_schema.py | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/declarative_component_schema.yaml b/airbyte-cdk/python/airbyte_cdk/sources/declarative/declarative_component_schema.yaml index 28bc5f4fdee4..303a04b6ac7c 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/declarative_component_schema.yaml +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/declarative_component_schema.yaml @@ -737,7 +737,7 @@ definitions: - "%Y-%m-%d %H:%M:%S.%f+00:00" refresh_token_updater: title: Token Updater - description: When the token updater is defined, new refresh tokens, access tokens and the access token expiry date are written back to the config object. This is important if the refresh token can only used once. + description: When the token updater is defined, new refresh tokens, access tokens and the access token expiry date are written back from the authentication response to the config object. This is important if the refresh token can only used once. properties: refresh_token_name: title: Refresh Token Response Field Name @@ -747,7 +747,7 @@ definitions: - "refresh_token" access_token_config_path: title: Config Path To Access Token - description: Config path to the access token. + description: Config path to the access token. Make sure the field actually exists in the config. type: array items: type: string @@ -757,7 +757,7 @@ definitions: - ["access_token"] refresh_token_config_path: title: Config Path To Refresh Token - description: Config path to the access token. + description: Config path to the access token. Make sure the field actually exists in the config. type: array items: type: string @@ -767,7 +767,7 @@ definitions: - ["refresh_token"] token_expiry_date_config_path: title: Config Path To Expiry Date - description: Config path to the expiry date. + description: Config path to the expiry date. Make sure actually exists in the config. type: array items: type: string diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/models/declarative_component_schema.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/models/declarative_component_schema.py index 3aef40a02c13..d2544de3be50 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/models/declarative_component_schema.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/models/declarative_component_schema.py @@ -268,19 +268,19 @@ class RefreshTokenUpdater(BaseModel): ) access_token_config_path: Optional[List[str]] = Field( ["credentials", "access_token"], - description="Config path to the access token.", + description="Config path to the access token. Make sure the field actually exists in the config.", examples=[["credentials", "access_token"], ["access_token"]], title="Config Path To Access Token", ) refresh_token_config_path: Optional[List[str]] = Field( ["credentials", "refresh_token"], - description="Config path to the access token.", + description="Config path to the access token. Make sure the field actually exists in the config.", examples=[["credentials", "refresh_token"], ["refresh_token"]], title="Config Path To Refresh Token", ) token_expiry_date_config_path: Optional[List[str]] = Field( ["credentials", "token_expiry_date"], - description="Config path to the expiry date.", + description="Config path to the expiry date. Make sure actually exists in the config.", examples=[["credentials", "token_expiry_date"]], title="Config Path To Expiry Date", ) @@ -368,7 +368,7 @@ class OAuthAuthenticator(BaseModel): ) refresh_token_updater: Optional[RefreshTokenUpdater] = Field( None, - description="When the token updater is defined, new refresh tokens, access tokens and the access token expiry date are written back to the config object. This is important if the refresh token can only used once.", + description="When the token updater is defined, new refresh tokens, access tokens and the access token expiry date are written back from the authentication response to the config object. This is important if the refresh token can only used once.", title="Token Updater", ) parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") From 9cfcd3fcc6cd12d5339fbdbfb594aae238cdff03 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Mon, 5 Jun 2023 11:21:42 +0200 Subject: [PATCH 07/10] revert --- .../connectors/source-xero/source_xero/source.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/source-xero/source_xero/source.py b/airbyte-integrations/connectors/source-xero/source_xero/source.py index 1e1ccf3f7727..b29cfaa27cbf 100644 --- a/airbyte-integrations/connectors/source-xero/source_xero/source.py +++ b/airbyte-integrations/connectors/source-xero/source_xero/source.py @@ -83,8 +83,8 @@ def get_authenticator(config: Mapping[str, Any]) -> Mapping[str, Any]: return XeroSingleUseRefreshTokenOauth2Authenticator( connector_config=config, token_refresh_endpoint="https://identity.xero.com/connect/token", - client_id=config["authentication"]["client_id"], - client_secret=config["authentication"]["client_secret"], + client_id_config_path=["authentication", "client_id"], + client_secret_config_path=["authentication", "client_secret"], access_token_config_path=["authentication", "access_token"], refresh_token_config_path=["authentication", "refresh_token"], token_expiry_date_config_path=["authentication", "token_expiry_date"], From e3e10b92a383dd320ab6fd00c0f99cf501cc79cf Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Mon, 5 Jun 2023 11:22:19 +0200 Subject: [PATCH 08/10] update xero --- .../connectors/source-xero/source_xero/source.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/source-xero/source_xero/source.py b/airbyte-integrations/connectors/source-xero/source_xero/source.py index b29cfaa27cbf..1e1ccf3f7727 100644 --- a/airbyte-integrations/connectors/source-xero/source_xero/source.py +++ b/airbyte-integrations/connectors/source-xero/source_xero/source.py @@ -83,8 +83,8 @@ def get_authenticator(config: Mapping[str, Any]) -> Mapping[str, Any]: return XeroSingleUseRefreshTokenOauth2Authenticator( connector_config=config, token_refresh_endpoint="https://identity.xero.com/connect/token", - client_id_config_path=["authentication", "client_id"], - client_secret_config_path=["authentication", "client_secret"], + client_id=config["authentication"]["client_id"], + client_secret=config["authentication"]["client_secret"], access_token_config_path=["authentication", "access_token"], refresh_token_config_path=["authentication", "refresh_token"], token_expiry_date_config_path=["authentication", "token_expiry_date"], From 5a212759e384918d2cea4fdb4fd21bcb10fee87d Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Mon, 5 Jun 2023 11:24:17 +0200 Subject: [PATCH 09/10] update documentation --- docs/integrations/sources/xero.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/integrations/sources/xero.md b/docs/integrations/sources/xero.md index ac4a5919ae47..0bdf4126d437 100644 --- a/docs/integrations/sources/xero.md +++ b/docs/integrations/sources/xero.md @@ -91,6 +91,7 @@ The connector is restricted by Xero [API rate limits](https://developer.xero.com | Version | Date | Pull Request | Subject | | :------ | :--------- | :------------------------------------------------------- | :-------------------------------- | +| 0.2.2 | 2023-06-06 | [27007](https://github.com/airbytehq/airbyte/pull/27007) | Update CDK | | 0.2.1 | 2023-03-20 | [24217](https://github.com/airbytehq/airbyte/pull/24217) | Certify to Beta | | 0.2.0 | 2023-03-14 | [24005](https://github.com/airbytehq/airbyte/pull/24005) | Enable in Cloud | | 0.1.0 | 2021-11-11 | [18666](https://github.com/airbytehq/airbyte/pull/18666) | 🎉 New Source - Xero [python cdk] | From b30568b72321203d86849ac45b45dfcc0d7e4406 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Mon, 5 Jun 2023 11:26:27 +0200 Subject: [PATCH 10/10] update version --- airbyte-integrations/connectors/source-xero/Dockerfile | 2 +- airbyte-integrations/connectors/source-xero/metadata.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/source-xero/Dockerfile b/airbyte-integrations/connectors/source-xero/Dockerfile index 81b3ed59f61e..e01c7939c5de 100644 --- a/airbyte-integrations/connectors/source-xero/Dockerfile +++ b/airbyte-integrations/connectors/source-xero/Dockerfile @@ -34,5 +34,5 @@ COPY source_xero ./source_xero ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.2.1 +LABEL io.airbyte.version=0.2.2 LABEL io.airbyte.name=airbyte/source-xero diff --git a/airbyte-integrations/connectors/source-xero/metadata.yaml b/airbyte-integrations/connectors/source-xero/metadata.yaml index e5b5e6b85d03..026b0ed664a2 100644 --- a/airbyte-integrations/connectors/source-xero/metadata.yaml +++ b/airbyte-integrations/connectors/source-xero/metadata.yaml @@ -5,7 +5,7 @@ data: connectorSubtype: api connectorType: source definitionId: 6fd1e833-dd6e-45ec-a727-ab917c5be892 - dockerImageTag: 0.2.1 + dockerImageTag: 0.2.2 dockerRepository: airbyte/source-xero githubIssueLabel: source-xero icon: xero.svg