diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/fe2b4084-3386-4d3b-9ad6-308f61a6f1e6.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/fe2b4084-3386-4d3b-9ad6-308f61a6f1e6.json index 3774938b28d5e..979da9076f57d 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/fe2b4084-3386-4d3b-9ad6-308f61a6f1e6.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/fe2b4084-3386-4d3b-9ad6-308f61a6f1e6.json @@ -2,6 +2,6 @@ "sourceDefinitionId": "fe2b4084-3386-4d3b-9ad6-308f61a6f1e6", "name": "Harvest", "dockerRepository": "airbyte/source-harvest", - "dockerImageTag": "0.1.5", + "dockerImageTag": "0.1.6", "documentationUrl": "https://docs.airbyte.io/integrations/sources/harvest" } diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index d28d11abadaad..aec9c9e5b7a58 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -236,7 +236,7 @@ - name: Harvest sourceDefinitionId: fe2b4084-3386-4d3b-9ad6-308f61a6f1e6 dockerRepository: airbyte/source-harvest - dockerImageTag: 0.1.5 + dockerImageTag: 0.1.6 documentationUrl: https://docs.airbyte.io/integrations/sources/harvest sourceType: api - name: HubSpot diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index bef52af9233b9..9b47693dd073f 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -2321,7 +2321,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-harvest:0.1.5" +- dockerImage: "airbyte/source-harvest:0.1.6" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/harvest" connectionSpecification: @@ -2329,22 +2329,17 @@ title: "Harvest Spec" type: "object" required: - - "api_token" - "account_id" - "replication_start_date" - additionalProperties: false + additionalProperties: true properties: - api_token: - title: "API Token" - description: "Harvest API Token." - airbyte_secret: true - type: "string" account_id: title: "Account ID" description: "Harvest account ID. Required for all Harvest requests in pair\ \ with API Key" airbyte_secret: true type: "string" + order: 0 replication_start_date: title: "Replication Start Date" description: "UTC date and time in the format 2017-01-25T00:00:00Z. Any\ @@ -2353,11 +2348,116 @@ examples: - "2017-01-25T00:00:00Z" type: "string" + order: 1 + credentials: + title: "Authentication mechanism" + description: "Choose how to authenticate to Harvest" + type: "object" + order: 2 + oneOf: + - type: "object" + title: "Authenticate via Harvest (Oauth)" + required: + - "client_id" + - "client_secret" + - "refresh_token" + additionalProperties: false + properties: + auth_type: + type: "string" + const: "Client" + enum: + - "Client" + default: "Client" + order: 0 + client_id: + title: "Client ID" + type: "string" + description: "The Client ID of your application" + client_secret: + title: "Client Secret" + type: "string" + description: "The client secret of your application" + airbyte_secret: true + refresh_token: + title: "Refresh Token" + type: "string" + description: "A refresh token generated using the above client ID\ + \ and secret" + airbyte_secret: true + - type: "object" + title: "Authenticate with Personal Access Token" + required: + - "api_token" + additionalProperties: false + properties: + auth_type: + type: "string" + const: "Token" + enum: + - "Token" + default: "Token" + order: 0 + api_token: + title: "Personal Access Token" + description: "Log into Harvest and then create new personal access token." + type: "string" + airbyte_secret: true supportsIncremental: true supportsNormalization: false supportsDBT: false supported_destination_sync_modes: - "append" + authSpecification: + auth_type: "oauth2.0" + oauth2Specification: + rootObject: + - "credentials" + - "0" + oauthFlowInitParameters: + - - "client_id" + - - "client_secret" + oauthFlowOutputParameters: + - - "refresh_token" + advancedAuth: + auth_flow_type: "oauth2.0" + predicate_key: + - "credentials" + - "auth_type" + predicate_value: "Client" + oauth_config_specification: + complete_oauth_output_specification: + type: "object" + additionalProperties: false + properties: + refresh_token: + type: "string" + path_in_connector_config: + - "credentials" + - "refresh_token" + complete_oauth_server_input_specification: + type: "object" + additionalProperties: false + properties: + client_id: + type: "string" + client_secret: + type: "string" + complete_oauth_server_output_specification: + type: "object" + additionalProperties: false + properties: + client_id: + type: "string" + path_in_connector_config: + - "credentials" + - "client_id" + client_secret: + type: "string" + path_in_connector_config: + - "credentials" + - "client_secret" - dockerImage: "airbyte/source-hubspot:0.1.24" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/hubspot" diff --git a/airbyte-integrations/connectors/source-harvest/Dockerfile b/airbyte-integrations/connectors/source-harvest/Dockerfile index 7d16c04752acf..0b9cd8c09544e 100644 --- a/airbyte-integrations/connectors/source-harvest/Dockerfile +++ b/airbyte-integrations/connectors/source-harvest/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.5 +LABEL io.airbyte.version=0.1.6 LABEL io.airbyte.name=airbyte/source-harvest diff --git a/airbyte-integrations/connectors/source-harvest/acceptance-test-config.yml b/airbyte-integrations/connectors/source-harvest/acceptance-test-config.yml index 5632dd24e9791..b23a0698ff48c 100644 --- a/airbyte-integrations/connectors/source-harvest/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-harvest/acceptance-test-config.yml @@ -5,6 +5,10 @@ tests: connection: - config_path: "secrets/config.json" status: "succeed" + - config_path: "secrets/old_config.json" + status: "succeed" + - config_path: "secrets/config_oauth.json" + status: "succeed" - config_path: "integration_tests/invalid_config.json" status: "failed" discovery: diff --git a/airbyte-integrations/connectors/source-harvest/source_harvest/auth.py b/airbyte-integrations/connectors/source-harvest/source_harvest/auth.py index b7c176ddc2916..fd52d0abe1290 100644 --- a/airbyte-integrations/connectors/source-harvest/source_harvest/auth.py +++ b/airbyte-integrations/connectors/source-harvest/source_harvest/auth.py @@ -4,14 +4,32 @@ from typing import Any, Mapping -from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator +from airbyte_cdk.sources.streams.http.auth import Oauth2Authenticator, TokenAuthenticator -class HarvestTokenAuthenticator(TokenAuthenticator): - def __init__(self, token: str, account_id: str, account_id_header: str = "Harvest-Account-ID", **kwargs): - super().__init__(token, **kwargs) +class HarvestMixin: + """ + Mixin class for providing additional HTTP header for specifying account ID + https://help.getharvest.com/api-v2/authentication-api/authentication/authentication/ + """ + def __init__(self, *, account_id: str, account_id_header: str = "Harvest-Account-ID", **kwargs): + super().__init__(**kwargs) self.account_id = account_id self.account_id_header = account_id_header def get_auth_header(self) -> Mapping[str, Any]: return {**super().get_auth_header(), self.account_id_header: self.account_id} + + +class HarvestTokenAuthenticator(HarvestMixin, TokenAuthenticator): + """ + Auth class for Personal Access Token + https://help.getharvest.com/api-v2/authentication-api/authentication/authentication/#personal-access-tokens + """ + + +class HarvestOauth2Authenticator(HarvestMixin, Oauth2Authenticator): + """ + Auth class for OAuth2 + https://help.getharvest.com/api-v2/authentication-api/authentication/authentication/#for-server-side-applications + """ diff --git a/airbyte-integrations/connectors/source-harvest/source_harvest/source.py b/airbyte-integrations/connectors/source-harvest/source_harvest/source.py index 730e0a781a8f4..a2b60b6e2aa34 100644 --- a/airbyte-integrations/connectors/source-harvest/source_harvest/source.py +++ b/airbyte-integrations/connectors/source-harvest/source_harvest/source.py @@ -45,13 +45,30 @@ Users, ) -from .auth import HarvestTokenAuthenticator +from .auth import HarvestOauth2Authenticator, HarvestTokenAuthenticator class SourceHarvest(AbstractSource): + @staticmethod + def get_authenticator(config): + credentials = config.get("credentials", {}) + if credentials and "client_id" in credentials: + return HarvestOauth2Authenticator( + token_refresh_endpoint="https://id.getharvest.com/api/v2/oauth2/token", + client_id=credentials.get("client_id"), + client_secret=credentials.get("client_secret"), + refresh_token=credentials.get("refresh_token"), + account_id=config["account_id"], + ) + + api_token = credentials.get("api_token", config.get("api_token")) + if not api_token: + raise Exception("Config validation error: 'api_token' is a required property") + return HarvestTokenAuthenticator(token=api_token, account_id=config["account_id"]) + def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> Tuple[bool, Any]: try: - auth = HarvestTokenAuthenticator(token=config["api_token"], account_id=config["account_id"]) + auth = self.get_authenticator(config) replication_start_date = pendulum.parse(config["replication_start_date"]) users_gen = Users(authenticator=auth, replication_start_date=replication_start_date).read_records( sync_mode=SyncMode.full_refresh @@ -65,7 +82,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: """ :param config: A Mapping of the user input configuration as defined in the connector spec. """ - auth = HarvestTokenAuthenticator(token=config["api_token"], account_id=config["account_id"]) + auth = self.get_authenticator(config) replication_start_date = pendulum.parse(config["replication_start_date"]) from_date = replication_start_date.date() diff --git a/airbyte-integrations/connectors/source-harvest/source_harvest/spec.json b/airbyte-integrations/connectors/source-harvest/source_harvest/spec.json index 82812a0e4773b..17269a455dd53 100644 --- a/airbyte-integrations/connectors/source-harvest/source_harvest/spec.json +++ b/airbyte-integrations/connectors/source-harvest/source_harvest/spec.json @@ -4,30 +4,138 @@ "$schema": "http://json-schema.org/draft-07/schema#", "title": "Harvest Spec", "type": "object", - "required": ["api_token", "account_id", "replication_start_date"], - "additionalProperties": false, + "required": ["account_id", "replication_start_date"], + "additionalProperties": true, "properties": { - "api_token": { - "title": "API Token", - "description": "Harvest API Token.", - "airbyte_secret": true, - "type": "string" - }, "account_id": { "title": "Account ID", "description": "Harvest account ID. Required for all Harvest requests in pair with API Key", "airbyte_secret": true, - "type": "string" + "type": "string", + "order": 0 }, "replication_start_date": { "title": "Replication Start Date", "description": "UTC date and time in the format 2017-01-25T00:00:00Z. Any data before this date will not be replicated.", "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", "examples": ["2017-01-25T00:00:00Z"], - "type": "string" + "type": "string", + "order": 1 + }, + "credentials": { + "title": "Authentication mechanism", + "description": "Choose how to authenticate to Harvest", + "type": "object", + "order": 2, + "oneOf": [ + { + "type": "object", + "title": "Authenticate via Harvest (Oauth)", + "required": ["client_id", "client_secret", "refresh_token"], + "additionalProperties": false, + "properties": { + "auth_type": { + "type": "string", + "const": "Client", + "enum": ["Client"], + "default": "Client", + "order": 0 + }, + "client_id": { + "title": "Client ID", + "type": "string", + "description": "The Client ID of your application" + }, + "client_secret": { + "title": "Client Secret", + "type": "string", + "description": "The client secret of your application", + "airbyte_secret": true + }, + "refresh_token": { + "title": "Refresh Token", + "type": "string", + "description": "A refresh token generated using the above client ID and secret", + "airbyte_secret": true + } + } + }, + { + "type": "object", + "title": "Authenticate with Personal Access Token", + "required": ["api_token"], + "additionalProperties": false, + "properties": { + "auth_type": { + "type": "string", + "const": "Token", + "enum": ["Token"], + "default": "Token", + "order": 0 + }, + "api_token": { + "title": "Personal Access Token", + "description": "Log into Harvest and then create new personal access token.", + "type": "string", + "airbyte_secret": true + } + } + } + ] } } }, "supportsIncremental": true, - "supported_destination_sync_modes": ["append"] + "supported_destination_sync_modes": ["append"], + "authSpecification": { + "auth_type": "oauth2.0", + "oauth2Specification": { + "rootObject": ["credentials", 0], + "oauthFlowInitParameters": [["client_id"], ["client_secret"]], + "oauthFlowOutputParameters": [["refresh_token"]] + } + }, + "advancedAuth": { + "auth_flow_type": "oauth2.0", + "predicate_key": ["credentials", "auth_type"], + "predicate_value": "Client", + "oauth_config_specification": { + "complete_oauth_output_specification": { + "type": "object", + "additionalProperties": false, + "properties": { + "refresh_token": { + "type": "string", + "path_in_connector_config": ["credentials", "refresh_token"] + } + } + }, + "complete_oauth_server_input_specification": { + "type": "object", + "additionalProperties": false, + "properties": { + "client_id": { + "type": "string" + }, + "client_secret": { + "type": "string" + } + } + }, + "complete_oauth_server_output_specification": { + "type": "object", + "additionalProperties": false, + "properties": { + "client_id": { + "type": "string", + "path_in_connector_config": ["credentials", "client_id"] + }, + "client_secret": { + "type": "string", + "path_in_connector_config": ["credentials", "client_secret"] + } + } + } + } + } } diff --git a/airbyte-oauth/src/main/java/io/airbyte/oauth/OAuthImplementationFactory.java b/airbyte-oauth/src/main/java/io/airbyte/oauth/OAuthImplementationFactory.java index d9240c21c5058..4211e0776d483 100644 --- a/airbyte-oauth/src/main/java/io/airbyte/oauth/OAuthImplementationFactory.java +++ b/airbyte-oauth/src/main/java/io/airbyte/oauth/OAuthImplementationFactory.java @@ -10,6 +10,7 @@ import io.airbyte.config.persistence.ConfigRepository; import io.airbyte.oauth.flows.AsanaOAuthFlow; import io.airbyte.oauth.flows.GithubOAuthFlow; +import io.airbyte.oauth.flows.HarvestOAuthFlow; import io.airbyte.oauth.flows.HubspotOAuthFlow; import io.airbyte.oauth.flows.IntercomOAuthFlow; import io.airbyte.oauth.flows.LinkedinAdsOAuthFlow; @@ -45,6 +46,7 @@ public OAuthImplementationFactory(final ConfigRepository configRepository, final .put("airbyte/source-google-analytics-v4", new GoogleAnalyticsOAuthFlow(configRepository, httpClient)) .put("airbyte/source-google-search-console", new GoogleSearchConsoleOAuthFlow(configRepository, httpClient)) .put("airbyte/source-google-sheets", new GoogleSheetsOAuthFlow(configRepository, httpClient)) + .put("airbyte/source-harvest", new HarvestOAuthFlow(configRepository, httpClient)) .put("airbyte/source-hubspot", new HubspotOAuthFlow(configRepository, httpClient)) .put("airbyte/source-intercom", new IntercomOAuthFlow(configRepository, httpClient)) .put("airbyte/source-instagram", new InstagramOAuthFlow(configRepository, httpClient)) diff --git a/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/HarvestOAuthFlow.java b/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/HarvestOAuthFlow.java new file mode 100644 index 0000000000000..d7cceff0de02f --- /dev/null +++ b/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/HarvestOAuthFlow.java @@ -0,0 +1,83 @@ +/* + * Copyright (c) 2021 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.oauth.flows; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.collect.ImmutableMap; +import io.airbyte.config.persistence.ConfigRepository; +import io.airbyte.oauth.BaseOAuth2Flow; +import java.io.IOException; +import java.net.URISyntaxException; +import java.net.http.HttpClient; +import java.util.Map; +import java.util.UUID; +import java.util.function.Supplier; +import org.apache.http.client.utils.URIBuilder; + +public class HarvestOAuthFlow extends BaseOAuth2Flow { + + private static final String AUTHORIZE_URL = "https://id.getharvest.com/oauth2/authorize"; + private static final String ACCESS_TOKEN_URL = "https://id.getharvest.com/api/v2/oauth2/token"; + + public HarvestOAuthFlow(final ConfigRepository configRepository, final HttpClient httpClient) { + super(configRepository, httpClient); + } + + public HarvestOAuthFlow(final ConfigRepository configRepository, final HttpClient httpClient, final Supplier stateSupplier) { + super(configRepository, httpClient, stateSupplier); + } + + /** + * Depending on the OAuth flow implementation, the URL to grant user's consent may differ, + * especially in the query parameters to be provided. This function should generate such consent URL + * accordingly. + * + * @param definitionId The configured definition ID of this client + * @param clientId The configured client ID + * @param redirectUrl the redirect URL + */ + @Override + protected String formatConsentUrl(final UUID definitionId, + final String clientId, + final String redirectUrl, + final JsonNode inputOAuthConfiguration) + throws IOException { + try { + return new URIBuilder(AUTHORIZE_URL) + .addParameter("client_id", clientId) + .addParameter("response_type", "code") + .addParameter("redirect_uri", redirectUrl) + .addParameter("state", getState()) + .build().toString(); + } catch (final URISyntaxException e) { + throw new IOException("Failed to format Consent URL for OAuth flow", e); + } + } + + @Override + protected Map getAccessTokenQueryParameters(final String clientId, + final String clientSecret, + final String authCode, + final String redirectUrl) { + return ImmutableMap.builder() + // required + .put("client_id", clientId) + .put("redirect_uri", redirectUrl) + .put("client_secret", clientSecret) + .put("code", authCode) + .put("grant_type", "authorization_code") + .build(); + } + + /** + * Returns the URL where to retrieve the access token from. + * + */ + @Override + protected String getAccessTokenUrl() { + return ACCESS_TOKEN_URL; + } + +} diff --git a/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/HarvestOAuthFlowTest.java b/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/HarvestOAuthFlowTest.java new file mode 100644 index 0000000000000..96bf7a75caa92 --- /dev/null +++ b/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/HarvestOAuthFlowTest.java @@ -0,0 +1,21 @@ +/* + * Copyright (c) 2021 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.oauth.flows; + +import io.airbyte.oauth.BaseOAuthFlow; + +public class HarvestOAuthFlowTest extends BaseOAuthFlowTest { + + @Override + protected BaseOAuthFlow getOAuthFlow() { + return new HarvestOAuthFlow(getConfigRepository(), getHttpClient(), this::getConstantState); + } + + @Override + protected String getExpectedConsentUrl() { + return "https://id.getharvest.com/oauth2/authorize?client_id=test_client_id&response_type=code&redirect_uri=https%3A%2F%2Fairbyte.io&state=state"; + } + +} diff --git a/docs/integrations/sources/harvest.md b/docs/integrations/sources/harvest.md index b4bc7f88642c2..1e9f59c2511e7 100644 --- a/docs/integrations/sources/harvest.md +++ b/docs/integrations/sources/harvest.md @@ -69,6 +69,7 @@ See [docs](https://help.getharvest.com/api-v2/authentication-api/authentication/ | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | +| 0.1.6 | 2021-11-14 | [7952](https://github.com/airbytehq/airbyte/pull/7952) | Implement OAuth 2.0 support | | 0.1.5 | 2021-09-28 | [5747](https://github.com/airbytehq/airbyte/pull/5747) | Update schema date-time fields | | 0.1.4 | 2021-06-22 | [5701](https://github.com/airbytehq/airbyte/pull/5071) | Harvest normalization failure: fixing the schemas | | 0.1.3 | 2021-06-22 | [4274](https://github.com/airbytehq/airbyte/pull/4274) | Fix wrong data type on `statement_key` in `clients` stream | diff --git a/tools/bin/ci_credentials.sh b/tools/bin/ci_credentials.sh index 5bae1e1855f17..6d3e3835cf053 100755 --- a/tools/bin/ci_credentials.sh +++ b/tools/bin/ci_credentials.sh @@ -224,7 +224,6 @@ read_secrets source-google-sheets "$GOOGLE_SHEETS_TESTS_CREDS_OLD" "old_config.j read_secrets source-google-workspace-admin-reports "$GOOGLE_WORKSPACE_ADMIN_REPORTS_TEST_CREDS" read_secrets source-greenhouse "$GREENHOUSE_TEST_CREDS" read_secrets source-greenhouse "$GREENHOUSE_TEST_CREDS_LIMITED" "config_users_only.json" -read_secrets source-harvest "$HARVEST_INTEGRATION_TESTS_CREDS" read_secrets source-hubspot "$HUBSPOT_INTEGRATION_TESTS_CREDS" read_secrets source-hubspot "$HUBSPOT_INTEGRATION_TESTS_CREDS_OAUTH" "config_oauth.json" read_secrets source-instagram "$INSTAGRAM_INTEGRATION_TESTS_CREDS"