diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 7df59d480d51..452238a59e0a 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -383,7 +383,7 @@ - name: Mailchimp sourceDefinitionId: b03a9f3e-22a5-11eb-adc1-0242ac120002 dockerRepository: airbyte/source-mailchimp - dockerImageTag: 0.2.10 + dockerImageTag: 0.2.11 documentationUrl: https://docs.airbyte.io/integrations/sources/mailchimp icon: mailchimp.svg sourceType: api diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 4909346bbea4..294e4080a640 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -3764,31 +3764,109 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-mailchimp:0.2.10" +- dockerImage: "airbyte/source-mailchimp:0.2.11" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/mailchimp" connectionSpecification: $schema: "http://json-schema.org/draft-07/schema#" title: "Mailchimp Spec" type: "object" - required: - - "username" - - "apikey" - additionalProperties: false + required: [] + additionalProperties: true properties: - username: - type: "string" - title: "Username" - description: "The Username or email you use to sign into Mailchimp." - apikey: - type: "string" - airbyte_secret: true - title: "API Key" - description: "Mailchimp API Key. See the docs for information on how to generate this key." + credentials: + type: "object" + title: "Authentication Method" + oneOf: + - title: "OAuth2.0" + type: "object" + required: + - "auth_type" + - "access_token" + properties: + auth_type: + type: "string" + const: "oauth2.0" + enum: + - "oauth2.0" + default: "oauth2.0" + order: 0 + client_id: + title: "Client ID" + type: "string" + description: "The Client ID of your OAuth application." + airbyte_secret: true + client_secret: + title: "Client Secret" + type: "string" + description: "The Client Secret of your OAuth application." + airbyte_secret: true + access_token: + title: "Access Token" + type: "string" + description: "An access token generated using the above client ID\ + \ and secret." + airbyte_secret: true + - type: "object" + title: "API Key" + required: + - "auth_type" + - "apikey" + properties: + auth_type: + type: "string" + const: "apikey" + enum: + - "apikey" + default: "apikey" + order: 1 + apikey: + type: "string" + title: "API Key" + description: "Mailchimp API Key. See the docs for information on how to generate this key." + airbyte_secret: true supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] + advanced_auth: + auth_flow_type: "oauth2.0" + predicate_key: + - "credentials" + - "auth_type" + predicate_value: "oauth2.0" + oauth_config_specification: + complete_oauth_output_specification: + type: "object" + additionalProperties: false + properties: + access_token: + type: "string" + path_in_connector_config: + - "credentials" + - "access_token" + complete_oauth_server_input_specification: + type: "object" + additionalProperties: false + properties: + client_id: + type: "string" + client_secret: + type: "string" + complete_oauth_server_output_specification: + type: "object" + additionalProperties: false + properties: + client_id: + type: "string" + path_in_connector_config: + - "credentials" + - "client_id" + client_secret: + type: "string" + path_in_connector_config: + - "credentials" + - "client_secret" - dockerImage: "airbyte/source-mailgun:0.1.0" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/mailgun" diff --git a/airbyte-integrations/connectors/source-mailchimp/Dockerfile b/airbyte-integrations/connectors/source-mailchimp/Dockerfile index 3968856ba0bb..5ec742f2978b 100644 --- a/airbyte-integrations/connectors/source-mailchimp/Dockerfile +++ b/airbyte-integrations/connectors/source-mailchimp/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.2.10 +LABEL io.airbyte.version=0.2.11 LABEL io.airbyte.name=airbyte/source-mailchimp diff --git a/airbyte-integrations/connectors/source-mailchimp/acceptance-test-config.yml b/airbyte-integrations/connectors/source-mailchimp/acceptance-test-config.yml index 160e8caa190f..76b211bf8aa3 100644 --- a/airbyte-integrations/connectors/source-mailchimp/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-mailchimp/acceptance-test-config.yml @@ -2,16 +2,41 @@ connector_image: airbyte/source-mailchimp:dev tests: spec: - spec_path: "source_mailchimp/spec.json" + timeout_seconds: 60 connection: + # for old spec config (without oneOf) - config_path: "secrets/config.json" status: "succeed" + timeout_seconds: 180 + # for auth with API token + - config_path: "secrets/config_apikey.json" + status: "succeed" + timeout_seconds: 180 + # for auth with oauth2 token + - config_path: "secrets/config_oauth.json" + status: "succeed" + timeout_seconds: 180 - config_path: "integration_tests/invalid_config.json" status: "failed" + timeout_seconds: 180 + - config_path: "integration_tests/invalid_config_apikey.json" + status: "failed" + timeout_seconds: 180 + - config_path: "integration_tests/invalid_config_oauth.json" + status: "failed" + timeout_seconds: 180 discovery: + # for old spec config (without oneOf) - config_path: "secrets/config.json" + # for auth with API token + - config_path: "secrets/config_apikey.json" + # for auth with oauth2 token + - config_path: "secrets/config_oauth.json" basic_read: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" + - config_path: "secrets/config_oauth.json" + configured_catalog_path: "integration_tests/configured_catalog.json" # THIS TEST IS COMMENTED OUT. Tests are supposed to accept # `state = {cursor_field: value}`. When we have dependent endpoint path # `path_begin/{some_id}/path_end` we need a complex state like below: @@ -30,3 +55,5 @@ tests: full_refresh: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" + - config_path: "secrets/config_oauth.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-mailchimp/integration_tests/integration_test.py b/airbyte-integrations/connectors/source-mailchimp/integration_tests/integration_test.py deleted file mode 100644 index e1814314fc3b..000000000000 --- a/airbyte-integrations/connectors/source-mailchimp/integration_tests/integration_test.py +++ /dev/null @@ -1,7 +0,0 @@ -# -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. -# - - -def test_example_method(): - assert True diff --git a/airbyte-integrations/connectors/source-mailchimp/integration_tests/invalid_config_apikey.json b/airbyte-integrations/connectors/source-mailchimp/integration_tests/invalid_config_apikey.json new file mode 100644 index 000000000000..f2fd16517bba --- /dev/null +++ b/airbyte-integrations/connectors/source-mailchimp/integration_tests/invalid_config_apikey.json @@ -0,0 +1,6 @@ +{ + "credentials": { + "auth_type": "apikey", + "apikey": "api-key-awesome" + } +} diff --git a/airbyte-integrations/connectors/source-mailchimp/integration_tests/invalid_config_oauth.json b/airbyte-integrations/connectors/source-mailchimp/integration_tests/invalid_config_oauth.json new file mode 100644 index 000000000000..ef7ef97ee241 --- /dev/null +++ b/airbyte-integrations/connectors/source-mailchimp/integration_tests/invalid_config_oauth.json @@ -0,0 +1,8 @@ +{ + "credentials": { + "auth_type": "oauth2.0", + "client_id": "client_id", + "client_secret": "client_secret", + "access_token": "access_token" + } +} diff --git a/airbyte-integrations/connectors/source-mailchimp/setup.py b/airbyte-integrations/connectors/source-mailchimp/setup.py index 7b8d35bf6ad3..aa117a3256bd 100644 --- a/airbyte-integrations/connectors/source-mailchimp/setup.py +++ b/airbyte-integrations/connectors/source-mailchimp/setup.py @@ -13,7 +13,6 @@ packages=find_packages(), install_requires=[ "airbyte-cdk~=0.1.35", - "mailchimp3==3.0.14", "pytest~=6.1", ], package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, diff --git a/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/source.py b/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/source.py index aaea7ec9c3b3..822f7d3e5d01 100644 --- a/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/source.py +++ b/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/source.py @@ -6,36 +6,61 @@ import base64 from typing import Any, List, Mapping, Tuple +import requests from airbyte_cdk import AirbyteLogger from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator -from mailchimp3 import MailChimp +from requests.auth import AuthBase from .streams import Campaigns, EmailActivity, Lists -class HttpBasicAuthenticator(TokenAuthenticator): - def __init__(self, auth: Tuple[str, str], auth_method: str = "Basic", **kwargs): - # API keys have the format -. - # See https://mailchimp.com/developer/marketing/docs/fundamentals/#api-structure - self.data_center = auth[1].split("-").pop() - auth_string = f"{auth[0]}:{auth[1]}".encode("utf8") - b64_encoded = base64.b64encode(auth_string).decode("utf8") - super().__init__(token=b64_encoded, auth_method=auth_method, **kwargs) +class MailChimpAuthenticator: + @staticmethod + def get_server_prefix(access_token: str) -> str: + try: + response = requests.get( + "https://login.mailchimp.com/oauth2/metadata", headers={"Authorization": "OAuth {}".format(access_token)} + ) + return response.json()["dc"] + except Exception as e: + raise Exception(f"Cannot retrieve server_prefix for you account. \n {repr(e)}") + + def get_auth(self, config: Mapping[str, Any]) -> AuthBase: + authorization = config.get("credentials", {}) + auth_type = authorization.get("auth_type") + if auth_type == "apikey" or not authorization: + # API keys have the format -. + # See https://mailchimp.com/developer/marketing/docs/fundamentals/#api-structure + apikey = authorization.get("apikey") or config.get("apikey") + if not apikey: + raise Exception("No apikey in creds") + auth_string = f"anystring:{apikey}".encode("utf8") + b64_encoded = base64.b64encode(auth_string).decode("utf8") + auth = TokenAuthenticator(token=b64_encoded, auth_method="Basic") + auth.data_center = apikey.split("-").pop() + + elif auth_type == "oauth2.0": + access_token = authorization["access_token"] + auth = TokenAuthenticator(token=access_token, auth_method="Bearer") + auth.data_center = self.get_server_prefix(access_token) + + else: + raise Exception(f"Invalid auth type: {auth_type}") + + return auth class SourceMailchimp(AbstractSource): def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> Tuple[bool, Any]: try: - client = MailChimp(mc_api=config["apikey"], mc_user=config["username"]) - client.ping.get() + authenticator = MailChimpAuthenticator().get_auth(config) + requests.get(f"https://{authenticator.data_center}.api.mailchimp.com/3.0/ping", headers=authenticator.get_auth_header()) return True, None except Exception as e: return False, repr(e) def streams(self, config: Mapping[str, Any]) -> List[Stream]: - authenticator = HttpBasicAuthenticator(auth=("anystring", config["apikey"])) - streams_ = [Lists(authenticator=authenticator), Campaigns(authenticator=authenticator), EmailActivity(authenticator=authenticator)] - - return streams_ + authenticator = MailChimpAuthenticator().get_auth(config) + return [Lists(authenticator=authenticator), Campaigns(authenticator=authenticator), EmailActivity(authenticator=authenticator)] diff --git a/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/spec.json b/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/spec.json index 3aee31fff53b..98de089a30c9 100644 --- a/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/spec.json +++ b/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/spec.json @@ -4,19 +4,109 @@ "$schema": "http://json-schema.org/draft-07/schema#", "title": "Mailchimp Spec", "type": "object", - "required": ["username", "apikey"], - "additionalProperties": false, + "required": [], + "additionalProperties": true, "properties": { - "username": { - "type": "string", - "title": "Username", - "description": "The Username or email you use to sign into Mailchimp." + "credentials": { + "type": "object", + "title": "Authentication Method", + "oneOf": [ + { + "title": "OAuth2.0", + "type": "object", + "required": ["auth_type", "access_token"], + "properties": { + "auth_type": { + "type": "string", + "const": "oauth2.0", + "enum": ["oauth2.0"], + "default": "oauth2.0", + "order": 0 + }, + "client_id": { + "title": "Client ID", + "type": "string", + "description": "The Client ID of your OAuth application.", + "airbyte_secret": true + }, + "client_secret": { + "title": "Client Secret", + "type": "string", + "description": "The Client Secret of your OAuth application.", + "airbyte_secret": true + }, + "access_token": { + "title": "Access Token", + "type": "string", + "description": "An access token generated using the above client ID and secret.", + "airbyte_secret": true + } + } + }, + { + "type": "object", + "title": "API Key", + "required": ["auth_type", "apikey"], + "properties": { + "auth_type": { + "type": "string", + "const": "apikey", + "enum": ["apikey"], + "default": "apikey", + "order": 1 + }, + "apikey": { + "type": "string", + "title": "API Key", + "description": "Mailchimp API Key. See the docs for information on how to generate this key.", + "airbyte_secret": true + } + } + } + ] + } + } + }, + "advanced_auth": { + "auth_flow_type": "oauth2.0", + "predicate_key": ["credentials", "auth_type"], + "predicate_value": "oauth2.0", + "oauth_config_specification": { + "complete_oauth_output_specification": { + "type": "object", + "additionalProperties": false, + "properties": { + "access_token": { + "type": "string", + "path_in_connector_config": ["credentials", "access_token"] + } + } + }, + "complete_oauth_server_input_specification": { + "type": "object", + "additionalProperties": false, + "properties": { + "client_id": { + "type": "string" + }, + "client_secret": { + "type": "string" + } + } }, - "apikey": { - "type": "string", - "airbyte_secret": true, - "title": "API Key", - "description": "Mailchimp API Key. See the docs for information on how to generate this key." + "complete_oauth_server_output_specification": { + "type": "object", + "additionalProperties": false, + "properties": { + "client_id": { + "type": "string", + "path_in_connector_config": ["credentials", "client_id"] + }, + "client_secret": { + "type": "string", + "path_in_connector_config": ["credentials", "client_secret"] + } + } } } } diff --git a/airbyte-oauth/src/main/java/io/airbyte/oauth/OAuthImplementationFactory.java b/airbyte-oauth/src/main/java/io/airbyte/oauth/OAuthImplementationFactory.java index 0894acbf4efc..d9de82bf9ecc 100644 --- a/airbyte-oauth/src/main/java/io/airbyte/oauth/OAuthImplementationFactory.java +++ b/airbyte-oauth/src/main/java/io/airbyte/oauth/OAuthImplementationFactory.java @@ -55,6 +55,7 @@ public OAuthImplementationFactory(final ConfigRepository configRepository, final .put("airbyte/source-zendesk-chat", new ZendeskChatOAuthFlow(configRepository, httpClient)) .put("airbyte/source-monday", new MondayOAuthFlow(configRepository, httpClient)) .put("airbyte/source-zendesk-sunshine", new ZendeskSunshineOAuthFlow(configRepository, httpClient)) + .put("airbyte/source-mailchimp", new MailchimpOAuthFlow(configRepository, httpClient)) .build(); } diff --git a/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/MailchimpOAuthFlow.java b/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/MailchimpOAuthFlow.java new file mode 100644 index 000000000000..84b87c91efb7 --- /dev/null +++ b/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/MailchimpOAuthFlow.java @@ -0,0 +1,89 @@ +/* + * Copyright (c) 2021 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.oauth.flows; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableMap; +import io.airbyte.config.persistence.ConfigRepository; +import io.airbyte.oauth.BaseOAuth2Flow; +import java.io.IOException; +import java.net.URISyntaxException; +import java.net.http.HttpClient; +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; +import java.util.function.Supplier; +import org.apache.http.client.utils.URIBuilder; + +/** + * Following docs from https://mailchimp.com/developer/marketing/guides/access-user-data-oauth-2/ + */ +public class MailchimpOAuthFlow extends BaseOAuth2Flow { + + private static final String ACCESS_TOKEN_URL = "https://login.mailchimp.com/oauth2/token"; + private static final String AUTHORIZE_URL = "https://login.mailchimp.com/oauth2/authorize"; + + public MailchimpOAuthFlow(final ConfigRepository configRepository, final HttpClient httpClient) { + super(configRepository, httpClient); + } + + @VisibleForTesting + public MailchimpOAuthFlow(final ConfigRepository configRepository, final HttpClient httpClient, final Supplier stateSupplier) { + super(configRepository, httpClient, stateSupplier); + } + + @Override + protected String formatConsentUrl(final UUID definitionId, + final String clientId, + final String redirectUrl, + final JsonNode inputOAuthConfiguration) + throws IOException { + + try { + return new URIBuilder(AUTHORIZE_URL) + .addParameter("client_id", clientId) + .addParameter("response_type", "code") + .addParameter("redirect_uri", redirectUrl) + .addParameter("state", getState()) + .build().toString(); + } catch (URISyntaxException e) { + throw new IOException("Failed to format Consent URL for OAuth flow", e); + } + } + + @Override + protected Map getAccessTokenQueryParameters(String clientId, + String clientSecret, + String authCode, + String redirectUrl) { + return ImmutableMap.builder() + // required + .put("grant_type", "authorization_code") + .put("code", authCode) + .put("client_id", clientId) + .put("client_secret", clientSecret) + .put("redirect_uri", redirectUrl) + .build(); + } + + @Override + protected String getAccessTokenUrl(final JsonNode inputOAuthConfiguration) { + return ACCESS_TOKEN_URL; + } + + @Override + protected Map extractOAuthOutput(final JsonNode data, final String accessTokenUrl) throws IOException { + final Map result = new HashMap<>(); + // getting out access_token + if (data.has("access_token")) { + result.put("access_token", data.get("access_token").asText()); + } else { + throw new IOException(String.format("Missing 'access_token' in query params from %s", accessTokenUrl)); + } + return result; + } + +} diff --git a/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/MailchimpOAuthFlowTest.java b/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/MailchimpOAuthFlowTest.java new file mode 100644 index 000000000000..138f93bba448 --- /dev/null +++ b/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/MailchimpOAuthFlowTest.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2021 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.oauth.flows; + +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.oauth.BaseOAuthFlow; +import io.airbyte.oauth.MoreOAuthParameters; +import java.util.Map; + +public class MailchimpOAuthFlowTest extends BaseOAuthFlowTest { + + @Override + protected BaseOAuthFlow getOAuthFlow() { + return new MailchimpOAuthFlow(getConfigRepository(), getHttpClient(), this::getConstantState); + } + + @Override + protected String getExpectedConsentUrl() { + return "https://login.mailchimp.com/oauth2/authorize?client_id=test_client_id&response_type=code&redirect_uri=https%3A%2F%2Fairbyte.io&state=state"; + } + + @Override + protected Map getExpectedOutput() { + return Map.of( + "access_token", "access_token_response", + "client_id", MoreOAuthParameters.SECRET_MASK, + "client_secret", MoreOAuthParameters.SECRET_MASK); + } + + @Override + protected JsonNode getCompleteOAuthOutputSpecification() { + return getJsonSchema(Map.of("access_token", Map.of("type", "string"))); + } + + @Override + protected Map getExpectedFilteredOutput() { + return Map.of( + "access_token", "access_token_response", + "client_id", MoreOAuthParameters.SECRET_MASK); + } + +} diff --git a/docs/integrations/sources/mailchimp.md b/docs/integrations/sources/mailchimp.md index 26aea681ad82..de9e8e250ff2 100644 --- a/docs/integrations/sources/mailchimp.md +++ b/docs/integrations/sources/mailchimp.md @@ -36,9 +36,15 @@ At the time of this writing, [Mailchimp does not impose rate limits](https://mai ### Requirements +For Apikey authorithation: * Mailchimp account * Mailchimp API key +For OAuth authorization: +* Mailchimp registered app +* Mailchimp client_id +* Mailchimp client_secret + ### Setup guide To start syncing Mailchimp data with Airbyte, you'll need two things: @@ -46,10 +52,15 @@ To start syncing Mailchimp data with Airbyte, you'll need two things: 1. Your Mailchimp username. Often this is just the email address or username you use to sign into Mailchimp. 2. A Mailchimp API Key. Follow the [Mailchimp documentation for generating an API key](https://mailchimp.com/help/about-api-keys/). +OR +1. Register an app in [Mailchimp](https://us2.admin.mailchimp.com/account/oauth2/). +2. Specify client_id and client_secret. + ## Changelog | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | +| 0.2.11 | 2021-12-24| [7159](https://github.com/airbytehq/airbyte/pull/7159) | Add oauth2.0 support | | 0.2.10 | 2021-12-21 | [9000](https://github.com/airbytehq/airbyte/pull/9000) | Update connector fields title/description | | 0.2.9 | 2021-12-13 | [7975](https://github.com/airbytehq/airbyte/pull/7975) | Updated JSON schemas | | 0.2.8 | 2021-08-17 | [5481](https://github.com/airbytehq/airbyte/pull/5481) | Remove date-time type from some fields |