Source Mailchimp - fix the way request params are built (#20765)

* 1058 source mailchimp - fix the way request params are built * auto-bump connector version Co-authored-by: Octavia Squidington III <octavia-squidington-iii@users.noreply.github.com>
airbytehq · Jan 13, 2023 · 4c60cb5 · 4c60cb5
1 parent de8a177
commit 4c60cb5
Show file tree

Hide file tree

Showing 15 changed files with 160 additions and 141 deletions.
diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml
@@ -898,7 +898,7 @@
 - name: Mailchimp
   sourceDefinitionId: b03a9f3e-22a5-11eb-adc1-0242ac120002
   dockerRepository: airbyte/source-mailchimp
-  dockerImageTag: 0.3.0
+  dockerImageTag: 0.3.1
   documentationUrl: https://docs.airbyte.com/integrations/sources/mailchimp
   icon: mailchimp.svg
   sourceType: api

diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml
@@ -7662,7 +7662,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-mailchimp:0.3.0"
+- dockerImage: "airbyte/source-mailchimp:0.3.1"
   spec:
     documentationUrl: "https://docs.airbyte.com/integrations/sources/mailchimp"
     connectionSpecification:
@@ -7718,6 +7718,10 @@
                 description: "Mailchimp API Key. See the <a href=\"https://docs.airbyte.com/integrations/sources/mailchimp\"\
                   >docs</a> for information on how to generate this key."
                 airbyte_secret: true
+        campaign_id:
+          type: "string"
+          title: "ID of a campaign to sync email activities"
+          airbyte_hidden: true
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []

diff --git a/airbyte-integrations/connectors/source-mailchimp/Dockerfile b/airbyte-integrations/connectors/source-mailchimp/Dockerfile
@@ -4,13 +4,13 @@ FROM python:3.9-slim
 RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/*
 
 WORKDIR /airbyte/integration_code
-COPY source_mailchimp ./source_mailchimp
-COPY main.py ./
 COPY setup.py ./
 RUN pip install .
+COPY source_mailchimp ./source_mailchimp
+COPY main.py ./
 
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.3.0
+LABEL io.airbyte.version=0.3.1
 LABEL io.airbyte.name=airbyte/source-mailchimp
diff --git a/airbyte-integrations/connectors/source-mailchimp/acceptance-test-config.yml b/airbyte-integrations/connectors/source-mailchimp/acceptance-test-config.yml
@@ -1,66 +1,57 @@
 connector_image: airbyte/source-mailchimp:dev
-tests:
+test_strictness_level: high
+acceptance_tests:
   spec:
-    - spec_path: "source_mailchimp/spec.json"
-      timeout_seconds: 60
+    tests:
+      - spec_path: "source_mailchimp/spec.json"
   connection:
-    # for old spec config (without oneOf)
-    - config_path: "secrets/config.json"
-      status: "succeed"
-      timeout_seconds: 180
-    # for auth with API token
-    - config_path: "secrets/config_apikey.json"
-      status: "succeed"
-      timeout_seconds: 180
-    # for auth with oauth2 token
-    - config_path: "secrets/config_oauth.json"
-      status: "succeed"
-      timeout_seconds: 180
-    - config_path: "integration_tests/invalid_config.json"
-      status: "failed"
-      timeout_seconds: 180
-    - config_path: "integration_tests/invalid_config_apikey.json"
-      status: "failed"
-      timeout_seconds: 180
-    - config_path: "integration_tests/invalid_config_oauth.json"
-      status: "failed"
-      timeout_seconds: 180
+    tests:
+      # for old spec config (without oneOf)
+      - config_path: "secrets/config.json"
+        status: "succeed"
+      # for auth with API token
+      - config_path: "secrets/config_apikey.json"
+        status: "succeed"
+      # for auth with oauth2 token
+      - config_path: "secrets/config_oauth.json"
+        status: "succeed"
+      - config_path: "integration_tests/invalid_config.json"
+        status: "failed"
+      - config_path: "integration_tests/invalid_config_apikey.json"
+        status: "failed"
+      - config_path: "integration_tests/invalid_config_oauth.json"
+        status: "failed"
   discovery:
-    # for old spec config (without oneOf)
-    - config_path: "secrets/config.json"
-    # for auth with API token
-    - config_path: "secrets/config_apikey.json"
-    # for auth with oauth2 token
-    - config_path: "secrets/config_oauth.json"
+    tests:
+      # for old spec config (without oneOf)
+      - config_path: "secrets/config.json"
+      # for auth with API token
+      - config_path: "secrets/config_apikey.json"
+      # for auth with oauth2 token
+      - config_path: "secrets/config_oauth.json"
   basic_read:
-    - config_path: "secrets/config.json"
-      configured_catalog_path: "integration_tests/configured_catalog.json"
-      timeout_seconds: 1800
-    - config_path: "secrets/config_oauth.json"
-      configured_catalog_path: "integration_tests/configured_catalog.json"
-      timeout_seconds: 1800
-#  THIS TEST IS COMMENTED OUT. Tests are supposed to accept
-#  `state = {cursor_field: value}`. When we have dependent endpoint path
-#  `path_begin/{some_id}/path_end` we need a complex state like below:
-#  `{"id1": {cursor_field: value}, "id2": {cursor_field: value}...}`
-#  The test currently is not supposed to accept this desired construction,
-#  so it is commented out
-
-#  incremental:
-#    - config_path: "secrets/config.json"
-#      configured_catalog_path: "integration_tests/configured_catalog.json"
-#      future_state_path: "integration_tests/state.json"
-#      cursor_paths:
-#        lists: [ "date_created" ]
-#        campaigns: [ "create_time" ]
-#        Email_activity: [ "timestamp" ]
-
+    tests:
+      - config_path: "secrets/config.json"
+        expect_records:
+          bypass_reason: "Risk to disclose internal data. Need to set up a sandbox account - https://github.com/airbytehq/airbyte/issues/20726"
+      - config_path: "secrets/config_oauth.json"
+        expect_records:
+          bypass_reason: "Risk to disclose internal data. Need to set up a sandbox account - https://github.com/airbytehq/airbyte/issues/20726"
+  incremental:
+    tests:
+      - config_path: "secrets/config.json"
+        configured_catalog_path: "integration_tests/configured_catalog.json"
+        future_state:
+          future_state_path: "integration_tests/state.json"
+        cursor_paths:
+          lists: ["date_created"]
+          campaigns: ["create_time"]
+          email_activity: ["49d68626f3", "timestamp"]
   # Email activities stream has working campaigns with email newsletters.
   # Due to this sequential_reads test could be failed.
   full_refresh:
-    - config_path: "secrets/config.json"
-      configured_catalog_path: "integration_tests/configured_catalog_without_email_activities.json"
-      timeout_seconds: 1800
-    - config_path: "secrets/config_oauth.json"
-      configured_catalog_path: "integration_tests/configured_catalog_without_email_activities.json"
-      timeout_seconds: 1800
+    tests:
+      - config_path: "secrets/config.json"
+        configured_catalog_path: "integration_tests/configured_catalog_without_email_activities.json"
+      - config_path: "secrets/config_oauth.json"
+        configured_catalog_path: "integration_tests/configured_catalog_without_email_activities.json"
diff --git a/airbyte-integrations/connectors/source-mailchimp/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-mailchimp/integration_tests/configured_catalog.json
@@ -11,6 +11,7 @@
       },
       "sync_mode": "incremental",
       "cursor_field": ["create_time"],
+      "primary_key": [["id"]],
       "destination_sync_mode": "append"
     },
     {
@@ -22,20 +23,23 @@
         "default_cursor_field": ["date_created"],
         "source_defined_primary_key": [["id"]]
       },
-      "sync_mode": "incremental",
+        "sync_mode": "incremental",
       "cursor_field": ["date_created"],
-      "destination_sync_mode": "append"
+      "destination_sync_mode": "append",
+      "primary_key": [["id"]]
     },
     {
       "stream": {
         "name": "email_activity",
         "json_schema": {},
         "supported_sync_modes": ["incremental", "full_refresh"],
         "source_defined_cursor": true,
-        "default_cursor_field": ["timestamp"]
+        "default_cursor_field": ["timestamp"],
+        "source_defined_primary_key": [["timestamp"], ["email_id"], ["action"]]
       },
       "sync_mode": "incremental",
       "cursor_field": ["timestamp"],
+      "primary_key": [["timestamp"], ["email_id"], ["action"]],
       "destination_sync_mode": "append"
     }
   ]

diff --git a/...ctors/source-mailchimp/integration_tests/configured_catalog_without_email_activities.json b/...ctors/source-mailchimp/integration_tests/configured_catalog_without_email_activities.json
@@ -11,6 +11,7 @@
       },
       "sync_mode": "incremental",
       "cursor_field": ["create_time"],
+      "primary_key": [["id"]],
       "destination_sync_mode": "append"
     },
     {
@@ -24,6 +25,7 @@
       },
       "sync_mode": "incremental",
       "cursor_field": ["date_created"],
+      "primary_key": [["id"]],
       "destination_sync_mode": "append"
     }
   ]

diff --git a/airbyte-integrations/connectors/source-mailchimp/integration_tests/state.json b/airbyte-integrations/connectors/source-mailchimp/integration_tests/state.json
@@ -2,22 +2,22 @@
   {
     "type": "STREAM",
     "stream": {
-      "stream_state": { "create_time": "2020-11-23T05:42:11+00:00" },
+      "stream_state": { "create_time": "2220-11-23T05:42:11+00:00" },
       "stream_descriptor": { "name": "campaigns" }
     }
   },
   {
     "type": "STREAM",
     "stream": {
-      "stream_state": { "date_created": "2020-09-25T04:47:31+00:00" },
+      "stream_state": { "date_created": "2220-09-25T04:47:31+00:00" },
       "stream_descriptor": { "name": "lists" }
     }
   },
   {
     "type": "STREAM",
     "stream": {
       "stream_state": {
-        "49d68626f3": { "timestamp": "2020-11-23T05:42:10+00:00" }
+        "49d68626f3": { "timestamp": "2220-11-23T05:42:10+00:00" }
       },
       "stream_descriptor": { "name": "email_activity" }
     }

diff --git a/airbyte-integrations/connectors/source-mailchimp/setup.py b/airbyte-integrations/connectors/source-mailchimp/setup.py
@@ -15,7 +15,7 @@
     author_email="contact@airbyte.io",
     packages=find_packages(),
     install_requires=[
-        "airbyte-cdk~=0.1",
+        "airbyte-cdk",
         "pytest~=6.1",
     ],
     package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]},

diff --git a/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/models/__init__.py b/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/models/__init__.py
diff --git a/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/models/mailchimp.py b/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/models/mailchimp.py
diff --git a/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/source.py b/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/source.py
@@ -63,4 +63,9 @@ def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) ->
 
     def streams(self, config: Mapping[str, Any]) -> List[Stream]:
         authenticator = MailChimpAuthenticator().get_auth(config)
-        return [Lists(authenticator=authenticator), Campaigns(authenticator=authenticator), EmailActivity(authenticator=authenticator)]
+        campaign_id = config.get("campaign_id")
+        return [
+            Lists(authenticator=authenticator),
+            Campaigns(authenticator=authenticator),
+            EmailActivity(authenticator=authenticator, campaign_id=campaign_id),
+        ]
diff --git a/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/spec.json b/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/spec.json
@@ -60,6 +60,11 @@
             }
           }
         ]
+      },
+      "campaign_id": {
+        "type": "string",
+        "title": "ID of a campaign to sync email activities",
+        "airbyte_hidden": true
       }
     }
   },