From 57a307540c24b59c509c4eb0322e9278f2347b2c Mon Sep 17 00:00:00 2001 From: Christo Grabowski <108154848+ChristoGrab@users.noreply.github.com> Date: Mon, 20 Nov 2023 15:24:12 -0500 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8=20Source=20Mailchimp:=20Add=20Interes?= =?UTF-8?q?ts,=20InterestCategories,=20Tags=20streams=20(#32218)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../integration_tests/configured_catalog.json | 33 +++++++ ...ured_catalog_without_email_activities.json | 33 +++++++ .../integration_tests/expected_records.jsonl | 3 + .../connectors/source-mailchimp/metadata.yaml | 2 +- .../schemas/interest_categories.json | 21 +++++ .../source_mailchimp/schemas/interests.json | 24 +++++ .../source_mailchimp/schemas/tags.json | 15 +++ .../source_mailchimp/source.py | 23 ++++- .../source_mailchimp/streams.py | 72 ++++++++++++++- .../unit_tests/test_source.py | 2 +- .../unit_tests/test_streams.py | 91 ++++++++++++++++++- docs/integrations/sources/mailchimp.md | 4 + 12 files changed, 314 insertions(+), 9 deletions(-) create mode 100644 airbyte-integrations/connectors/source-mailchimp/source_mailchimp/schemas/interest_categories.json create mode 100644 airbyte-integrations/connectors/source-mailchimp/source_mailchimp/schemas/interests.json create mode 100644 airbyte-integrations/connectors/source-mailchimp/source_mailchimp/schemas/tags.json diff --git a/airbyte-integrations/connectors/source-mailchimp/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-mailchimp/integration_tests/configured_catalog.json index b53623609db3..ac0531ed58f3 100644 --- a/airbyte-integrations/connectors/source-mailchimp/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-mailchimp/integration_tests/configured_catalog.json @@ -28,6 +28,28 @@ "primary_key": [["id"]], "destination_sync_mode": "append" }, + { + "stream": { + "name": "interest_categories", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "primary_key": [["id"]], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "interests", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "primary_key": [["id"]], + "destination_sync_mode": "append" + }, { "stream": { "name": "lists", @@ -98,6 +120,17 @@ "primary_key": [["id"]], "destination_sync_mode": "append" }, + { + "stream": { + "name": "tags", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "primary_key": [["id"]], + "destination_sync_mode": "append" + }, { "stream": { "name": "unsubscribes", diff --git a/airbyte-integrations/connectors/source-mailchimp/integration_tests/configured_catalog_without_email_activities.json b/airbyte-integrations/connectors/source-mailchimp/integration_tests/configured_catalog_without_email_activities.json index 96e86a56fdf6..e5d348c8e507 100644 --- a/airbyte-integrations/connectors/source-mailchimp/integration_tests/configured_catalog_without_email_activities.json +++ b/airbyte-integrations/connectors/source-mailchimp/integration_tests/configured_catalog_without_email_activities.json @@ -28,6 +28,28 @@ "primary_key": [["id"]], "destination_sync_mode": "append" }, + { + "stream": { + "name": "interest_categories", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "primary_key": [["id"]], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "interests", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "primary_key": [["id"]], + "destination_sync_mode": "append" + }, { "stream": { "name": "lists", @@ -84,6 +106,17 @@ "primary_key": [["id"]], "destination_sync_mode": "append" }, + { + "stream": { + "name": "tags", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "primary_key": [["id"]], + "destination_sync_mode": "append" + }, { "stream": { "name": "unsubscribes", diff --git a/airbyte-integrations/connectors/source-mailchimp/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-mailchimp/integration_tests/expected_records.jsonl index 96b38c9c28eb..31d306357aae 100644 --- a/airbyte-integrations/connectors/source-mailchimp/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-mailchimp/integration_tests/expected_records.jsonl @@ -1,7 +1,10 @@ {"stream": "campaigns", "data": {"id": "7847cdaeff", "web_id": 13701472, "type": "regular", "create_time": "2023-11-06T20:12:07+00:00", "archive_url": "http://eepurl.com/iDnTtY", "long_archive_url": "https://mailchi.mp/5e0065d29854/invitation-to-unsubscribe", "status": "sent", "emails_sent": 1, "send_time": "2023-11-06T20:17:44+00:00", "content_type": "multichannel", "needs_block_refresh": false, "resendable": false, "recipients": {"list_id": "16d6ec4ffc", "list_is_active": true, "list_name": "Airbyte", "segment_text": "
Contacts that match any of the following conditions:
For a total of 1 emails sent.", "recipient_count": 1, "segment_opts": {"saved_segment_id": 14351532, "match": "any", "conditions": [{"condition_type": "StaticSegment", "field": "static_segment", "op": "static_is", "value": 14351532}]}}, "settings": {"subject_line": "Invitation to Unsubscribe", "title": "Invitation to unsubscribe", "from_name": "yurii", "reply_to": "integration-test+yurii@airbyte.io", "use_conversation": false, "to_name": "", "folder_id": "", "authenticate": true, "auto_footer": false, "inline_css": false, "auto_tweet": false, "fb_comments": true, "timewarp": false, "template_id": 13, "drag_and_drop": false}, "tracking": {"opens": true, "html_clicks": true, "text_clicks": false, "goal_tracking": false, "ecomm360": false, "google_analytics": "", "clicktale": ""}, "report_summary": {"opens": 2, "unique_opens": 1, "open_rate": 1, "clicks": 0, "subscriber_clicks": 0, "click_rate": 0, "ecommerce": {"total_orders": 0, "total_spent": 0, "total_revenue": 0}}, "delivery_status": {"enabled": false}, "_links": [{"rel": "parent", "href": "https://us10.api.mailchimp.com/3.0/campaigns", "method": "GET", "targetSchema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Campaigns/CollectionResponse.json", "schema": "https://us10.api.mailchimp.com/schema/3.0/Paths/Campaigns/Collection.json"}, {"rel": "self", "href": "https://us10.api.mailchimp.com/3.0/campaigns/7847cdaeff", "method": "GET", "targetSchema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Campaigns/Response.json"}, {"rel": "delete", "href": "https://us10.api.mailchimp.com/3.0/campaigns/7847cdaeff", "method": "DELETE"}, {"rel": "send", "href": "https://us10.api.mailchimp.com/3.0/campaigns/7847cdaeff/actions/send", "method": "POST"}, {"rel": "cancel_send", "href": "https://us10.api.mailchimp.com/3.0/campaigns/7847cdaeff/actions/cancel-send", "method": "POST"}, {"rel": "feedback", "href": "https://us10.api.mailchimp.com/3.0/campaigns/7847cdaeff/feedback", "method": "GET", "targetSchema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Campaigns/Feedback/CollectionResponse.json"}, {"rel": "content", "href": "https://us10.api.mailchimp.com/3.0/campaigns/7847cdaeff/content", "method": "GET", "targetSchema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Campaigns/Content/Response.json"}, {"rel": "send_checklist", "href": "https://us10.api.mailchimp.com/3.0/campaigns/7847cdaeff/send-checklist", "method": "GET", "targetSchema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Campaigns/Checklist/Response.json"}, {"rel": "pause", "href": "https://us10.api.mailchimp.com/3.0/campaigns/7847cdaeff/actions/pause", "method": "POST"}, {"rel": "resume", "href": "https://us10.api.mailchimp.com/3.0/campaigns/7847cdaeff/actions/resume", "method": "POST"}, {"rel": "replicate", "href": "https://us10.api.mailchimp.com/3.0/campaigns/7847cdaeff/actions/replicate", "method": "POST"}, {"rel": "create_resend", "href": "https://us10.api.mailchimp.com/3.0/campaigns/7847cdaeff/actions/create-resend", "method": "POST"}]}, "emitted_at": 1699461462352} {"stream": "email_activity", "data": {"campaign_id": "7847cdaeff", "list_id": "16d6ec4ffc", "list_is_active": true, "email_id": "11273c9a5dc6ae6c5aaccfb77b2addfb", "email_address": "AirbyteMailchimpUser@gmail.com", "_links": [{"rel": "parent", "href": "https://us10.api.mailchimp.com/3.0/reports/7847cdaeff/email-activity", "method": "GET", "targetSchema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Reports/EmailActivity/CollectionResponse.json"}, {"rel": "self", "href": "https://us10.api.mailchimp.com/3.0/reports/7847cdaeff/email-activity/11273c9a5dc6ae6c5aaccfb77b2addfb", "method": "GET", "targetSchema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Reports/EmailActivity/Response.json"}, {"rel": "member", "href": "https://us10.api.mailchimp.com/3.0/lists/16d6ec4ffc/members/11273c9a5dc6ae6c5aaccfb77b2addfb", "method": "GET", "targetSchema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Lists/Members/Response.json"}], "action": "open", "timestamp": "2023-11-06T20:17:57+00:00", "ip": "74.125.212.231"}, "emitted_at": 1699457307719} +{"stream": "interests", "data": {"category_id": "a194ba131d", "list_id": "16d6ec4ffc", "id": "bbbb369575", "name": "Donating", "subscriber_count": "0", "display_order": 1}, "emitted_at": 1699963797987} +{"stream": "interest_categories", "data": {"list_id": "16d6ec4ffc", "id": "1bcbe8ba9b", "title": "Product Preferences", "display_order": 0, "type": "checkboxes"}, "emitted_at": 1699963796751} {"stream": "list_members", "data": {"id": "458f50b08c829a8ab901d3f8f88df914", "email_address": "integration-test+Thomas@airbyte.io", "unique_email_id": "42d6d67d11", "contact_id": "475a8f7f7b5087d7be924c9b331c8316", "full_name": "Thomas", "web_id": 546044608, "email_type": "html", "status": "unsubscribed", "unsubscribe_reason": "N/A (Unsubscribed by admin)", "consents_to_one_to_one_messaging": true, "merge_fields": {"FNAME": "Thomas", "LNAME": "", "ADDRESS": "", "PHONE": "", "BIRTHDAY": ""}, "interests": {"bbbb369575": false, "97bbc1227a": false, "d802d794f8": false, "b35e48738e": false, "44d2c158e3": false, "29f73b8209": false, "2010f3c101": false, "75f1cb79fd": false, "aa2fd02c59": false, "f7b60a3c3d": false, "7733d60f61": false, "cc454d76d6": false, "797533254b": false, "9ea08b864b": false, "e2e5fdcac9": false, "8eccc648d6": false, "a7c814599e": false, "20ef45c5d3": false, "1824f5d1a5": false, "644f34517f": false, "c57e1a9ff6": false, "b97fee61c8": false, "b9d16768e3": false, "810348679c": false, "43ebb04472": false, "73ee7c1d1b": false, "045738fa17": false, "0a7cbd4449": false, "fef00a4695": false, "4a19201dc9": false, "571a80ed60": false}, "stats": {"avg_open_rate": 1, "avg_click_rate": 1}, "ip_signup": "", "timestamp_signup": "", "ip_opt": "93.73.161.112", "timestamp_opt": "2022-12-27T08:34:39+00:00", "member_rating": 2, "last_changed": "2023-11-03T20:53:12+00:00", "language": "", "vip": false, "email_client": "", "location": {"latitude": 0, "longitude": 0, "gmtoff": 0, "dstoff": 0, "country_code": "", "timezone": "", "region": ""}, "source": "Import", "tags_count": 0, "tags": [], "list_id": "16d6ec4ffc"}, "emitted_at": 1699302001460} {"stream": "lists", "data": {"id": "16d6ec4ffc", "web_id": 903380, "name": "Airbyte", "contact": {"company": "Airbyte", "address1": "kyiv", "address2": "", "city": "Kiev", "state": "30", "zip": "04200", "country": "UA", "phone": ""}, "permission_reminder": "You are receiving this email because you opted in via our website.", "use_archive_bar": true, "campaign_defaults": {"from_name": "yurii", "from_email": "integration-test+yurii@airbyte.io", "subject": "", "language": "en"}, "notify_on_subscribe": "", "notify_on_unsubscribe": "", "date_created": "2022-12-27T07:56:47+00:00", "list_rating": 0, "email_type_option": false, "subscribe_url_short": "http://eepurl.com/ihg3RD", "subscribe_url_long": "https://airbyte.us10.list-manage.com/subscribe?u=caf9055242d41edd9215d1898&id=16d6ec4ffc", "beamer_address": "us10-d527bd96ba-6d1a9988db@inbound.mailchimp.com", "visibility": "prv", "double_optin": false, "has_welcome": false, "marketing_permissions": false, "modules": [], "stats": {"member_count": 47, "unsubscribe_count": 4, "cleaned_count": 0, "member_count_since_send": 0, "unsubscribe_count_since_send": 1, "cleaned_count_since_send": 0, "campaign_count": 6, "campaign_last_sent": "2022-12-27T08:37:53+00:00", "merge_field_count": 5, "avg_sub_rate": 0, "avg_unsub_rate": 1, "target_sub_rate": 1, "open_rate": 100, "click_rate": 64.70588235294117, "last_sub_date": "2022-12-27T08:34:39+00:00", "last_unsub_date": "2023-11-06T20:18:01+00:00"}, "_links": [{"rel": "self", "href": "https://us10.api.mailchimp.com/3.0/lists/16d6ec4ffc", "method": "GET", "targetSchema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Lists/Response.json"}, {"rel": "parent", "href": "https://us10.api.mailchimp.com/3.0/lists", "method": "GET", "targetSchema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Lists/CollectionResponse.json", "schema": "https://us10.api.mailchimp.com/schema/3.0/Paths/Lists/Collection.json"}, {"rel": "update", "href": "https://us10.api.mailchimp.com/3.0/lists/16d6ec4ffc", "method": "PATCH", "targetSchema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Lists/Response.json", "schema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Lists/PATCH.json"}, {"rel": "batch-sub-unsub-members", "href": "https://us10.api.mailchimp.com/3.0/lists/16d6ec4ffc", "method": "POST", "targetSchema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Lists/BatchPOST-Response.json", "schema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Lists/BatchPOST.json"}, {"rel": "delete", "href": "https://us10.api.mailchimp.com/3.0/lists/16d6ec4ffc", "method": "DELETE"}, {"rel": "abuse-reports", "href": "https://us10.api.mailchimp.com/3.0/lists/16d6ec4ffc/abuse-reports", "method": "GET", "targetSchema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Lists/Abuse/CollectionResponse.json", "schema": "https://us10.api.mailchimp.com/schema/3.0/Paths/Lists/Abuse/Collection.json"}, {"rel": "activity", "href": "https://us10.api.mailchimp.com/3.0/lists/16d6ec4ffc/activity", "method": "GET", "targetSchema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Lists/Activity/Response.json"}, {"rel": "clients", "href": "https://us10.api.mailchimp.com/3.0/lists/16d6ec4ffc/clients", "method": "GET", "targetSchema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Lists/Clients/Response.json"}, {"rel": "growth-history", "href": "https://us10.api.mailchimp.com/3.0/lists/16d6ec4ffc/growth-history", "method": "GET", "targetSchema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Lists/Growth/CollectionResponse.json", "schema": "https://us10.api.mailchimp.com/schema/3.0/Paths/Lists/Growth/Collection.json"}, {"rel": "interest-categories", "href": "https://us10.api.mailchimp.com/3.0/lists/16d6ec4ffc/interest-categories", "method": "GET", "targetSchema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Lists/InterestCategories/CollectionResponse.json", "schema": "https://us10.api.mailchimp.com/schema/3.0/Paths/Lists/InterestCategories/Collection.json"}, {"rel": "members", "href": "https://us10.api.mailchimp.com/3.0/lists/16d6ec4ffc/members", "method": "GET", "targetSchema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Lists/Members/CollectionResponse.json", "schema": "https://us10.api.mailchimp.com/schema/3.0/Paths/Lists/Members/Collection.json"}, {"rel": "merge-fields", "href": "https://us10.api.mailchimp.com/3.0/lists/16d6ec4ffc/merge-fields", "method": "GET", "targetSchema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Lists/MergeFields/CollectionResponse.json", "schema": "https://us10.api.mailchimp.com/schema/3.0/Paths/Lists/MergeFields/Collection.json"}, {"rel": "segments", "href": "https://us10.api.mailchimp.com/3.0/lists/16d6ec4ffc/segments", "method": "GET", "targetSchema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Lists/Segments/CollectionResponse.json", "schema": "https://us10.api.mailchimp.com/schema/3.0/Paths/Lists/Segments/Collection.json"}, {"rel": "webhooks", "href": "https://us10.api.mailchimp.com/3.0/lists/16d6ec4ffc/webhooks", "method": "GET", "targetSchema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Lists/Webhooks/CollectionResponse.json", "schema": "https://us10.api.mailchimp.com/schema/3.0/Paths/Lists/Webhooks/Collection.json"}, {"rel": "signup-forms", "href": "https://us10.api.mailchimp.com/3.0/lists/16d6ec4ffc/signup-forms", "method": "GET", "targetSchema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Lists/SignupForms/CollectionResponse.json", "schema": "https://us10.api.mailchimp.com/schema/3.0/Paths/Lists/SignupForms/Collection.json"}, {"rel": "locations", "href": "https://us10.api.mailchimp.com/3.0/lists/16d6ec4ffc/locations", "method": "GET", "targetSchema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Lists/Locations/CollectionResponse.json", "schema": "https://us10.api.mailchimp.com/schema/3.0/Paths/Lists/Locations/Collection.json"}]}, "emitted_at": 1699626450570} {"stream": "reports", "data": {"id": "a79651273b", "campaign_title": "Untitled", "type": "regular", "list_id": "16d6ec4ffc", "list_is_active": true, "list_name": "Airbyte", "subject_line": "Airbyte Test", "preview_text": "", "emails_sent": 50, "abuse_reports": 0, "unsubscribed": 0, "send_time": "2022-12-27T08:36:55+00:00", "bounces": {"hard_bounces": 0, "soft_bounces": 0, "syntax_errors": 0}, "forwards": {"forwards_count": 0, "forwards_opens": 0}, "opens": {"opens_total": 412, "unique_opens": 50, "open_rate": 1, "last_open": "2023-01-09T10:07:54+00:00"}, "clicks": {"clicks_total": 48, "unique_clicks": 47, "unique_subscriber_clicks": 33, "click_rate": 0.66, "last_click": "2022-12-27T15:28:11+00:00"}, "facebook_likes": {"recipient_likes": 0, "unique_likes": 0, "facebook_likes": 0}, "list_stats": {"sub_rate": 0, "unsub_rate": 1, "open_rate": 100, "click_rate": 64.70588235294117}, "timeseries": [{"timestamp": "2022-12-27T08:00:00+00:00", "emails_sent": 50, "unique_opens": 6, "recipients_clicks": 1}, {"timestamp": "2022-12-27T09:00:00+00:00", "emails_sent": 0, "unique_opens": 43, "recipients_clicks": 0}, {"timestamp": "2022-12-27T10:00:00+00:00", "emails_sent": 0, "unique_opens": 1, "recipients_clicks": 3}, {"timestamp": "2022-12-27T11:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 11}, {"timestamp": "2022-12-27T12:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 10}, {"timestamp": "2022-12-27T13:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 3}, {"timestamp": "2022-12-27T14:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 2}, {"timestamp": "2022-12-27T15:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 3}, {"timestamp": "2022-12-27T16:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2022-12-27T17:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2022-12-27T18:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2022-12-27T19:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2022-12-27T20:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2022-12-27T21:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2022-12-27T22:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2022-12-27T23:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2022-12-28T00:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2022-12-28T01:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2022-12-28T02:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2022-12-28T03:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2022-12-28T04:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2022-12-28T05:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2022-12-28T06:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2022-12-28T07:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}], "ecommerce": {"total_orders": 0, "total_spent": 0, "total_revenue": 0, "currency_code": "USD"}, "delivery_status": {"enabled": false}, "_links": [{"rel": "parent", "href": "https://us10.api.mailchimp.com/3.0/reports", "method": "GET", "targetSchema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Reports/CollectionResponse.json", "schema": "https://us10.api.mailchimp.com/schema/3.0/Paths/Reports/Collection.json"}, {"rel": "self", "href": "https://us10.api.mailchimp.com/3.0/reports/a79651273b", "method": "GET", "targetSchema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Reports/Response.json"}, {"rel": "campaign", "href": "https://us10.api.mailchimp.com/3.0/campaigns/a79651273b", "method": "GET", "targetSchema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Campaigns/Response.json"}, {"rel": "sub-reports", "href": "https://us10.api.mailchimp.com/3.0/reports/a79651273b/sub-reports", "method": "GET", "targetSchema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Reports/Sub/Response.json"}, {"rel": "abuse-reports", "href": "https://us10.api.mailchimp.com/3.0/reports/a79651273b/abuse-reports", "method": "GET", "targetSchema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Reports/Abuse/CollectionResponse.json"}, {"rel": "advice", "href": "https://us10.api.mailchimp.com/3.0/reports/a79651273b/advice", "method": "GET", "targetSchema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Reports/Advice/Response.json"}, {"rel": "open-details", "href": "https://us10.api.mailchimp.com/3.0/reports/a79651273b/open-details", "method": "GET", "targetSchema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Reports/OpenDetails/CollectionResponse.json"}, {"rel": "click-details", "href": "https://us10.api.mailchimp.com/3.0/reports/a79651273b/click-details", "method": "GET", "targetSchema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Reports/ClickDetails/CollectionResponse.json"}, {"rel": "domain-performance", "href": "https://us10.api.mailchimp.com/3.0/reports/a79651273b/domain-performance", "method": "GET", "targetSchema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Reports/DomainPerformance/Response.json"}, {"rel": "eepurl", "href": "https://us10.api.mailchimp.com/3.0/reports/a79651273b/eepurl", "method": "GET", "targetSchema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Reports/Eepurl/CollectionResponse.json"}, {"rel": "email-activity", "href": "https://us10.api.mailchimp.com/3.0/reports/a79651273b/email-activity", "method": "GET", "targetSchema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Reports/EmailActivity/CollectionResponse.json"}, {"rel": "locations", "href": "https://us10.api.mailchimp.com/3.0/reports/a79651273b/locations", "method": "GET", "targetSchema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Reports/Locations/Response.json"}, {"rel": "sent-to", "href": "https://us10.api.mailchimp.com/3.0/reports/a79651273b/sent-to", "method": "GET", "targetSchema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Reports/SentTo/CollectionResponse.json"}, {"rel": "unsubscribed", "href": "https://us10.api.mailchimp.com/3.0/reports/a79651273b/unsubscribed", "method": "GET", "targetSchema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Reports/Unsubs/CollectionResponse.json"}]}, "emitted_at": 1699627079113} {"stream": "segments", "data": {"id": 13506132, "name": "Influencer", "member_count": 3, "type": "static", "created_at": "2022-12-27T08:33:35+00:00", "updated_at": "2022-12-27T08:33:35+00:00", "list_id": "16d6ec4ffc"}, "emitted_at": 1699302003309} +{"stream": "tags", "data": {"id": 13506128, "name": "2022", "list_id": "16d6ec4ffc"}, "emitted_at": 1699963804499} {"stream": "unsubscribes", "data": {"email_id": "11273c9a5dc6ae6c5aaccfb77b2addfb", "email_address": "AirbyteMailchimpUser@gmail.com", "merge_fields": {"FNAME": "Joe", "LNAME": "Barry", "ADDRESS": {"addr1": "109 Barry St", "addr2": "", "city": "Gary", "state": "IN", "zip": "46401", "country": "US"}, "PHONE": "", "BIRTHDAY": ""}, "vip": false, "timestamp": "2023-11-06T20:18:01+00:00", "reason": "Did not signup for list", "campaign_id": "7847cdaeff", "list_id": "16d6ec4ffc", "list_is_active": true}, "emitted_at": 1699302005437} diff --git a/airbyte-integrations/connectors/source-mailchimp/metadata.yaml b/airbyte-integrations/connectors/source-mailchimp/metadata.yaml index ef06fb27bda0..aec91d9fecba 100644 --- a/airbyte-integrations/connectors/source-mailchimp/metadata.yaml +++ b/airbyte-integrations/connectors/source-mailchimp/metadata.yaml @@ -10,7 +10,7 @@ data: connectorSubtype: api connectorType: source definitionId: b03a9f3e-22a5-11eb-adc1-0242ac120002 - dockerImageTag: 0.8.3 + dockerImageTag: 0.9.0 dockerRepository: airbyte/source-mailchimp documentationUrl: https://docs.airbyte.com/integrations/sources/mailchimp githubIssueLabel: source-mailchimp diff --git a/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/schemas/interest_categories.json b/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/schemas/interest_categories.json new file mode 100644 index 000000000000..7d808ecd6ab2 --- /dev/null +++ b/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/schemas/interest_categories.json @@ -0,0 +1,21 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "list_id": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "string"] + }, + "title": { + "type": ["null", "string"] + }, + "display_order": { + "type": ["null", "integer"] + }, + "type": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/schemas/interests.json b/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/schemas/interests.json new file mode 100644 index 000000000000..b936326faa09 --- /dev/null +++ b/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/schemas/interests.json @@ -0,0 +1,24 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "category_id": { + "type": ["null", "string"] + }, + "list_id": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "subscriber_count": { + "type": ["null", "string"] + }, + "display_order": { + "type": ["null", "integer"] + } + } +} diff --git a/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/schemas/tags.json b/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/schemas/tags.json new file mode 100644 index 000000000000..93e81d28f940 --- /dev/null +++ b/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/schemas/tags.json @@ -0,0 +1,15 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "list_id": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/source.py b/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/source.py index f6d4ed7a75aa..4c688d2f557e 100644 --- a/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/source.py +++ b/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/source.py @@ -13,7 +13,19 @@ from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator from requests.auth import AuthBase -from .streams import Automations, Campaigns, EmailActivity, ListMembers, Lists, Reports, Segments, Unsubscribes +from .streams import ( + Automations, + Campaigns, + EmailActivity, + InterestCategories, + Interests, + ListMembers, + Lists, + Reports, + Segments, + Tags, + Unsubscribes, +) class MailChimpAuthenticator: @@ -89,13 +101,20 @@ def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> def streams(self, config: Mapping[str, Any]) -> List[Stream]: authenticator = MailChimpAuthenticator().get_auth(config) campaign_id = config.get("campaign_id") + + lists = Lists(authenticator=authenticator) + interest_categories = InterestCategories(authenticator=authenticator, parent=lists) + return [ Automations(authenticator=authenticator), Campaigns(authenticator=authenticator), EmailActivity(authenticator=authenticator, campaign_id=campaign_id), - Lists(authenticator=authenticator), + interest_categories, + Interests(authenticator=authenticator, parent=interest_categories), + lists, ListMembers(authenticator=authenticator), Reports(authenticator=authenticator), Segments(authenticator=authenticator), + Tags(authenticator=authenticator, parent=lists), Unsubscribes(authenticator=authenticator, campaign_id=campaign_id), ] diff --git a/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/streams.py b/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/streams.py index 27df31f5b05c..1f3d1c63b394 100644 --- a/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/streams.py +++ b/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/streams.py @@ -10,7 +10,7 @@ import requests from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.streams.core import StreamData -from airbyte_cdk.sources.streams.http import HttpStream +from airbyte_cdk.sources.streams.http import HttpStream, HttpSubStream logger = logging.getLogger("airbyte") @@ -260,6 +260,53 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp yield {**item, **activity_item} +class InterestCategories(MailChimpStream, HttpSubStream): + """ + Get information about interest categories for a specific list. + Docs link: https://mailchimp.com/developer/marketing/api/interest-categories/list-interest-categories/ + """ + + data_field = "categories" + + def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: + """ + Get the list_id from the parent stream slice and use it to construct the path. + """ + list_id = stream_slice.get("parent").get("id") + return f"lists/{list_id}/interest-categories" + + def request_params(self, **kwargs): + + # Exclude the _links field, as it is not user-relevant data + params = super().request_params(**kwargs) + params["exclude_fields"] = "categories._links" + return params + + +class Interests(MailChimpStream, HttpSubStream): + """ + Get a list of interests for a specific interest category. + Docs link: https://mailchimp.com/developer/marketing/api/interests/list-interests-in-category/ + """ + + data_field = "interests" + + def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: + """ + Get the list_id from the parent stream slice and use it to construct the path. + """ + list_id = stream_slice.get("parent").get("list_id") + category_id = stream_slice.get("parent").get("id") + return f"lists/{list_id}/interest-categories/{category_id}/interests" + + def request_params(self, **kwargs): + + # Exclude the _links field, as it is not user-relevant data + params = super().request_params(**kwargs) + params["exclude_fields"] = "interests._links" + return params + + class ListMembers(MailChimpListSubStream): """ Get information about members in a specific Mailchimp list. @@ -308,6 +355,29 @@ class Segments(MailChimpListSubStream): data_field = "segments" +class Tags(MailChimpStream, HttpSubStream): + """ + Get information about tags for a specific list. + Docs link: https://mailchimp.com/developer/marketing/api/list-tags/list-tags-for-list/ + """ + + data_field = "tags" + + def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: + list_id = stream_slice.get("parent").get("id") + return f"lists/{list_id}/tag-search" + + def parse_response(self, response: requests.Response, stream_slice, **kwargs) -> Iterable[Mapping]: + """ + Tags do not reference parent_ids, so we need to add the list_id to each record. + """ + response = super().parse_response(response, **kwargs) + + for record in response: + record["list_id"] = stream_slice.get("parent").get("id") + yield record + + class Unsubscribes(IncrementalMailChimpStream): """ List of members who have unsubscribed from a specific campaign. diff --git a/airbyte-integrations/connectors/source-mailchimp/unit_tests/test_source.py b/airbyte-integrations/connectors/source-mailchimp/unit_tests/test_source.py index 0e56d333da62..ed5d06630c21 100644 --- a/airbyte-integrations/connectors/source-mailchimp/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-mailchimp/unit_tests/test_source.py @@ -90,4 +90,4 @@ def test_wrong_config(wrong_config): def test_streams_count(config): streams = SourceMailchimp().streams(config) - assert len(streams) == 8 + assert len(streams) == 11 diff --git a/airbyte-integrations/connectors/source-mailchimp/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-mailchimp/unit_tests/test_streams.py index 094eb4fe0bf5..e39f27b0aa70 100644 --- a/airbyte-integrations/connectors/source-mailchimp/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-mailchimp/unit_tests/test_streams.py @@ -10,7 +10,19 @@ import responses from airbyte_cdk.models import SyncMode from requests.exceptions import HTTPError -from source_mailchimp.streams import Campaigns, EmailActivity, ListMembers, Lists, Reports, Segments +from source_mailchimp.streams import ( + Automations, + Campaigns, + EmailActivity, + InterestCategories, + Interests, + ListMembers, + Lists, + Reports, + Segments, + Tags, + Unsubscribes, +) from utils import read_full_refresh, read_incremental @@ -58,21 +70,43 @@ def test_next_page_token(auth): @pytest.mark.parametrize( - "inputs, expected_params", + "stream, inputs, expected_params", [ ( + Lists, {"stream_slice": None, "stream_state": None, "next_page_token": None}, {"count": 1000, "sort_dir": "ASC", "sort_field": "date_created"}, ), ( + Lists, {"stream_slice": None, "stream_state": None, "next_page_token": {"offset": 1000}}, {"count": 1000, "sort_dir": "ASC", "sort_field": "date_created", "offset": 1000}, ), + ( + InterestCategories, + {"stream_slice": {"parent": {"id": "123"}}, "stream_state": None, "next_page_token": None}, + {"count": 1000, "exclude_fields": "categories._links"}, + ), + ( + Interests, + {"stream_slice": {"parent": {"id": "123"}}, "stream_state": None, "next_page_token": {"offset": 2000}}, + {"count": 1000, "exclude_fields": "interests._links", "offset": 2000}, + ), + ], + ids=[ + "Lists: no next_page_token or state to add to request params", + "Lists: next_page_token added to request params", + "InterestCategories: no next_page_token to add to request params", + "Interests: next_page_token added to request params", ], ) -def test_request_params(auth, inputs, expected_params): +def test_request_params(auth, stream, inputs, expected_params): args = {"authenticator": auth} - stream = Lists(**args) + if stream == InterestCategories: + args["parent"] = Lists(**args) + elif stream == Interests: + args["parent"] = InterestCategories(authenticator=auth, parent=Lists(authenticator=auth)) + stream = stream(**args) assert stream.request_params(**inputs) == expected_params @@ -414,6 +448,54 @@ def test_403_error_handling( except HTTPError as e: assert e.response.status_code == status_code + +@pytest.mark.parametrize( + "stream, stream_slice, expected_endpoint", + [ + (Automations, {}, "automations"), + (Lists, {}, "lists"), + (Campaigns, {}, "campaigns"), + (EmailActivity, {"campaign_id": "123"}, "reports/123/email-activity"), + (InterestCategories, {"parent": {"id": "123"}}, "lists/123/interest-categories"), + (Interests, {"parent": {"list_id": "123", "id": "456"}}, "lists/123/interest-categories/456/interests"), + (ListMembers, {"list_id": "123"}, "lists/123/members"), + (Reports, {}, "reports"), + (Segments, {"list_id": "123"}, "lists/123/segments"), + (Tags, {"parent": {"id": "123"}}, "lists/123/tag-search"), + (Unsubscribes, {"campaign_id": "123"}, "reports/123/unsubscribed"), + ], + ids=[ + "Automations", + "Lists", + "Campaigns", + "EmailActivity", + "InterestCategories", + "Interests", + "ListMembers", + "Reports", + "Segments", + "Tags", + "Unsubscribes", + ], +) +def test_path(auth, stream, stream_slice, expected_endpoint): + """ + Test the path method for each stream. + """ + + # Add parent stream where necessary + if stream is InterestCategories or stream is Tags: + stream = stream(authenticator=auth, parent=Lists(authenticator=auth)) + elif stream is Interests: + stream = stream(authenticator=auth, parent=InterestCategories(authenticator=auth, parent=Lists(authenticator=auth))) + else: + stream = stream(authenticator=auth) + + endpoint = stream.path(stream_slice=stream_slice) + + assert endpoint == expected_endpoint, f"Stream {stream}: expected path '{expected_endpoint}', got '{endpoint}'" + + @pytest.mark.parametrize( "record, expected_return", [ @@ -449,3 +531,4 @@ def test_reports_remove_empty_datetime_fields(auth, record, expected_return): """ stream = Reports(authenticator=auth) assert stream.remove_empty_datetime_fields(record) == expected_return, f"Expected: {expected_return}, Actual: {stream.remove_empty_datetime_fields(record)}" + diff --git a/docs/integrations/sources/mailchimp.md b/docs/integrations/sources/mailchimp.md index 20523890da5e..9106108da3fd 100644 --- a/docs/integrations/sources/mailchimp.md +++ b/docs/integrations/sources/mailchimp.md @@ -41,10 +41,13 @@ The Mailchimp source connector supports the following streams: [Automations](https://mailchimp.com/developer/marketing/api/automation/list-automations/) [Campaigns](https://mailchimp.com/developer/marketing/api/campaigns/get-campaign-info/) [Email Activity](https://mailchimp.com/developer/marketing/api/email-activity-reports/list-email-activity/) +[Interests](https://mailchimp.com/developer/marketing/api/interests/list-interests-in-category/) +[InterestCategories](https://mailchimp.com/developer/marketing/api/interest-categories/list-interest-categories/) [Lists](https://mailchimp.com/developer/api/marketing/lists/get-list-info) [ListMembers](https://mailchimp.com/developer/marketing/api/list-members/list-members-info/) [Reports](https://mailchimp.com/developer/marketing/api/reports/list-campaign-reports/) [Segments](https://mailchimp.com/developer/marketing/api/list-segments/list-segments/) +[Tags](https://mailchimp.com/developer/marketing/api/lists-tags-search/search-for-tags-on-a-list-by-name/) [Unsubscribes](https://mailchimp.com/developer/marketing/api/unsub-reports/list-unsubscribed-members/) ### A note on primary keys @@ -76,6 +79,7 @@ Now that you have set up the Mailchimp source connector, check out the following | Version | Date | Pull Request | Subject | |---------|------------|----------------------------------------------------------|----------------------------------------------------------------------------| +| 0.9.0 | 2023-11-17 | [32218](https://github.com/airbytehq/airbyte/pull/32218) | Add Interests, InterestCategories, Tags streams | | 0.8.3 | 2023-11-15 | [32543](https://github.com/airbytehq/airbyte/pull/32543) | Handle empty datetime fields in Reports stream | | 0.8.2 | 2023-11-13 | [32466](https://github.com/airbytehq/airbyte/pull/32466) | Improve error handling during connection check | | 0.8.1 | 2023-11-06 | [32226](https://github.com/airbytehq/airbyte/pull/32226) | Unmute expected records test after data anonymisation |