🎉 Source Pinterest - new streams

airbyte-integrations/connectors/source-pinterest/acceptance-test-config.yml

@@ -12,7 +12,7 @@ acceptance_tests:
       - config_path: secrets/config.json
         status: succeed
       - config_path: integration_tests/invalid_config.json
-        status: exception
+        status: failed
       - config_path: secrets/config_oauth.json
         status: succeed
   discovery:
@@ -23,26 +23,24 @@ acceptance_tests:
     tests:
       - config_path: secrets/config.json
         empty_streams:
-          - bypass_reason: The stream could return 0 records, because of low rate-limits
-            name: ad_account_analytics
-          - bypass_reason: The stream could return 0 records, because of low rate-limits
-            name: ad_analytics
-          - bypass_reason: The stream could return 0 records, because of low rate-limits
-            name: ad_group_analytics
-          - bypass_reason: The stream could return 0 records, because of low rate-limits
-            name: ad_groups
-          - bypass_reason: The stream could return 0 records, because of low rate-limits
-            name: ads
-          - bypass_reason: The stream could return 0 records, because of low rate-limits
-            name: board_section_pins
-          - bypass_reason: The stream could return 0 records, because of low rate-limits
-            name: board_sections
-          - bypass_reason: The stream could return 0 records, because of low rate-limits
-            name: campaign_analytics
-          - bypass_reason: The stream could return 0 records, because of low rate-limits
-            name: campaigns
-          - bypass_reason: The stream could return 0 records, because of low rate-limits
-            name: user_account_analytics
+          - name: conversion_tags
+            bypass_reason: Not possible to add data
+          - name: customer_lists
+            bypass_reason: Not possible to add data
+          - name: catalogs
+            bypass_reason: Not possible to add data
+          - name: catalogs_feeds
+            bypass_reason: Not possible to add data
+          - name: catalogs_product_groups
+            bypass_reason: Not possible to add data
+          - name: product_group_report
+            bypass_reason: Not possible to add data
+          - name: product_group_targeting_report
+            bypass_reason: Not possible to add data
+          - name: product_item_report
+            bypass_reason: Not possible to add data
+          - name: keyword_report
+            bypass_reason: Not possible to add data
         timeout_seconds: 1200
         expect_records:
           path: "integration_tests/expected_records.jsonl"
@@ -52,8 +50,27 @@ acceptance_tests:
         fail_on_extra_columns: false
         ignored_fields:
           board_pins:
-            - name: "links"
-              bypass_reason: "because it contains non-secure http link, which leads to failed QA tests"
+            - name: "media"
+              bypass_reason: "urls may change"
+          board_section_pins:
+            - name: "media"
+              bypass_reason: "urls may change"
+          ads:
+            - name: "updated_time"
+              bypass_reason: "can be updated"
+          ad_groups:
+            - name: "updated_time"
+              bypass_reason: "can be updated"
+          campaigns:
+            - name: "updated_time"
+              bypass_reason: "can be updated"
+          audiences:
+            - name: "size"
+              bypass_reason: "can be changed"
+            - name: "updated_timestamp"
+              bypass_reason: "can be changed"
+            - name: "created_timestamp"
+              bypass_reason: "can be changed"
   incremental:
     tests:
       - config_path: secrets/config.json

airbyte-integrations/connectors/source-pinterest/integration_tests/expected_records.jsonl

@@ -1,4 +1,23 @@
 {"stream": "ad_accounts", "data": {"id": "549761668032", "name": "Airbyte", "owner": {"username": "integrationtest0375", "id": "666744057242074926"}, "country": "US", "currency": "USD", "permissions": ["OWNER"], "created_time": 1603772920, "updated_time": 1623173784}, "emitted_at": 1688461289470}
+{"stream": "ad_account_analytics", "data": {"TOTAL_IMPRESSION_FREQUENCY": 1.0, "TOTAL_IMPRESSION_USER": 1.0, "ADVERTISER_ID": "549761668032", "DATE": "2023-10-29", "IMPRESSION_2": 1.0, "AD_ACCOUNT_ID": "549761668032"}, "emitted_at": 1699893121669}
+{"stream": "ads", "data": {"id": "687218400118", "ad_group_id": "2680068678965", "ad_account_id": "549761668032", "android_deep_link": null, "campaign_id": "626744128956", "carousel_android_deep_links": null, "carousel_destination_urls": null, "carousel_ios_deep_links": null, "click_tracking_url": null, "collection_items_destination_url_template": null, "created_time": 1623245885, "creative_type": "REGULAR", "destination_url": "https://airbyte.io/", "ios_deep_link": null, "is_pin_deleted": false, "is_removable": false, "name": "2021-06-09 | Traffic | Keywords | Data Integration", "pin_id": "666743919837294988", "rejected_reasons": [], "rejection_labels": [], "review_status": "APPROVED", "status": "PAUSED", "summary_status": "PAUSED", "tracking_urls": null, "type": "ad", "updated_time": 1699373013, "view_tracking_url": null, "lead_form_id": null}, "emitted_at": 1699393433303}
+{"stream": "ad_analytics", "data": {"PIN_ID": 6.66743919837295e+17, "AD_GROUP_ID": "2680068678993", "AD_GROUP_ENTITY_STATUS": "1", "CAMPAIGN_ENTITY_STATUS": 1.0, "TOTAL_IMPRESSION_FREQUENCY": 1.0, "CAMPAIGN_LIFETIME_SPEND_CAP": 0.0, "TOTAL_IMPRESSION_USER": 1.0, "CAMPAIGN_DAILY_SPEND_CAP": 25000000.0, "AD_ID": "687218400210", "ADVERTISER_ID": "549761668032", "PIN_PROMOTION_ID": 687218400210.0, "DATE": "2023-10-29", "IMPRESSION_2": 1.0, "AD_ACCOUNT_ID": "549761668032", "CAMPAIGN_ID": 626744128982.0, "CAMPAIGN_NAME": "2021-06-08 09:08 UTC | Brand awareness"}, "emitted_at": 1699893196846}
+{"stream": "ad_groups", "data": {"id": "2680068678965", "created_time": 1623245885.0, "updated_time": 1699373013.0, "start_time": null, "end_time": null, "bid_in_micro_currency": null, "budget_in_micro_currency": null, "campaign_id": "626744128956", "ad_account_id": "549761668032", "auto_targeting_enabled": true, "type": "adgroup", "budget_type": "CBO_ADGROUP", "billable_event": "CLICKTHROUGH", "status": "ACTIVE", "lifetime_frequency_cap": -1.0, "targeting_spec": {"GENDER": ["female", "male", "unknown"], "APPTYPE": ["web", "web_mobile", "iphone", "ipad", "android_mobile", "android_tablet"], "LOCALE": ["cs", "da", "de", "el", "en", "es", "fi", "fr", "hu", "id", "it", "ja", "ko", "nb", "nl", "pl", "pt", "ro", "ru", "sk", "sv", "tr", "uk", "zh"], "TARGETING_STRATEGY": ["CHOOSE_YOUR_OWN"], "LOCATION": ["US"]}, "name": "2021-06-09 | Traffic | Keywords | Data Integration", "placement_group": "ALL", "pacing_delivery_type": "STANDARD", "tracking_urls": null, "conversion_learning_mode_type": null, "summary_status": "COMPLETED", "feed_profile_id": "0", "placement_traffic_type": null, "optimization_goal_metadata": {}, "bid_strategy_type": "AUTOMATIC_BID"}, "emitted_at": 1699393433712}
+{"stream": "ad_group_analytics", "data": {"AD_GROUP_ID": "2680068678993", "AD_GROUP_ENTITY_STATUS": "1", "CAMPAIGN_ENTITY_STATUS": 1.0, "TOTAL_IMPRESSION_FREQUENCY": 1.0, "CAMPAIGN_LIFETIME_SPEND_CAP": 0.0, "TOTAL_IMPRESSION_USER": 1.0, "CAMPAIGN_DAILY_SPEND_CAP": 25000000.0, "ADVERTISER_ID": "549761668032", "DATE": "2023-10-29", "IMPRESSION_2": 1.0, "AD_ACCOUNT_ID": "549761668032", "CAMPAIGN_ID": 626744128982.0, "CAMPAIGN_NAME": "2021-06-08 09:08 UTC | Brand awareness"}, "emitted_at": 1699893280169}
 {"stream": "boards", "data": {"media": {"pin_thumbnail_urls": [], "image_cover_url": "https://i.pinimg.com/400x300/c6/b6/0d/c6b60d6b5f2ec04db7748d35fb1a8004.jpg"}, "owner": {"username": "integrationtest0375"}, "created_at": "2021-06-08T09:37:18", "board_pins_modified_at": "2021-10-25T11:17:56.715000", "id": "666743988523388559", "collaborator_count": 0, "follower_count": 2, "pin_count": 1, "privacy": "PUBLIC", "name": "business", "description": ""}, "emitted_at": 1680356853019}
-{"stream": "board_pins", "data": {"description": "Data Integration", "board_owner": {"username": "integrationtest0375"}, "product_tags": [], "has_been_promoted": true, "created_at": "2021-06-08T09:37:30", "board_id": "666743988523388559", "note": "", "creative_type": "REGULAR", "parent_pin_id": null, "title": "Airbyte", "alt_text": null, "pin_metrics": null, "dominant_color": "#cacafe", "id": "666743919837294988", "is_owner": true, "board_section_id": "5195034916661798218", "link": "http://airbyte.io/", "media": {"media_type": "image", "images": {"150x150": {"width": 150, "height": 150, "url": "https://i.pinimg.com/150x150/c6/b6/0d/c6b60d6b5f2ec04db7748d35fb1a8004.jpg"}, "400x300": {"width": 400, "height": 300, "url": "https://i.pinimg.com/400x300/c6/b6/0d/c6b60d6b5f2ec04db7748d35fb1a8004.jpg"}, "600x": {"width": 564, "height": 337, "url": "https://i.pinimg.com/564x/c6/b6/0d/c6b60d6b5f2ec04db7748d35fb1a8004.jpg"}, "1200x": {"width": 1200, "height": 718, "url": "https://i.pinimg.com/1200x/c6/b6/0d/c6b60d6b5f2ec04db7748d35fb1a8004.jpg"}}}, "is_standard": true}, "emitted_at": 1698398201666}
-{"stream": "campaign_analytics_report", "data": {"ADVERTISER_ID": 549761668032.0, "AD_ACCOUNT_ID": "549761668032", "CAMPAIGN_DAILY_SPEND_CAP": 750000.0, "CAMPAIGN_ENTITY_STATUS": "ACTIVE", "CAMPAIGN_ID": 626744128982.0, "CAMPAIGN_LIFETIME_SPEND_CAP": 0.0, "CAMPAIGN_NAME": "2021-06-08 09:08 UTC | Brand awareness", "IMPRESSION_2": 3.0, "TOTAL_IMPRESSION_FREQUENCY": 1.5, "TOTAL_IMPRESSION_USER": 2.0, "DATE": "2023-07-14"}, "emitted_at": 1690299367301}
+{"stream": "board_pins", "data": {"description": "Data Integration", "board_owner": {"username": "integrationtest0375"}, "product_tags": [], "has_been_promoted": true,"link":"http://airbyte.io/", "created_at": "2021-06-08T09:37:30", "board_id": "666743988523388559", "note": "", "creative_type": "REGULAR", "parent_pin_id": null, "title": "Airbyte", "alt_text": null, "pin_metrics": null, "dominant_color": "#cacafe", "id": "666743919837294988", "is_owner": true, "board_section_id": "5195034916661798218", "is_standard": true}, "emitted_at": 1698398201666}
+{"stream": "board_sections", "data": {"name": "Airbyte_board_section_new", "id": "5195035116725909603"}, "emitted_at": 1699893323493}
+{"stream": "board_section_pins","data":{"id":"666743919837294988","dominant_color":"#cacafe","pin_metrics":null,"title":"Airbyte","creative_type":"REGULAR","link":"http://airbyte.io/","board_id":"666743988523388559","created_at":"2021-06-08T09:37:30","is_owner":true,"description":"Data Integration","note":"","alt_text":null,"board_section_id":"5195034916661798218","parent_pin_id":null,"product_tags":[],"board_owner":{"username":"integrationtest0375"},"is_standard":true,"has_been_promoted":true},"emitted_at":1699893364884}
+{"stream": "campaigns", "data": {"id": "626744128956", "ad_account_id": "549761668032", "name": "2021-06-09 | Traffic | Keywords | Data Integration", "status": "ACTIVE", "objective_type": "CONSIDERATION", "lifetime_spend_cap": 0, "daily_spend_cap": 3000000, "order_line_id": null, "tracking_urls": null, "created_time": 1623245885, "updated_time": 1691447502, "type": "campaign", "is_flexible_daily_budgets": false, "summary_status": "COMPLETED", "is_campaign_budget_optimization": true, "start_time": 1623196800, "end_time": 1624060800}, "emitted_at": 1699393571700}
+{"stream": "campaign_analytics", "data": {"TOTAL_IMPRESSION_FREQUENCY": 1.0, "CAMPAIGN_LIFETIME_SPEND_CAP": 0.0, "TOTAL_IMPRESSION_USER": 1.0, "CAMPAIGN_ENTITY_STATUS": 1.0, "CAMPAIGN_DAILY_SPEND_CAP": 25000000.0, "ADVERTISER_ID": 549761668032.0, "DATE": "2023-10-29", "IMPRESSION_2": 1.0, "AD_ACCOUNT_ID": "549761668032", "CAMPAIGN_ID": 626744128982.0, "CAMPAIGN_NAME": "2021-06-08 09:08 UTC | Brand awareness"}, "emitted_at": 1699894065462}
+{"stream": "campaign_analytics_report", "data": {"ADVERTISER_ID": 549761668032.0, "AD_ACCOUNT_ID": "549761668032", "CAMPAIGN_DAILY_SPEND_CAP": 25000000.0, "CAMPAIGN_ENTITY_STATUS": "ACTIVE", "CAMPAIGN_ID": 626744128982.0, "CAMPAIGN_LIFETIME_SPEND_CAP": 0.0, "CAMPAIGN_NAME": "2021-06-08 09:08 UTC | Brand awareness", "IMPRESSION_2": 3.0, "TOTAL_IMPRESSION_FREQUENCY": 1.5, "TOTAL_IMPRESSION_USER": 2.0, "DATE": "2023-07-14"}, "emitted_at": 1690299367301}
+{"stream": "campaign_targeting_report", "data": {"ADVERTISER_ID": 549761668032.0, "AD_ACCOUNT_ID": "549761668032", "CAMPAIGN_DAILY_SPEND_CAP": 25000000.0, "CAMPAIGN_ENTITY_STATUS": "ACTIVE", "CAMPAIGN_ID": 626744128982.0, "CAMPAIGN_LIFETIME_SPEND_CAP": 0.0, "CAMPAIGN_NAME": "2021-06-08 09:08 UTC | Brand awareness", "IMPRESSION_2": 1.0, "TARGETING_VALUE": "TWOCOLUMN_FEED", "TARGETING_TYPE": "FEED_TYPE", "DATE": "2023-10-29"}, "emitted_at": 1699894287823}
+{"stream": "user_account_analytics", "data": {"date": "2023-11-09", "data_status": "READY", "metrics": {"SAVE": 2.0, "OUTBOUND_CLICK_RATE": 0.0043859649122807015, "IMPRESSION": 912.0, "VIDEO_START": 0, "SAVE_RATE": 0.0021929824561403508, "QUARTILE_95_PERCENT_VIEW": 0, "ENGAGEMENT": 22.0, "VIDEO_AVG_WATCH_TIME": 0.0, "ENGAGEMENT_RATE": 0.02412280701754386, "PIN_CLICK": 17, "VIDEO_10S_VIEW": 0, "FULL_SCREEN_PLAY": 0, "CLOSEUP_RATE": 0.017543859649122806, "FULL_SCREEN_PLAYTIME": 0, "VIDEO_V50_WATCH_TIME": 0, "VIDEO_MRC_VIEW": 0, "CLICKTHROUGH": 4.0, "CLICKTHROUGH_RATE": 0.0043859649122807015, "OUTBOUND_CLICK": 4, "CLOSEUP": 16.0, "PIN_CLICK_RATE": 0.01864035087719298}}, "emitted_at": 1699894362486}
+{"stream": "keywords", "data": {"archived": false, "id": "2886935172273", "parent_id": "2680068678965", "parent_type": "AD_GROUP", "type": "KEYWORD", "bid": null, "match_type": "BROAD", "value": "data science"}, "emitted_at": 1699393669235}
+{"stream": "audiences", "data": {"type": "audience", "id": "2542622254639", "name": "airbyte audience", "ad_account_id": "549761668032", "audience_type": "ENGAGEMENT", "description": "airbyte audience", "status": "TOO_SMALL", "rule": {"engager_type": 1}}, "emitted_at": 1699293090886}
+{"stream": "advertizer_report", "data": {"ADVERTISER_ID": 549761668032.0, "AD_ACCOUNT_ID": "549761668032", "EENGAGEMENT_RATE": 0.1, "ENGAGEMENT_2": 1.0, "IMPRESSION_2": 10.0, "REPIN_2": 1.0, "TOTAL_ENGAGEMENT": 1.0, "TOTAL_IMPRESSION_FREQUENCY": 5.0, "TOTAL_IMPRESSION_USER": 2.0, "TOTAL_REPIN_RATE": 0.1, "DATE": "2023-02-10"}, "emitted_at": 1699894848024}
+{"stream": "advertizer_targeting_report", "data": {"ADVERTISER_ID": 549761668032.0, "AD_ACCOUNT_ID": "549761668032", "IMPRESSION_2": 1.0, "TARGETING_VALUE": "Education > Subjects > Science > Applied Science > Technology", "TARGETING_TYPE": "TARGETED_INTEREST", "DATE": "2023-10-29"}, "emitted_at": 1699894982269}
+{"stream": "ad_group_report", "data": {"ADVERTISER_ID": 549761668032.0, "AD_ACCOUNT_ID": "549761668032", "AD_GROUP_ENTITY_STATUS": "ACTIVE", "AD_GROUP_ID": "2680068678993", "CAMPAIGN_DAILY_SPEND_CAP": 25000000.0, "CAMPAIGN_ENTITY_STATUS": "ACTIVE", "CAMPAIGN_ID": 626744128982.0, "CAMPAIGN_LIFETIME_SPEND_CAP": 0.0, "CAMPAIGN_NAME": "2021-06-08 09:08 UTC | Brand awareness", "IMPRESSION_2": 1.0, "TOTAL_IMPRESSION_FREQUENCY": 1.0, "TOTAL_IMPRESSION_USER": 1.0, "DATE": "2023-10-29"}, "emitted_at": 1699895043538}
+{"stream": "ad_group_targeting_report", "data": {"ADVERTISER_ID": 549761668032.0, "AD_ACCOUNT_ID": "549761668032", "AD_GROUP_ENTITY_STATUS": "ACTIVE", "AD_GROUP_ID": "2680068678993", "CAMPAIGN_DAILY_SPEND_CAP": 25000000.0, "CAMPAIGN_ENTITY_STATUS": "ACTIVE", "CAMPAIGN_ID": 626744128982.0, "CAMPAIGN_LIFETIME_SPEND_CAP": 0.0, "CAMPAIGN_NAME": "2021-06-08 09:08 UTC | Brand awareness", "IMPRESSION_2": 1.0, "TARGETING_VALUE": "TWOCOLUMN_FEED", "TARGETING_TYPE": "FEED_TYPE", "DATE": "2023-10-29"}, "emitted_at": 1699895106949}
+{"stream": "pin_promotion_report", "data": {"ADVERTISER_ID": 549761668032.0, "AD_ACCOUNT_ID": "549761668032", "AD_GROUP_ENTITY_STATUS": "ACTIVE", "AD_GROUP_ID": "2680068678993", "AD_ID": "687218400210", "CAMPAIGN_DAILY_SPEND_CAP": 25000000.0, "CAMPAIGN_ENTITY_STATUS": "ACTIVE", "CAMPAIGN_ID": 626744128982.0, "CAMPAIGN_LIFETIME_SPEND_CAP": 0.0, "CAMPAIGN_NAME": "2021-06-08 09:08 UTC | Brand awareness", "IMPRESSION_2": 1.0, "PIN_ID": 6.66743919837295e+17, "PIN_PROMOTION_ID": 687218400210.0, "TOTAL_IMPRESSION_FREQUENCY": 1.0, "TOTAL_IMPRESSION_USER": 1.0, "DATE": "2023-10-29"}, "emitted_at": 1699895200157}
+{"stream": "pin_promotion_targeting_report", "data": {"ADVERTISER_ID": 549761668032.0, "AD_ACCOUNT_ID": "549761668032", "AD_GROUP_ENTITY_STATUS": "ACTIVE", "AD_GROUP_ID": "2680068678993", "AD_ID": "687218400210", "CAMPAIGN_DAILY_SPEND_CAP": 25000000.0, "CAMPAIGN_ENTITY_STATUS": "ACTIVE", "CAMPAIGN_ID": 626744128982.0, "CAMPAIGN_LIFETIME_SPEND_CAP": 0.0, "CAMPAIGN_NAME": "2021-06-08 09:08 UTC | Brand awareness", "IMPRESSION_2": 1.0, "PIN_ID": 6.66743919837295e+17, "PIN_PROMOTION_ID": 687218400210.0, "TARGETING_VALUE": "Education > Subjects > Science > Applied Science > Technology", "TARGETING_TYPE": "TARGETED_INTEREST", "DATE": "2023-10-29"}, "emitted_at": 1699895289749}

airbyte-integrations/connectors/source-pinterest/metadata.yaml

@@ -5,7 +5,7 @@ data:
   connectorSubtype: api
   connectorType: source
   definitionId: 5cb7e5fe-38c2-11ec-8d3d-0242ac130003
-  dockerImageTag: 0.7.1
+  dockerImageTag: 0.7.2
   dockerRepository: airbyte/source-pinterest
   connectorBuildOptions:
     baseImage: docker.io/airbyte/python-connector-base:1.1.0@sha256:bd98f6505c6764b1b5f99d3aedc23dfc9e9af631a62533f60eb32b1d3dbab20c