Skip to content

Commit

Permalink
🎉 Source Pinterest - new streams (#32299)
Browse files Browse the repository at this point in the history
Co-authored-by: midavadim <midavadim@users.noreply.github.com>
Co-authored-by: Augustin <augustin@airbyte.io>
Co-authored-by: alafanechere <alafanechere@users.noreply.github.com>
  • Loading branch information
4 people authored Nov 15, 2023
1 parent 3d48b92 commit 2ae2586
Show file tree
Hide file tree
Showing 20 changed files with 1,069 additions and 103 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ acceptance_tests:
- config_path: secrets/config.json
status: succeed
- config_path: integration_tests/invalid_config.json
status: exception
status: failed
- config_path: secrets/config_oauth.json
status: succeed
discovery:
Expand All @@ -23,26 +23,24 @@ acceptance_tests:
tests:
- config_path: secrets/config.json
empty_streams:
- bypass_reason: The stream could return 0 records, because of low rate-limits
name: ad_account_analytics
- bypass_reason: The stream could return 0 records, because of low rate-limits
name: ad_analytics
- bypass_reason: The stream could return 0 records, because of low rate-limits
name: ad_group_analytics
- bypass_reason: The stream could return 0 records, because of low rate-limits
name: ad_groups
- bypass_reason: The stream could return 0 records, because of low rate-limits
name: ads
- bypass_reason: The stream could return 0 records, because of low rate-limits
name: board_section_pins
- bypass_reason: The stream could return 0 records, because of low rate-limits
name: board_sections
- bypass_reason: The stream could return 0 records, because of low rate-limits
name: campaign_analytics
- bypass_reason: The stream could return 0 records, because of low rate-limits
name: campaigns
- bypass_reason: The stream could return 0 records, because of low rate-limits
name: user_account_analytics
- name: conversion_tags
bypass_reason: Not possible to add data
- name: customer_lists
bypass_reason: Not possible to add data
- name: catalogs
bypass_reason: Not possible to add data
- name: catalogs_feeds
bypass_reason: Not possible to add data
- name: catalogs_product_groups
bypass_reason: Not possible to add data
- name: product_group_report
bypass_reason: Not possible to add data
- name: product_group_targeting_report
bypass_reason: Not possible to add data
- name: product_item_report
bypass_reason: Not possible to add data
- name: keyword_report
bypass_reason: Not possible to add data
timeout_seconds: 1200
expect_records:
path: "integration_tests/expected_records.jsonl"
Expand All @@ -52,8 +50,27 @@ acceptance_tests:
fail_on_extra_columns: false
ignored_fields:
board_pins:
- name: "links"
bypass_reason: "because it contains non-secure http link, which leads to failed QA tests"
- name: "media"
bypass_reason: "urls may change"
board_section_pins:
- name: "media"
bypass_reason: "urls may change"
ads:
- name: "updated_time"
bypass_reason: "can be updated"
ad_groups:
- name: "updated_time"
bypass_reason: "can be updated"
campaigns:
- name: "updated_time"
bypass_reason: "can be updated"
audiences:
- name: "size"
bypass_reason: "can be changed"
- name: "updated_timestamp"
bypass_reason: "can be changed"
- name: "created_timestamp"
bypass_reason: "can be changed"
incremental:
tests:
- config_path: secrets/config.json
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,23 @@
{"stream": "ad_accounts", "data": {"id": "549761668032", "name": "Airbyte", "owner": {"username": "integrationtest0375", "id": "666744057242074926"}, "country": "US", "currency": "USD", "permissions": ["OWNER"], "created_time": 1603772920, "updated_time": 1623173784}, "emitted_at": 1688461289470}
{"stream": "ad_account_analytics", "data": {"TOTAL_IMPRESSION_FREQUENCY": 1.0, "TOTAL_IMPRESSION_USER": 1.0, "ADVERTISER_ID": "549761668032", "DATE": "2023-10-29", "IMPRESSION_2": 1.0, "AD_ACCOUNT_ID": "549761668032"}, "emitted_at": 1699893121669}
{"stream": "ads", "data": {"id": "687218400118", "ad_group_id": "2680068678965", "ad_account_id": "549761668032", "android_deep_link": null, "campaign_id": "626744128956", "carousel_android_deep_links": null, "carousel_destination_urls": null, "carousel_ios_deep_links": null, "click_tracking_url": null, "collection_items_destination_url_template": null, "created_time": 1623245885, "creative_type": "REGULAR", "destination_url": "https://airbyte.io/", "ios_deep_link": null, "is_pin_deleted": false, "is_removable": false, "name": "2021-06-09 | Traffic | Keywords | Data Integration", "pin_id": "666743919837294988", "rejected_reasons": [], "rejection_labels": [], "review_status": "APPROVED", "status": "PAUSED", "summary_status": "PAUSED", "tracking_urls": null, "type": "ad", "updated_time": 1699373013, "view_tracking_url": null, "lead_form_id": null}, "emitted_at": 1699393433303}
{"stream": "ad_analytics", "data": {"PIN_ID": 6.66743919837295e+17, "AD_GROUP_ID": "2680068678993", "AD_GROUP_ENTITY_STATUS": "1", "CAMPAIGN_ENTITY_STATUS": 1.0, "TOTAL_IMPRESSION_FREQUENCY": 1.0, "CAMPAIGN_LIFETIME_SPEND_CAP": 0.0, "TOTAL_IMPRESSION_USER": 1.0, "CAMPAIGN_DAILY_SPEND_CAP": 25000000.0, "AD_ID": "687218400210", "ADVERTISER_ID": "549761668032", "PIN_PROMOTION_ID": 687218400210.0, "DATE": "2023-10-29", "IMPRESSION_2": 1.0, "AD_ACCOUNT_ID": "549761668032", "CAMPAIGN_ID": 626744128982.0, "CAMPAIGN_NAME": "2021-06-08 09:08 UTC | Brand awareness"}, "emitted_at": 1699893196846}
{"stream": "ad_groups", "data": {"id": "2680068678965", "created_time": 1623245885.0, "updated_time": 1699373013.0, "start_time": null, "end_time": null, "bid_in_micro_currency": null, "budget_in_micro_currency": null, "campaign_id": "626744128956", "ad_account_id": "549761668032", "auto_targeting_enabled": true, "type": "adgroup", "budget_type": "CBO_ADGROUP", "billable_event": "CLICKTHROUGH", "status": "ACTIVE", "lifetime_frequency_cap": -1.0, "targeting_spec": {"GENDER": ["female", "male", "unknown"], "APPTYPE": ["web", "web_mobile", "iphone", "ipad", "android_mobile", "android_tablet"], "LOCALE": ["cs", "da", "de", "el", "en", "es", "fi", "fr", "hu", "id", "it", "ja", "ko", "nb", "nl", "pl", "pt", "ro", "ru", "sk", "sv", "tr", "uk", "zh"], "TARGETING_STRATEGY": ["CHOOSE_YOUR_OWN"], "LOCATION": ["US"]}, "name": "2021-06-09 | Traffic | Keywords | Data Integration", "placement_group": "ALL", "pacing_delivery_type": "STANDARD", "tracking_urls": null, "conversion_learning_mode_type": null, "summary_status": "COMPLETED", "feed_profile_id": "0", "placement_traffic_type": null, "optimization_goal_metadata": {}, "bid_strategy_type": "AUTOMATIC_BID"}, "emitted_at": 1699393433712}
{"stream": "ad_group_analytics", "data": {"AD_GROUP_ID": "2680068678993", "AD_GROUP_ENTITY_STATUS": "1", "CAMPAIGN_ENTITY_STATUS": 1.0, "TOTAL_IMPRESSION_FREQUENCY": 1.0, "CAMPAIGN_LIFETIME_SPEND_CAP": 0.0, "TOTAL_IMPRESSION_USER": 1.0, "CAMPAIGN_DAILY_SPEND_CAP": 25000000.0, "ADVERTISER_ID": "549761668032", "DATE": "2023-10-29", "IMPRESSION_2": 1.0, "AD_ACCOUNT_ID": "549761668032", "CAMPAIGN_ID": 626744128982.0, "CAMPAIGN_NAME": "2021-06-08 09:08 UTC | Brand awareness"}, "emitted_at": 1699893280169}
{"stream": "boards", "data": {"media": {"pin_thumbnail_urls": [], "image_cover_url": "https://i.pinimg.com/400x300/c6/b6/0d/c6b60d6b5f2ec04db7748d35fb1a8004.jpg"}, "owner": {"username": "integrationtest0375"}, "created_at": "2021-06-08T09:37:18", "board_pins_modified_at": "2021-10-25T11:17:56.715000", "id": "666743988523388559", "collaborator_count": 0, "follower_count": 2, "pin_count": 1, "privacy": "PUBLIC", "name": "business", "description": ""}, "emitted_at": 1680356853019}
{"stream": "board_pins", "data": {"description": "Data Integration", "board_owner": {"username": "integrationtest0375"}, "product_tags": [], "has_been_promoted": true, "created_at": "2021-06-08T09:37:30", "board_id": "666743988523388559", "note": "", "creative_type": "REGULAR", "parent_pin_id": null, "title": "Airbyte", "alt_text": null, "pin_metrics": null, "dominant_color": "#cacafe", "id": "666743919837294988", "is_owner": true, "board_section_id": "5195034916661798218", "link": "http://airbyte.io/", "media": {"media_type": "image", "images": {"150x150": {"width": 150, "height": 150, "url": "https://i.pinimg.com/150x150/c6/b6/0d/c6b60d6b5f2ec04db7748d35fb1a8004.jpg"}, "400x300": {"width": 400, "height": 300, "url": "https://i.pinimg.com/400x300/c6/b6/0d/c6b60d6b5f2ec04db7748d35fb1a8004.jpg"}, "600x": {"width": 564, "height": 337, "url": "https://i.pinimg.com/564x/c6/b6/0d/c6b60d6b5f2ec04db7748d35fb1a8004.jpg"}, "1200x": {"width": 1200, "height": 718, "url": "https://i.pinimg.com/1200x/c6/b6/0d/c6b60d6b5f2ec04db7748d35fb1a8004.jpg"}}}, "is_standard": true}, "emitted_at": 1698398201666}
{"stream": "campaign_analytics_report", "data": {"ADVERTISER_ID": 549761668032.0, "AD_ACCOUNT_ID": "549761668032", "CAMPAIGN_DAILY_SPEND_CAP": 750000.0, "CAMPAIGN_ENTITY_STATUS": "ACTIVE", "CAMPAIGN_ID": 626744128982.0, "CAMPAIGN_LIFETIME_SPEND_CAP": 0.0, "CAMPAIGN_NAME": "2021-06-08 09:08 UTC | Brand awareness", "IMPRESSION_2": 3.0, "TOTAL_IMPRESSION_FREQUENCY": 1.5, "TOTAL_IMPRESSION_USER": 2.0, "DATE": "2023-07-14"}, "emitted_at": 1690299367301}
{"stream": "board_pins", "data": {"description": "Data Integration", "board_owner": {"username": "integrationtest0375"}, "product_tags": [], "has_been_promoted": true,"link":"http://airbyte.io/", "created_at": "2021-06-08T09:37:30", "board_id": "666743988523388559", "note": "", "creative_type": "REGULAR", "parent_pin_id": null, "title": "Airbyte", "alt_text": null, "pin_metrics": null, "dominant_color": "#cacafe", "id": "666743919837294988", "is_owner": true, "board_section_id": "5195034916661798218", "is_standard": true}, "emitted_at": 1698398201666}
{"stream": "board_sections", "data": {"name": "Airbyte_board_section_new", "id": "5195035116725909603"}, "emitted_at": 1699893323493}
{"stream": "board_section_pins","data":{"id":"666743919837294988","dominant_color":"#cacafe","pin_metrics":null,"title":"Airbyte","creative_type":"REGULAR","link":"http://airbyte.io/","board_id":"666743988523388559","created_at":"2021-06-08T09:37:30","is_owner":true,"description":"Data Integration","note":"","alt_text":null,"board_section_id":"5195034916661798218","parent_pin_id":null,"product_tags":[],"board_owner":{"username":"integrationtest0375"},"is_standard":true,"has_been_promoted":true},"emitted_at":1699893364884}
{"stream": "campaigns", "data": {"id": "626744128956", "ad_account_id": "549761668032", "name": "2021-06-09 | Traffic | Keywords | Data Integration", "status": "ACTIVE", "objective_type": "CONSIDERATION", "lifetime_spend_cap": 0, "daily_spend_cap": 3000000, "order_line_id": null, "tracking_urls": null, "created_time": 1623245885, "updated_time": 1691447502, "type": "campaign", "is_flexible_daily_budgets": false, "summary_status": "COMPLETED", "is_campaign_budget_optimization": true, "start_time": 1623196800, "end_time": 1624060800}, "emitted_at": 1699393571700}
{"stream": "campaign_analytics", "data": {"TOTAL_IMPRESSION_FREQUENCY": 1.0, "CAMPAIGN_LIFETIME_SPEND_CAP": 0.0, "TOTAL_IMPRESSION_USER": 1.0, "CAMPAIGN_ENTITY_STATUS": 1.0, "CAMPAIGN_DAILY_SPEND_CAP": 25000000.0, "ADVERTISER_ID": 549761668032.0, "DATE": "2023-10-29", "IMPRESSION_2": 1.0, "AD_ACCOUNT_ID": "549761668032", "CAMPAIGN_ID": 626744128982.0, "CAMPAIGN_NAME": "2021-06-08 09:08 UTC | Brand awareness"}, "emitted_at": 1699894065462}
{"stream": "campaign_analytics_report", "data": {"ADVERTISER_ID": 549761668032.0, "AD_ACCOUNT_ID": "549761668032", "CAMPAIGN_DAILY_SPEND_CAP": 25000000.0, "CAMPAIGN_ENTITY_STATUS": "ACTIVE", "CAMPAIGN_ID": 626744128982.0, "CAMPAIGN_LIFETIME_SPEND_CAP": 0.0, "CAMPAIGN_NAME": "2021-06-08 09:08 UTC | Brand awareness", "IMPRESSION_2": 3.0, "TOTAL_IMPRESSION_FREQUENCY": 1.5, "TOTAL_IMPRESSION_USER": 2.0, "DATE": "2023-07-14"}, "emitted_at": 1690299367301}
{"stream": "campaign_targeting_report", "data": {"ADVERTISER_ID": 549761668032.0, "AD_ACCOUNT_ID": "549761668032", "CAMPAIGN_DAILY_SPEND_CAP": 25000000.0, "CAMPAIGN_ENTITY_STATUS": "ACTIVE", "CAMPAIGN_ID": 626744128982.0, "CAMPAIGN_LIFETIME_SPEND_CAP": 0.0, "CAMPAIGN_NAME": "2021-06-08 09:08 UTC | Brand awareness", "IMPRESSION_2": 1.0, "TARGETING_VALUE": "TWOCOLUMN_FEED", "TARGETING_TYPE": "FEED_TYPE", "DATE": "2023-10-29"}, "emitted_at": 1699894287823}
{"stream": "user_account_analytics", "data": {"date": "2023-11-09", "data_status": "READY", "metrics": {"SAVE": 2.0, "OUTBOUND_CLICK_RATE": 0.0043859649122807015, "IMPRESSION": 912.0, "VIDEO_START": 0, "SAVE_RATE": 0.0021929824561403508, "QUARTILE_95_PERCENT_VIEW": 0, "ENGAGEMENT": 22.0, "VIDEO_AVG_WATCH_TIME": 0.0, "ENGAGEMENT_RATE": 0.02412280701754386, "PIN_CLICK": 17, "VIDEO_10S_VIEW": 0, "FULL_SCREEN_PLAY": 0, "CLOSEUP_RATE": 0.017543859649122806, "FULL_SCREEN_PLAYTIME": 0, "VIDEO_V50_WATCH_TIME": 0, "VIDEO_MRC_VIEW": 0, "CLICKTHROUGH": 4.0, "CLICKTHROUGH_RATE": 0.0043859649122807015, "OUTBOUND_CLICK": 4, "CLOSEUP": 16.0, "PIN_CLICK_RATE": 0.01864035087719298}}, "emitted_at": 1699894362486}
{"stream": "keywords", "data": {"archived": false, "id": "2886935172273", "parent_id": "2680068678965", "parent_type": "AD_GROUP", "type": "KEYWORD", "bid": null, "match_type": "BROAD", "value": "data science"}, "emitted_at": 1699393669235}
{"stream": "audiences", "data": {"type": "audience", "id": "2542622254639", "name": "airbyte audience", "ad_account_id": "549761668032", "audience_type": "ENGAGEMENT", "description": "airbyte audience", "status": "TOO_SMALL", "rule": {"engager_type": 1}}, "emitted_at": 1699293090886}
{"stream": "advertizer_report", "data": {"ADVERTISER_ID": 549761668032.0, "AD_ACCOUNT_ID": "549761668032", "EENGAGEMENT_RATE": 0.1, "ENGAGEMENT_2": 1.0, "IMPRESSION_2": 10.0, "REPIN_2": 1.0, "TOTAL_ENGAGEMENT": 1.0, "TOTAL_IMPRESSION_FREQUENCY": 5.0, "TOTAL_IMPRESSION_USER": 2.0, "TOTAL_REPIN_RATE": 0.1, "DATE": "2023-02-10"}, "emitted_at": 1699894848024}
{"stream": "advertizer_targeting_report", "data": {"ADVERTISER_ID": 549761668032.0, "AD_ACCOUNT_ID": "549761668032", "IMPRESSION_2": 1.0, "TARGETING_VALUE": "Education > Subjects > Science > Applied Science > Technology", "TARGETING_TYPE": "TARGETED_INTEREST", "DATE": "2023-10-29"}, "emitted_at": 1699894982269}
{"stream": "ad_group_report", "data": {"ADVERTISER_ID": 549761668032.0, "AD_ACCOUNT_ID": "549761668032", "AD_GROUP_ENTITY_STATUS": "ACTIVE", "AD_GROUP_ID": "2680068678993", "CAMPAIGN_DAILY_SPEND_CAP": 25000000.0, "CAMPAIGN_ENTITY_STATUS": "ACTIVE", "CAMPAIGN_ID": 626744128982.0, "CAMPAIGN_LIFETIME_SPEND_CAP": 0.0, "CAMPAIGN_NAME": "2021-06-08 09:08 UTC | Brand awareness", "IMPRESSION_2": 1.0, "TOTAL_IMPRESSION_FREQUENCY": 1.0, "TOTAL_IMPRESSION_USER": 1.0, "DATE": "2023-10-29"}, "emitted_at": 1699895043538}
{"stream": "ad_group_targeting_report", "data": {"ADVERTISER_ID": 549761668032.0, "AD_ACCOUNT_ID": "549761668032", "AD_GROUP_ENTITY_STATUS": "ACTIVE", "AD_GROUP_ID": "2680068678993", "CAMPAIGN_DAILY_SPEND_CAP": 25000000.0, "CAMPAIGN_ENTITY_STATUS": "ACTIVE", "CAMPAIGN_ID": 626744128982.0, "CAMPAIGN_LIFETIME_SPEND_CAP": 0.0, "CAMPAIGN_NAME": "2021-06-08 09:08 UTC | Brand awareness", "IMPRESSION_2": 1.0, "TARGETING_VALUE": "TWOCOLUMN_FEED", "TARGETING_TYPE": "FEED_TYPE", "DATE": "2023-10-29"}, "emitted_at": 1699895106949}
{"stream": "pin_promotion_report", "data": {"ADVERTISER_ID": 549761668032.0, "AD_ACCOUNT_ID": "549761668032", "AD_GROUP_ENTITY_STATUS": "ACTIVE", "AD_GROUP_ID": "2680068678993", "AD_ID": "687218400210", "CAMPAIGN_DAILY_SPEND_CAP": 25000000.0, "CAMPAIGN_ENTITY_STATUS": "ACTIVE", "CAMPAIGN_ID": 626744128982.0, "CAMPAIGN_LIFETIME_SPEND_CAP": 0.0, "CAMPAIGN_NAME": "2021-06-08 09:08 UTC | Brand awareness", "IMPRESSION_2": 1.0, "PIN_ID": 6.66743919837295e+17, "PIN_PROMOTION_ID": 687218400210.0, "TOTAL_IMPRESSION_FREQUENCY": 1.0, "TOTAL_IMPRESSION_USER": 1.0, "DATE": "2023-10-29"}, "emitted_at": 1699895200157}
{"stream": "pin_promotion_targeting_report", "data": {"ADVERTISER_ID": 549761668032.0, "AD_ACCOUNT_ID": "549761668032", "AD_GROUP_ENTITY_STATUS": "ACTIVE", "AD_GROUP_ID": "2680068678993", "AD_ID": "687218400210", "CAMPAIGN_DAILY_SPEND_CAP": 25000000.0, "CAMPAIGN_ENTITY_STATUS": "ACTIVE", "CAMPAIGN_ID": 626744128982.0, "CAMPAIGN_LIFETIME_SPEND_CAP": 0.0, "CAMPAIGN_NAME": "2021-06-08 09:08 UTC | Brand awareness", "IMPRESSION_2": 1.0, "PIN_ID": 6.66743919837295e+17, "PIN_PROMOTION_ID": 687218400210.0, "TARGETING_VALUE": "Education > Subjects > Science > Applied Science > Technology", "TARGETING_TYPE": "TARGETED_INTEREST", "DATE": "2023-10-29"}, "emitted_at": 1699895289749}
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ data:
connectorSubtype: api
connectorType: source
definitionId: 5cb7e5fe-38c2-11ec-8d3d-0242ac130003
dockerImageTag: 0.7.1
dockerImageTag: 0.7.2
dockerRepository: airbyte/source-pinterest
connectorBuildOptions:
baseImage: docker.io/airbyte/python-connector-base:1.1.0@sha256:bd98f6505c6764b1b5f99d3aedc23dfc9e9af631a62533f60eb32b1d3dbab20c
Expand Down
Loading

0 comments on commit 2ae2586

Please sign in to comment.