🐛 Source tiktok-marketing: fix include deleted option (#44048)

airbyte-integrations/connectors/source-tiktok-marketing/acceptance-test-config.yml

@@ -51,6 +51,8 @@ acceptance_tests:
             bypass_reason: "Tested with daily granularity."
           - name: advertisers_reports_hourly
             bypass_reason: "Tested with daily granularity."
+          - name: audiences
+            bypass_reason: "No data in the integration test account. We should seed the sandbox later on."
           - name: campaigns_reports_hourly
             bypass_reason: "Tested with daily granularity."
           - name: creative_assets_portfolios
@@ -74,6 +76,8 @@ acceptance_tests:
             bypass_reason: "This stream is tested on the new style config."
           - name: advertisers
             bypass_reason: "This stream is tested on the new style config."
+          - name: audiences
+            bypass_reason: "No data in the integration test account. We should seed the sandbox later on."
           - name: campaigns
             bypass_reason: "This stream is tested on the new style config."
           - name: creative_assets_portfolios

airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/expected_records.jsonl

@@ -5,8 +5,6 @@
 {"stream": "ad_groups", "data": {"schedule_infos": null, "adgroup_app_profile_page_state": null, "billing_event": "CPC", "skip_learning_phase": 0, "secondary_status": "ADGROUP_STATUS_CAMPAIGN_DISABLE", "rf_purchased_type": null, "excluded_custom_actions": [], "gender": "GENDER_UNLIMITED", "is_hfss": false, "conversion_window": null, "video_download_disabled": false, "rf_estimated_cpr": null, "budget_mode": "BUDGET_MODE_DYNAMIC_DAILY_BUDGET", "network_types": [], "auto_targeting_enabled": false, "schedule_end_time": "2033-10-13 15:35:45", "contextual_tag_ids": [], "isp_ids": [], "zipcode_ids": [], "adgroup_name": "Ad group 20231016073545", "inventory_filter_enabled": false, "modify_time": "2023-10-16 21:47:26", "campaign_name": "UTM_PARAMSTraffic20231016173112", "household_income": [], "bid_price": 0, "dayparting": "111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111", "comment_disabled": false, "next_day_retention": null, "interest_category_ids": [], "purchased_reach": null, "is_new_structure": true, "app_download_url": null, "device_price_ranges": [], "pacing": "PACING_MODE_SMOOTH", "purchased_impression": null, "audience_ids": [], "category_exclusion_ids": [], "interest_keyword_ids": [], "delivery_mode": null, "statistic_type": null, "budget": 30, "pixel_id": null, "device_model_ids": [], "optimization_goal": "CLICK", "excluded_audience_ids": [], "share_disabled": false, "location_ids": [6252001], "scheduled_budget": 0, "creative_material_mode": "CUSTOM", "keywords": null, "app_id": null, "brand_safety_type": "NO_BRAND_SAFETY", "advertiser_id": 7002238017842757633, "frequency": null, "languages": [], "age_groups": null, "feed_type": null, "frequency_schedule": null, "placement_type": "PLACEMENT_TYPE_AUTOMATIC", "adgroup_id": 1779923881029666, "promotion_type": "WEBSITE", "rf_estimated_frequency": null, "secondary_optimization_event": null, "create_time": "2023-10-16 14:48:28", "campaign_id": 1779923887578145, "brand_safety_partner": null, "ios14_quota_type": "UNOCCUPIED", "app_type": null, "category_id": 0, "conversion_bid_price": 0, "deep_bid_type": null, "bid_type": "BID_TYPE_NO_BID", "operation_status": "ENABLE", "operating_systems": [], "optimization_event": null, "placements": ["PLACEMENT_TIKTOK", "PLACEMENT_GLOBAL_APP_BUNDLE", "PLACEMENT_PANGLE"], "bid_display_mode": "CPMV", "spending_power": "ALL", "search_result_enabled": true, "deep_cpa_bid": 0, "included_custom_actions": [], "schedule_type": "SCHEDULE_FROM_NOW", "is_smart_performance_campaign": false, "schedule_start_time": "2023-10-16 15:35:45", "actions": []}, "emitted_at": 1698061648761}
 {"stream": "ad_groups", "data": {"schedule_infos": null, "adgroup_app_profile_page_state": null, "billing_event": "CPC", "skip_learning_phase": 0, "secondary_status": "ADGROUP_STATUS_CAMPAIGN_DISABLE", "rf_purchased_type": null, "excluded_custom_actions": [], "gender": "GENDER_UNLIMITED", "is_hfss": false, "conversion_window": null, "video_download_disabled": false, "rf_estimated_cpr": null, "budget_mode": "BUDGET_MODE_DAY", "network_types": [], "auto_targeting_enabled": false, "schedule_end_time": "2032-03-25 13:02:23", "adgroup_name": "AdGroupVadim", "inventory_filter_enabled": false, "modify_time": "2023-10-17 14:31:10", "campaign_name": "CampaignVadimTraffic", "bid_price": 0, "dayparting": "111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111", "comment_disabled": false, "next_day_retention": null, "interest_category_ids": [15], "purchased_reach": null, "is_new_structure": true, "app_download_url": null, "device_price_ranges": [], "pacing": "PACING_MODE_SMOOTH", "purchased_impression": null, "audience_ids": [], "category_exclusion_ids": [], "interest_keyword_ids": [], "delivery_mode": null, "statistic_type": null, "budget": 20, "pixel_id": null, "device_model_ids": [], "optimization_goal": "CLICK", "excluded_audience_ids": [], "share_disabled": false, "location_ids": [6252001], "scheduled_budget": 0, "creative_material_mode": "CUSTOM", "keywords": null, "app_id": null, "brand_safety_type": "NO_BRAND_SAFETY", "advertiser_id": 7002238017842757633, "frequency": null, "languages": [], "age_groups": ["AGE_25_34", "AGE_35_44"], "feed_type": null, "frequency_schedule": null, "placement_type": "PLACEMENT_TYPE_AUTOMATIC", "adgroup_id": 1728545385226289, "promotion_type": "WEBSITE", "rf_estimated_frequency": null, "secondary_optimization_event": null, "create_time": "2022-03-28 12:09:07", "campaign_id": 1728545382536225, "brand_safety_partner": null, "ios14_quota_type": "UNOCCUPIED", "app_type": null, "category_id": 0, "conversion_bid_price": 0, "deep_bid_type": null, "bid_type": "BID_TYPE_NO_BID", "operation_status": "ENABLE", "operating_systems": [], "optimization_event": null, "placements": ["PLACEMENT_TIKTOK", "PLACEMENT_TOPBUZZ", "PLACEMENT_PANGLE"], "bid_display_mode": "CPMV", "search_result_enabled": false, "deep_cpa_bid": 0, "included_custom_actions": [], "schedule_type": "SCHEDULE_FROM_NOW", "is_smart_performance_campaign": false, "schedule_start_time": "2022-03-28 13:02:23", "actions": []}, "emitted_at": 1698061648763}
 {"stream": "ad_groups", "data": {"schedule_infos": null, "adgroup_app_profile_page_state": null, "billing_event": "CPC", "skip_learning_phase": 0, "secondary_status": "ADGROUP_STATUS_CAMPAIGN_DISABLE", "rf_purchased_type": null, "excluded_custom_actions": [], "gender": "GENDER_UNLIMITED", "is_hfss": false, "conversion_window": null, "video_download_disabled": false, "rf_estimated_cpr": null, "budget_mode": "BUDGET_MODE_DAY", "network_types": [], "auto_targeting_enabled": false, "schedule_end_time": "2021-10-31 09:01:07", "adgroup_name": "Ad Group20211020010107", "inventory_filter_enabled": false, "modify_time": "2022-03-24 12:06:54", "campaign_name": "Website Traffic20211020010104", "bid_price": 0, "dayparting": "111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111", "comment_disabled": false, "next_day_retention": null, "interest_category_ids": [], "purchased_reach": null, "is_new_structure": true, "app_download_url": null, "device_price_ranges": [], "pacing": "PACING_MODE_SMOOTH", "purchased_impression": null, "audience_ids": [], "category_exclusion_ids": [], "interest_keyword_ids": [], "delivery_mode": null, "statistic_type": null, "budget": 20, "pixel_id": null, "device_model_ids": [], "optimization_goal": "CLICK", "excluded_audience_ids": [], "share_disabled": false, "location_ids": [6252001], "scheduled_budget": 0, "creative_material_mode": "CUSTOM", "keywords": null, "app_id": null, "brand_safety_type": "NO_BRAND_SAFETY", "advertiser_id": 7002238017842757633, "frequency": null, "languages": ["en"], "age_groups": ["AGE_25_34", "AGE_35_44", "AGE_45_54"], "feed_type": null, "frequency_schedule": null, "placement_type": "PLACEMENT_TYPE_AUTOMATIC", "adgroup_id": 1714125049901106, "promotion_type": "WEBSITE", "rf_estimated_frequency": null, "secondary_optimization_event": null, "create_time": "2021-10-20 08:04:05", "campaign_id": 1714125042508817, "brand_safety_partner": null, "ios14_quota_type": "UNOCCUPIED", "app_type": null, "category_id": 0, "conversion_bid_price": 0, "deep_bid_type": null, "bid_type": "BID_TYPE_NO_BID", "operation_status": "ENABLE", "operating_systems": [], "optimization_event": null, "placements": ["PLACEMENT_TIKTOK", "PLACEMENT_TOPBUZZ", "PLACEMENT_PANGLE"], "bid_display_mode": "CPMV", "search_result_enabled": false, "deep_cpa_bid": 0, "included_custom_actions": [], "schedule_type": "SCHEDULE_START_END", "is_smart_performance_campaign": false, "schedule_start_time": "2021-10-20 09:01:07", "actions": []}, "emitted_at": 1698061648765}
-{"stream": "audiences", "data": {"create_time": "2021-10-20 07:26:39", "shared": false, "expired_time": "2022-10-20 07:26:39", "is_expiring": true, "is_creator": true, "is_valid": false, "audience_id": "125451003", "cover_num": 0, "name": "Airbyte2", "audience_type": "Lead Generation", "calculate_type": null}, "emitted_at": 1698061649885}
-{"stream": "audiences", "data": {"create_time": "2021-10-20 07:10:04", "shared": false, "expired_time": "2022-10-20 07:10:04", "is_expiring": true, "is_creator": true, "is_valid": false, "audience_id": "125450951", "cover_num": 0, "name": "Airbyte", "audience_type": "Website Audience", "calculate_type": null}, "emitted_at": 1698061649886}
 {"stream": "campaigns", "data": {"campaign_id": 1779923887578145, "is_search_campaign": false, "create_time": "2023-10-16 14:48:26", "objective_type": "TRAFFIC", "secondary_status": "CAMPAIGN_STATUS_DISABLE", "app_promotion_type": "UNSET", "operation_status": "DISABLE", "rf_campaign_type": "STANDARD", "campaign_name": "UTM_PARAMSTraffic20231016173112", "campaign_type": "REGULAR_CAMPAIGN", "modify_time": "2023-10-17 14:33:35", "advertiser_id": 7002238017842757633, "is_new_structure": true, "objective": "LANDING_PAGE", "budget_mode": "BUDGET_MODE_INFINITE", "roas_bid": 0, "deep_bid_type": null, "budget": 0, "is_smart_performance_campaign": false}, "emitted_at": 1698061651114}
 {"stream": "campaigns", "data": {"campaign_id": 1728545382536225, "is_search_campaign": false, "create_time": "2022-03-28 12:09:05", "objective_type": "TRAFFIC", "secondary_status": "CAMPAIGN_STATUS_DISABLE", "operation_status": "DISABLE", "campaign_name": "CampaignVadimTraffic", "campaign_type": "REGULAR_CAMPAIGN", "modify_time": "2022-03-30 21:23:52", "advertiser_id": 7002238017842757633, "is_new_structure": true, "objective": "LANDING_PAGE", "budget_mode": "BUDGET_MODE_INFINITE", "roas_bid": 0, "deep_bid_type": null, "budget": 0, "is_smart_performance_campaign": false}, "emitted_at": 1698061651115}
 {"stream": "campaigns", "data": {"campaign_id": 1714125042508817, "is_search_campaign": false, "create_time": "2021-10-20 08:04:04", "objective_type": "TRAFFIC", "secondary_status": "CAMPAIGN_STATUS_DISABLE", "operation_status": "DISABLE", "campaign_name": "Website Traffic20211020010104", "campaign_type": "REGULAR_CAMPAIGN", "modify_time": "2022-03-24 12:08:29", "advertiser_id": 7002238017842757633, "is_new_structure": true, "objective": "LANDING_PAGE", "budget_mode": "BUDGET_MODE_INFINITE", "roas_bid": 0, "deep_bid_type": null, "budget": 0, "is_smart_performance_campaign": false}, "emitted_at": 1698061651115}

airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/expected_records2.jsonl

@@ -1,5 +1,3 @@
-{"stream": "audiences", "data": {"create_time": "2021-10-20 07:26:39", "is_valid": false, "audience_id": "125451003", "cover_num": 0, "shared": false, "is_expiring": true, "audience_type": "Lead Generation", "calculate_type": null, "name": "Airbyte2", "expired_time": "2022-10-20 07:26:39", "is_creator": true}, "emitted_at": 1698065374424}
-{"stream": "audiences", "data": {"create_time": "2021-10-20 07:10:04", "is_valid": false, "audience_id": "125450951", "cover_num": 0, "shared": false, "is_expiring": true, "audience_type": "Website Audience", "calculate_type": null, "name": "Airbyte", "expired_time": "2022-10-20 07:10:04", "is_creator": true}, "emitted_at": 1698065374425}
 {"stream": "creative_assets_images", "data": {"image_url": "https://p21-ad-sg.ibyteimg.com/obj/ad-site-i18n-sg/202108305d0dfc5fa236678842faa2e0", "is_carousel_usable": true, "file_name": "imgonline-com-ua-Resize-xODPXZ4mmUGvUpJ_1630293661041.jpg", "material_id": "7002056724858683393", "signature": "794ea6266f30750ff0158651a42c8e4c", "create_time": "2021-08-30T03:21:16Z", "format": "jpeg", "width": 640, "modify_time": "2021-08-30T03:21:17Z", "size": 30364, "displayable": true, "height": 640, "image_id": "ad-site-i18n-sg/202108305d0dfc5fa236678842faa2e0"}, "emitted_at": 1698065377209}
 {"stream": "creative_assets_images", "data": {"width": 720, "file_name": "7080121373767221250", "displayable": false, "is_carousel_usable": false, "material_id": "7080121373767221250", "create_time": "2022-03-28T12:11:34Z", "size": 33342, "modify_time": "2022-03-28T12:09:10Z", "format": "jpeg", "height": 1280, "image_url": "https://p16-ad-site-sign-sg.ibyteimg.com/v0201/7f371ff6f0764f8b8ef4f37d7b980d50~tplv-d5opwmad15-image.jpeg?x-expires=2013425378&x-signature=mN6uOUjxW82AQ25O%2F2xVDe89xqE%3D", "image_id": "v0201/7f371ff6f0764f8b8ef4f37d7b980d50", "signature": "2fb70f79f5b11f1bf102039bdd9315df"}, "emitted_at": 1698065378369}
 {"stream": "creative_assets_images", "data": {"width": 720, "file_name": "a2_1648468243469.png", "displayable": false, "is_carousel_usable": true, "material_id": "7080116242086625281", "create_time": "2022-03-28T11:50:49Z", "size": 117823, "modify_time": "2022-03-28T11:50:50Z", "format": "jpeg", "height": 1280, "image_url": "https://p21-ad-sg.ibyteimg.com/obj/ad-site-i18n-sg/10623349cd96274a8bf6650030e15214", "image_id": "ad-site-i18n-sg/10623349cd96274a8bf6650030e15214", "signature": "f564bd399410d9c271f79f0c785414e1"}, "emitted_at": 1698065378369}

airbyte-integrations/connectors/source-tiktok-marketing/metadata.yaml

@@ -11,7 +11,7 @@ data:
   connectorSubtype: api
   connectorType: source
   definitionId: 4bfac00d-ce15-44ff-95b9-9e3c3e8fbd35
-  dockerImageTag: 4.2.2
+  dockerImageTag: 4.2.3
   dockerRepository: airbyte/source-tiktok-marketing
   documentationUrl: https://docs.airbyte.com/integrations/sources/tiktok-marketing
   githubIssueLabel: source-tiktok-marketing

airbyte-integrations/connectors/source-tiktok-marketing/pyproject.toml

@@ -3,7 +3,7 @@ requires = [ "poetry-core>=1.0.0",]
 build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
-version = "4.2.2"
+version = "4.2.3"
 name = "source-tiktok-marketing"
 description = "Source implementation for Tiktok Marketing."
 authors = [ "Airbyte <contact@airbyte.io>",]

airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/manifest.yaml

@@ -397,11 +397,7 @@ definitions:
         metrics: '{{ (parameters.get("report_metrics", []) + ["spend", "cpc", "cpm", "impressions", "clicks", "ctr", "reach", "cost_per_1000_reached", "frequency", "video_play_actions", "video_watched_2s", "video_watched_6s", "average_video_play", "average_video_play_per_user", "video_views_p25", "video_views_p50", "video_views_p75", "video_views_p100", "profile_visits", "likes", "comments", "shares", "follows", "clicks_on_music_disc", "real_time_app_install", "real_time_app_install_cost", "app_install"]) | string }}'
         start_date: "{{ stream_interval['start_time'] }}"
         end_date: "{{ stream_interval['end_time'] }}"
-        filters: '{{ [
-          {"filter_value": ["STATUS_ALL"], "field_name": "ad_status", "filter_type": "IN"},
-          {"filter_value": ["STATUS_ALL"], "field_name": "campaign_status", "filter_type": "IN"},
-          {"filter_value": ["STATUS_ALL"], "field_name": "adgroup_status", "filter_type": "IN"},
-          ] | string if config.get("include_deleted", False)}}'
+        filtering: '{{ parameters["filtering"] | string if parameters.get("filtering") and config.get("include_deleted", False)}}'
       authenticator:
         $ref: "#/definitions/authenticator"
       request_body_json: {}
@@ -517,6 +513,14 @@ definitions:
     $parameters:
       path: "report/integrated/get/"
       data_level: "AUCTION_AD"
+      filtering:
+        [
+          {
+            "field_name": "ad_status",
+            "filter_type": "IN",
+            "filter_value": "[\\\"STATUS_ALL\\\"]",
+          },
+        ]
       report_metrics:
         [
           "campaign_name",
@@ -579,6 +583,14 @@ definitions:
     $parameters:
       path: "report/integrated/get/"
       data_level: "AUCTION_ADGROUP"
+      filtering:
+        [
+          {
+            "field_name": "adgroup_status",
+            "filter_type": "IN",
+            "filter_value": "[\\\"STATUS_ALL\\\"]",
+          },
+        ]
       report_metrics:
         [
           "campaign_name",
@@ -649,6 +661,14 @@ definitions:
     $parameters:
       path: "report/integrated/get/"
       data_level: "AUCTION_CAMPAIGN"
+      filtering:
+        [
+          {
+            "field_name": "campaign_status",
+            "filter_type": "IN",
+            "filter_value": "[\\\"STATUS_ALL\\\"]",
+          },
+        ]
       report_metrics: ["campaign_name"]
       dimensions: ["campaign_id", "stat_time_day"]
     primary_key:
@@ -1227,6 +1247,14 @@ definitions:
     $parameters:
       path: "report/integrated/get/"
       data_level: "AUCTION_AD"
+      filtering:
+        [
+          {
+            "field_name": "ad_status",
+            "filter_type": "IN",
+            "filter_value": "[\\\"STATUS_ALL\\\"]",
+          },
+        ]
       report_metrics:
         [
           "campaign_name",
@@ -1310,6 +1338,14 @@ definitions:
     $parameters:
       path: "report/integrated/get/"
       data_level: "AUCTION_CAMPAIGN"
+      filtering:
+        [
+          {
+            "field_name": "campaign_status",
+            "filter_type": "IN",
+            "filter_value": "[\\\"STATUS_ALL\\\"]",
+          },
+        ]
       report_metrics: ["campaign_name"]
       dimensions: ["campaign_id", "stat_time_hour"]
     primary_key:
@@ -1332,6 +1368,14 @@ definitions:
     $parameters:
       path: "report/integrated/get/"
       data_level: "AUCTION_ADGROUP"
+      filtering:
+        [
+          {
+            "field_name": "adgroup_status",
+            "filter_type": "IN",
+            "filter_value": "[\\\"STATUS_ALL\\\"]",
+          },
+        ]
       report_metrics:
         [
           "campaign_name",