Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

🐛 Source FB Marketing: fix execute_in_batch when batch is bigger than 50 #10588

Merged
merged 10 commits into from
Feb 23, 2022
Merged

🐛 Source FB Marketing: fix execute_in_batch when batch is bigger than 50 #10588

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
9b53b40
fix execute_in_batch
eugene-kulak Feb 23, 2022
70d4dbb
add tests
eugene-kulak Feb 23, 2022
3e27c56
Merge remote-tracking branch 'origin/master' into keu/source-fb-marke…
eugene-kulak Feb 23, 2022
fd26c7b
fix pre-commit config
eugene-kulak Feb 23, 2022
edde169
format
eugene-kulak Feb 23, 2022
d6094b6
bump
eugene-kulak Feb 23, 2022
d630b14
format
eugene-kulak Feb 23, 2022
c00a1cc
address comment from @sherifnada
eugene-kulak Feb 23, 2022
55541c5
Update airbyte-integrations/connectors/source-facebook-marketing/sour…
keu Feb 23, 2022
3b4c249
bumping
eugene-kulak Feb 23, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions .pre-commit-config.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ repos:
- id: black
args: ["--config", "pyproject.toml"]
- repo: https://github.com/timothycrosley/isort
rev: 5.6.4
rev: 5.10.1
hooks:
- id: isort
args:
Expand All @@ -38,14 +38,14 @@ repos:
).?$

- repo: https://github.com/csachs/pyproject-flake8
rev: 0.0.1a2
rev: v0.0.1a2.post1
hooks:
- id: pyproject-flake8
args: ["--config", "pyproject.toml"]
additional_dependencies: ["mccabe"]
alias: flake8
- repo: https://github.com/pre-commit/mirrors-mypy
rev: 0.930
rev: v0.930
hooks:
- id: mypy
args: ["--config-file", "pyproject.toml"]
Expand Down
2 changes: 1 addition & 1 deletion airbyte-config/init/src/main/resources/seed/source_definitions.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -203,7 +203,7 @@
- name: Facebook Marketing
sourceDefinitionId: e7778cfc-e97c-4458-9ecb-b4f2bba8946c
dockerRepository: airbyte/source-facebook-marketing
dockerImageTag: 0.2.35
dockerImageTag: 0.2.36
documentationUrl: https://docs.airbyte.io/integrations/sources/facebook-marketing
icon: facebook.svg
sourceType: api
Expand Down
289 changes: 211 additions & 78 deletions airbyte-config/init/src/main/resources/seed/source_specs.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1648,145 +1648,278 @@
supportsNormalization: false
supportsDBT: false
supported_destination_sync_modes: []
- dockerImage: "airbyte/source-facebook-marketing:0.2.35"
- dockerImage: "airbyte/source-facebook-marketing:0.2.36"
spec:
documentationUrl: "https://docs.airbyte.io/integrations/sources/facebook-marketing"
changelogUrl: "https://docs.airbyte.io/integrations/sources/facebook-marketing"
connectionSpecification:
title: "Source Facebook Marketing"
type: "object"
properties:
account_id:
title: "Account Id"
description: "The Facebook Ad account ID to use when pulling data from the\
\ Facebook Marketing API."
type: "string"
access_token:
title: "Access Token"
description: "The value of the access token generated. See the <a href=\"\
https://docs.airbyte.io/integrations/sources/facebook-marketing\">docs</a>\
\ for more information"
airbyte_secret: true
type: "string"
start_date:
title: "Start Date"
description: "The date from which you'd like to replicate data for AdCreatives\
\ and AdInsights APIs, in the format YYYY-MM-DDT00:00:00Z. All data generated\
\ after this date will be replicated."
description: "The date from which you'd like to replicate data for all incremental\
\ streams, in the format YYYY-MM-DDT00:00:00Z. All data generated after\
\ this date will be replicated."
order: 0
pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$"
examples:
- "2017-01-25T00:00:00Z"
type: "string"
format: "date-time"
end_date:
title: "End Date"
description: "The date until which you'd like to replicate data for AdCreatives\
\ and AdInsights APIs, in the format YYYY-MM-DDT00:00:00Z. All data generated\
description: "The date until which you'd like to replicate data for all\
\ incremental streams, in the format YYYY-MM-DDT00:00:00Z. All data generated\
\ between start_date and this date will be replicated. Not setting this\
\ option will result in always syncing the latest data."
order: 1
pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$"
examples:
- "2017-01-26T00:00:00Z"
type: "string"
format: "date-time"
account_id:
title: "Account ID"
description: "The Facebook Ad account ID to use when pulling data from the\
\ Facebook Marketing API."
order: 2
examples:
- "111111111111111"
type: "string"
access_token:
title: "Access Token"
description: "The value of the access token generated. See the <a href=\"\
https://docs.airbyte.io/integrations/sources/facebook-marketing\">docs</a>\
\ for more information"
order: 3
airbyte_secret: true
type: "string"
include_deleted:
title: "Include Deleted"
description: "Include data from deleted Campaigns, Ads, and AdSets"
default: false
order: 4
type: "boolean"
fetch_thumbnail_images:
title: "Fetch Thumbnail Images"
description: "In each Ad Creative, fetch the thumbnail_url and store the\
\ result in thumbnail_data_url"
default: false
order: 5
type: "boolean"
include_deleted:
title: "Include Deleted"
description: "Include data from deleted campaigns, ads, and adsets"
default: false
type: "boolean"
insights_lookback_window:
title: "Insights Lookback Window"
description: "The attribution window for the actions"
default: 28
minimum: 0
maximum: 28
type: "integer"
insights_days_per_job:
title: "Insights Days Per Job"
description: "Number of days to sync in one job (the more data you have,\
\ the smaller this parameter should be)"
default: 7
minimum: 1
maximum: 30
type: "integer"
custom_insights:
title: "Custom Insights"
description: "A list wich contains insights entries, each entry must have\
description: "A list which contains insights entries, each entry must have\
\ a name and can contains fields, breakdowns or action_breakdowns)"
order: 6
type: "array"
items:
title: "InsightConfig"
description: "Config for custom insights"
type: "object"
properties:
name:
title: "Name"
description: "The name value of insight"
type: "string"
fields:
title: "Fields"
description: "A list of chosen fields for fields parameter"
default: []
type: "array"
items:
type: "string"
title: "ValidEnums"
description: "Generic enumeration.\n\nDerive from this class to\
\ define new enumerations."
enum:
- "account_currency"
- "account_id"
- "account_name"
- "action_values"
- "actions"
- "ad_bid_value"
- "ad_click_actions"
- "ad_id"
- "ad_impression_actions"
- "ad_name"
- "adset_bid_value"
- "adset_end"
- "adset_id"
- "adset_name"
- "adset_start"
- "age_targeting"
- "attribution_setting"
- "auction_bid"
- "auction_competitiveness"
- "auction_max_competitor_bid"
- "buying_type"
- "campaign_id"
- "campaign_name"
- "canvas_avg_view_percent"
- "canvas_avg_view_time"
- "catalog_segment_actions"
- "catalog_segment_value"
- "catalog_segment_value_mobile_purchase_roas"
- "catalog_segment_value_omni_purchase_roas"
- "catalog_segment_value_website_purchase_roas"
- "clicks"
- "conversion_rate_ranking"
- "conversion_values"
- "conversions"
- "converted_product_quantity"
- "converted_product_value"
- "cost_per_15_sec_video_view"
- "cost_per_2_sec_continuous_video_view"
- "cost_per_action_type"
- "cost_per_ad_click"
- "cost_per_conversion"
- "cost_per_dda_countby_convs"
- "cost_per_estimated_ad_recallers"
- "cost_per_inline_link_click"
- "cost_per_inline_post_engagement"
- "cost_per_one_thousand_ad_impression"
- "cost_per_outbound_click"
- "cost_per_thruplay"
- "cost_per_unique_action_type"
- "cost_per_unique_click"
- "cost_per_unique_conversion"
- "cost_per_unique_inline_link_click"
- "cost_per_unique_outbound_click"
- "cpc"
- "cpm"
- "cpp"
- "created_time"
- "ctr"
- "date_start"
- "date_stop"
- "dda_countby_convs"
- "dda_results"
- "engagement_rate_ranking"
- "estimated_ad_recall_rate"
- "estimated_ad_recall_rate_lower_bound"
- "estimated_ad_recall_rate_upper_bound"
- "estimated_ad_recallers"
- "estimated_ad_recallers_lower_bound"
- "estimated_ad_recallers_upper_bound"
- "frequency"
- "full_view_impressions"
- "full_view_reach"
- "gender_targeting"
- "impressions"
- "inline_link_click_ctr"
- "inline_link_clicks"
- "inline_post_engagement"
- "instant_experience_clicks_to_open"
- "instant_experience_clicks_to_start"
- "instant_experience_outbound_clicks"
- "interactive_component_tap"
- "labels"
- "location"
- "mobile_app_purchase_roas"
- "objective"
- "optimization_goal"
- "outbound_clicks"
- "outbound_clicks_ctr"
- "place_page_name"
- "purchase_roas"
- "qualifying_question_qualify_answer_rate"
- "quality_ranking"
- "quality_score_ectr"
- "quality_score_ecvr"
- "quality_score_organic"
- "reach"
- "social_spend"
- "spend"
- "total_postbacks"
- "unique_actions"
- "unique_clicks"
- "unique_conversions"
- "unique_ctr"
- "unique_inline_link_click_ctr"
- "unique_inline_link_clicks"
- "unique_link_clicks_ctr"
- "unique_outbound_clicks"
- "unique_outbound_clicks_ctr"
- "unique_video_continuous_2_sec_watched_actions"
- "unique_video_view_15_sec"
- "updated_time"
- "video_15_sec_watched_actions"
- "video_30_sec_watched_actions"
- "video_avg_time_watched_actions"
- "video_continuous_2_sec_watched_actions"
- "video_p100_watched_actions"
- "video_p25_watched_actions"
- "video_p50_watched_actions"
- "video_p75_watched_actions"
- "video_p95_watched_actions"
- "video_play_actions"
- "video_play_curve_actions"
- "video_play_retention_0_to_15s_actions"
- "video_play_retention_20_to_60s_actions"
- "video_play_retention_graph_actions"
- "video_thruplay_watched_actions"
- "video_time_watched_actions"
- "website_ctr"
- "website_purchase_roas"
- "wish_bid"
breakdowns:
title: "Breakdowns"
description: "A list of chosen breakdowns for breakdowns"
default: []
type: "array"
items:
type: "string"
title: "ValidBreakdowns"
description: "Generic enumeration.\n\nDerive from this class to\
\ define new enumerations."
enum:
- "ad_format_asset"
- "age"
- "app_id"
- "body_asset"
- "call_to_action_asset"
- "country"
- "description_asset"
- "device_platform"
- "dma"
- "frequency_value"
- "gender"
- "hourly_stats_aggregated_by_advertiser_time_zone"
- "hourly_stats_aggregated_by_audience_time_zone"
- "image_asset"
- "impression_device"
- "link_url_asset"
- "place_page_id"
- "platform_position"
- "product_id"
- "publisher_platform"
- "region"
- "skan_conversion_id"
- "title_asset"
- "video_asset"
action_breakdowns:
title: "Action Breakdowns"
description: "A list of chosen action_breakdowns for action_breakdowns"
default: []
type: "array"
items:
type: "string"
title: "ValidActionBreakdowns"
description: "Generic enumeration.\n\nDerive from this class to\
\ define new enumerations."
enum:
- "action_canvas_component_name"
- "action_carousel_card_id"
- "action_carousel_card_name"
- "action_destination"
- "action_device"
- "action_reaction"
- "action_target_id"
- "action_type"
- "action_video_sound"
- "action_video_type"
required:
- "name"
required:
- "start_date"
- "account_id"
- "access_token"
- "start_date"
definitions:
InsightConfig:
title: "InsightConfig"
type: "object"
properties:
name:
title: "Name"
description: "The name value of insight"
type: "string"
fields:
title: "Fields"
description: "A list of chosen fields for fields parameter"
default: []
type: "array"
items:
type: "string"
breakdowns:
title: "Breakdowns"
description: "A list of chosen breakdowns for breakdowns"
default: []
type: "array"
items:
type: "string"
action_breakdowns:
title: "Action Breakdowns"
description: "A list of chosen action_breakdowns for action_breakdowns"
default: []
type: "array"
items:
type: "string"
required:
- "name"
supportsIncremental: true
supportsNormalization: false
supportsDBT: false
Expand Down
2 changes: 1 addition & 1 deletion airbyte-integrations/connectors/source-facebook-marketing/Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,5 +13,5 @@ ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]


LABEL io.airbyte.version=0.2.35
LABEL io.airbyte.version=0.2.36
LABEL io.airbyte.name=airbyte/source-facebook-marketing
Loading