From 84818319edaf3a1e5cf48b450e81ae162e8ab952 Mon Sep 17 00:00:00 2001 From: Andy Lu Date: Tue, 12 Nov 2019 21:26:36 +0000 Subject: [PATCH 1/6] Update RESULT_RETURN_LIMIT if passed in from config --- tap_facebook/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tap_facebook/__init__.py b/tap_facebook/__init__.py index e41a6d9d..807aaede 100755 --- a/tap_facebook/__init__.py +++ b/tap_facebook/__init__.py @@ -658,6 +658,9 @@ def main_impl(): CONFIG.update(args.config) + if CONFIG.get('result_return_limit'): + RESULT_RETURN_LIMIT = CONFIG.get('result_return_limit') + FacebookAdsApi.init(access_token=access_token) user = fb_user.User(fbid='me') accounts = user.get_ad_accounts() From cabbbe28da735c45dbd29b92cebb56313ecb89c8 Mon Sep 17 00:00:00 2001 From: Andy Lu Date: Wed, 13 Nov 2019 18:01:49 +0000 Subject: [PATCH 2/6] WIP: Date windowing on adcreative stream --- tap_facebook/__init__.py | 31 ++++++++++++++++++++++--------- 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/tap_facebook/__init__.py b/tap_facebook/__init__.py index 807aaede..759b2f1b 100755 --- a/tap_facebook/__init__.py +++ b/tap_facebook/__init__.py @@ -149,6 +149,7 @@ class Stream(object): account = attr.ib() stream_alias = attr.ib() catalog_entry = attr.ib() + state = attr.ib() def automatic_fields(self): fields = set() @@ -178,8 +179,6 @@ def fields(self): @attr.s class IncrementalStream(Stream): - state = attr.ib() - def __attrs_post_init__(self): self.current_bookmark = get_start(self, UPDATED_TIME_KEY) @@ -210,12 +209,26 @@ class AdCreative(Stream): def __iter__(self): @retry_pattern(backoff.expo, FacebookRequestError, max_tries=5, factor=5) - def do_request(): + def do_request(params): return self.account.get_ad_creatives(fields=self.fields(), # pylint: disable=no-member - params={'limit': RESULT_RETURN_LIMIT}) - ad_creative = do_request() - for a in ad_creative: # pylint: disable=invalid-name - yield {'record': a.export_all_data()} + params={'limit': RESULT_RETURN_LIMIT, 'time_range': params}) + + date_window_start = pendulum.parse(CONFIG['start_date']).date() + date_window_end = date_window_start.add(days=1) + end_date = TODAY.date() + + while date_window_end < end_date: + + params = { + 'since' : str(date_window_start), + 'until' : str(date_window_end) + } + LOGGER.info("Date window: %s - %s", date_window_start, date_window_end) + ad_creative = do_request(params) + for a in ad_creative: # pylint: disable=invalid-name + yield {'record': a.export_all_data()} + date_window_start = date_window_start.add(days=1) + date_window_end = date_window_start.add(days=1) class Ads(IncrementalStream): @@ -398,7 +411,6 @@ class AdsInsights(Stream): field_class = adsinsights.AdsInsights.Field base_properties = ['campaign_id', 'adset_id', 'ad_id', 'date_start'] - state = attr.ib() options = attr.ib() action_breakdowns = attr.ib(default=ALL_ACTION_BREAKDOWNS) level = attr.ib(default='ad') @@ -544,7 +556,7 @@ def initialize_stream(account, catalog_entry, state): # pylint: disable=too-many elif name == 'ads': return Ads(name, account, stream_alias, catalog_entry, state=state) elif name == 'adcreative': - return AdCreative(name, account, stream_alias, catalog_entry) + return AdCreative(name, account, stream_alias, catalog_entry, state=state) else: raise TapFacebookException('Unknown stream {}'.format(name)) @@ -659,6 +671,7 @@ def main_impl(): CONFIG.update(args.config) if CONFIG.get('result_return_limit'): + LOGGER.info('Overriding RESULT_RETURN_LIMIT from %s to %s', RESULT_RETURN_LIMIT, CONFIG.get('result_return_limit')) RESULT_RETURN_LIMIT = CONFIG.get('result_return_limit') FacebookAdsApi.init(access_token=access_token) From 927b53ea6a4c94976e14db67a0c0791f467bd167 Mon Sep 17 00:00:00 2001 From: Andy Lu Date: Wed, 13 Nov 2019 19:38:43 +0000 Subject: [PATCH 3/6] Undo changes to state --- tap_facebook/__init__.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tap_facebook/__init__.py b/tap_facebook/__init__.py index 759b2f1b..140b94eb 100755 --- a/tap_facebook/__init__.py +++ b/tap_facebook/__init__.py @@ -149,7 +149,6 @@ class Stream(object): account = attr.ib() stream_alias = attr.ib() catalog_entry = attr.ib() - state = attr.ib() def automatic_fields(self): fields = set() @@ -179,6 +178,8 @@ def fields(self): @attr.s class IncrementalStream(Stream): + state = attr.ib() + def __attrs_post_init__(self): self.current_bookmark = get_start(self, UPDATED_TIME_KEY) @@ -411,6 +412,7 @@ class AdsInsights(Stream): field_class = adsinsights.AdsInsights.Field base_properties = ['campaign_id', 'adset_id', 'ad_id', 'date_start'] + state = attr.ib() options = attr.ib() action_breakdowns = attr.ib(default=ALL_ACTION_BREAKDOWNS) level = attr.ib(default='ad') @@ -556,7 +558,7 @@ def initialize_stream(account, catalog_entry, state): # pylint: disable=too-many elif name == 'ads': return Ads(name, account, stream_alias, catalog_entry, state=state) elif name == 'adcreative': - return AdCreative(name, account, stream_alias, catalog_entry, state=state) + return AdCreative(name, account, stream_alias, catalog_entry) else: raise TapFacebookException('Unknown stream {}'.format(name)) From e11791aa12a54531a9724fc8716d0a2e852bc62f Mon Sep 17 00:00:00 2001 From: Andy Lu Date: Wed, 13 Nov 2019 19:42:45 +0000 Subject: [PATCH 4/6] Simplify result_return_limit override --- tap_facebook/__init__.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tap_facebook/__init__.py b/tap_facebook/__init__.py index 140b94eb..00745f47 100755 --- a/tap_facebook/__init__.py +++ b/tap_facebook/__init__.py @@ -672,9 +672,8 @@ def main_impl(): CONFIG.update(args.config) - if CONFIG.get('result_return_limit'): - LOGGER.info('Overriding RESULT_RETURN_LIMIT from %s to %s', RESULT_RETURN_LIMIT, CONFIG.get('result_return_limit')) - RESULT_RETURN_LIMIT = CONFIG.get('result_return_limit') + global RESULT_RETURN_LIMIT + RESULT_RETURN_LIMIT = CONFIG.get('result_return_limit', RESULT_RETURN_LIMIT) FacebookAdsApi.init(access_token=access_token) user = fb_user.User(fbid='me') From 90b05fa9d0624470357bdc6b30a1a07a0e25f565 Mon Sep 17 00:00:00 2001 From: Andy Lu Date: Wed, 13 Nov 2019 19:52:57 +0000 Subject: [PATCH 5/6] Add a comment --- tap_facebook/__init__.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tap_facebook/__init__.py b/tap_facebook/__init__.py index 00745f47..8f0388ce 100755 --- a/tap_facebook/__init__.py +++ b/tap_facebook/__init__.py @@ -214,6 +214,9 @@ def do_request(params): return self.account.get_ad_creatives(fields=self.fields(), # pylint: disable=no-member params={'limit': RESULT_RETURN_LIMIT, 'time_range': params}) + # `time_range` wants a value in the shape of {"since": + # "YYYY-MM-DD", "until": "YYYY-MM-DD"}, which is what + # pendulum.parse().date() returns date_window_start = pendulum.parse(CONFIG['start_date']).date() date_window_end = date_window_start.add(days=1) end_date = TODAY.date() @@ -224,10 +227,11 @@ def do_request(params): 'since' : str(date_window_start), 'until' : str(date_window_end) } - LOGGER.info("Date window: %s - %s", date_window_start, date_window_end) ad_creative = do_request(params) + for a in ad_creative: # pylint: disable=invalid-name yield {'record': a.export_all_data()} + date_window_start = date_window_start.add(days=1) date_window_end = date_window_start.add(days=1) From 72201d3af6a6bbd8d68adf564eccd8ce2fb9d858 Mon Sep 17 00:00:00 2001 From: Andy Lu Date: Wed, 13 Nov 2019 20:20:45 +0000 Subject: [PATCH 6/6] Include end date in the sync --- tap_facebook/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tap_facebook/__init__.py b/tap_facebook/__init__.py index 8f0388ce..5d8fa0b4 100755 --- a/tap_facebook/__init__.py +++ b/tap_facebook/__init__.py @@ -221,7 +221,7 @@ def do_request(params): date_window_end = date_window_start.add(days=1) end_date = TODAY.date() - while date_window_end < end_date: + while date_window_end <= end_date: params = { 'since' : str(date_window_start),