Children speed improvements

CHANGELOG.md

@@ -1,5 +1,8 @@
 # Changelog
 
+## 0.3.3
+  * Added after param for activities stream [#24](https://github.com/singer-io/tap-activecampaign/pull/24)
+
 ## 0.3.2
   * Fix the transformation error of forms stream [#22](https://github.com/singer-io/tap-activecampaign/pull/22)
 

setup.py

@@ -3,7 +3,7 @@
 from setuptools import setup, find_packages
 
 setup(name='tap-activecampaign',
-      version='0.3.2',
+      version='0.3.3',
       description='Singer.io tap for extracting data from the Google Search Console API',
       author='jeff.huth@bytecode.io',
       classifiers=['Programming Language :: Python :: 3 :: Only'],

tap_activecampaign/client.py

@@ -16,6 +16,8 @@ class Server5xxError(Exception):
 class Server429Error(Exception):
     pass
 
+class Server403Error(Exception):
+    pass
 
 class ActiveCampaignError(Exception):
     pass
@@ -25,7 +27,7 @@ class ActiveCampaignBadRequestError(ActiveCampaignError):
 class ActiveCampaignUnauthorizedError(ActiveCampaignError):
     pass
 
-class ActiveCampaignForbiddenError(ActiveCampaignError):
+class ActiveCampaignForbiddenError(Server403Error):
     pass
 
 class ActiveCampaignNotFoundError(ActiveCampaignError):
@@ -79,7 +81,7 @@ def should_retry_error(exception):
         Return true if exception is required to retry otherwise return false
     """
 
-    if isinstance(exception, OSError) or isinstance(exception, Server5xxError) or isinstance(exception, Server429Error):
+    if isinstance(exception, OSError) or isinstance(exception, Server5xxError) or isinstance(exception, Server429Error) or isinstance(exception, Server403Error):
         # Retry Server5xxError and Server429Error exception. Retry exception if it is child class of OSError.
         # OSError is Parent class of ConnectionError, ConnectionResetError, TimeoutError and other errors mentioned in https://docs.python.org/3/library/exceptions.html#os-exceptions
         return True
@@ -191,7 +193,7 @@ def check_api_token(self):
     @backoff.on_exception(backoff.expo,
                           (Exception),
                           giveup=lambda e: not should_retry_error(e),
-                          max_tries=5,
+                          max_tries=50,
                           factor=2)
     # Rate limit: https://developers.activecampaign.com/reference#rate-limits
     @utils.ratelimit(5, 1)

tap_activecampaign/streams.py

@@ -36,6 +36,7 @@ class ActiveCampaign:
     data_key = None
     created_timestamp = None
     bookmark_query_field = None
+    limit_to_start_date_query_field = None
     links = []
     children = []
 
@@ -185,6 +186,7 @@ def sync(
 
         static_params = self.params
         bookmark_query_field = self.bookmark_query_field
+        limit_to_start_date_query_field = self.limit_to_start_date_query_field;
         bookmark_field = next(iter(self.replication_keys or []), None)
         # Get the latest bookmark for the stream and set the last_integer/datetime
         last_datetime = None
@@ -194,8 +196,7 @@ def sync(
         max_bookmark_value = last_datetime
         LOGGER.info('stream: {}, bookmark_field: {}, last_datetime: {}'.format(
             self.stream_name, bookmark_field, last_datetime))
-        now_datetime = utils.now()
-        last_dttm = strptime_to_utc(last_datetime)
+        now_datetime = utils.now().strftime('%Y-%m-%dT%H:%M:%SZ')
         endpoint_total = 0
 
         # pagination: loop thru all pages of data
@@ -219,6 +220,9 @@ def sync(
             if bookmark_query_field:
                 params[bookmark_query_field] = last_datetime
 
+            if limit_to_start_date_query_field:
+                params[limit_to_start_date_query_field] = now_datetime
+
             # Need URL querystring for 1st page; subsequent pages provided by next_url
             # querystring: Squash query params into string
             querystring = None
@@ -535,7 +539,9 @@ class Contacts(ActiveCampaign):
     data_key = 'contacts'
     created_timestamp = 'created_timestamp'
     bookmark_query_field = 'filters[updated_after]'
+    limit_to_start_date_query_field = 'filters[updated_before]'
     links = ['contactGoals', 'contactLogs', 'geoIps', 'trackingLogs']
+    children= ['contact_custom_field_values', 'contact_automations', 'contact_tags', 'contact_lists', 'bounce_logs', 'activities']
 
 class ContactAutomations(ActiveCampaign):
     """
@@ -544,9 +550,10 @@ class ContactAutomations(ActiveCampaign):
     """
     stream_name = 'contact_automations'
     replication_keys = ['lastdate']
-    path = 'contactAutomations'
     data_key = 'contactAutomations'
     created_timestamp = 'adddate'
+    path = 'contacts/{}/contactAutomations'
+    parent = 'contacts'
 
 class ContactCustomFields(ActiveCampaign):
     """
@@ -585,9 +592,10 @@ class ContactCustomFieldValues(ActiveCampaign):
     """
     stream_name = 'contact_custom_field_values'
     replication_keys = ['udate']
-    path = 'fieldValues'
     data_key = 'fieldValues'
     created_timestamp = 'cdate'
+    path = 'contacts/{}/fieldValues'
+    parent = 'contacts'
 
 class ContactDeals(ActiveCampaign):
     """
@@ -849,8 +857,9 @@ class Activities(ActiveCampaign):
     """
     stream_name = 'activities'
     replication_keys = ['tstamp']
-    path = 'activities'
     data_key = 'activities'
+    path = 'activities?contact={}'
+    parent = 'contacts'
     bookmark_query_field = 'after'
 
 class AutomationBlocks(ActiveCampaign):
@@ -869,9 +878,10 @@ class BounceLogs(ActiveCampaign):
     """
     stream_name = 'bounce_logs'
     replication_keys = ['updated_timestamp']
-    path = 'bounceLogs'
     data_key = 'bounceLogs'
     created_timestamp = 'created_timestamp'
+    path = 'contacts/{}/bounceLogs'
+    parent = 'contacts'
 
 class CampaignLists(ActiveCampaign):
     """
@@ -926,19 +936,21 @@ class ContactLists(ActiveCampaign):
     """
     stream_name = 'contact_lists'
     replication_keys = ['updated_timestamp']
-    path = 'contactLists'
     data_key = 'contactLists'
     created_timestamp = 'created_timestamp'
+    path = 'contacts/{}/contactLists'
+    parent = 'contacts'
 
 class ContactTags(ActiveCampaign):
     """
     Get data for contact_tags. 
     """
     stream_name = 'contact_tags'
     replication_keys = ['updated_timestamp']
-    path = 'contactTags'
     data_key = 'contactTags'
     created_timestamp = 'created_timestamp'
+    path = 'contacts/{}/contactTags'
+    parent = 'contacts'
 
 class ContactConversions(ActiveCampaign):
     """
@@ -1097,7 +1109,13 @@ class Sms(ActiveCampaign):
 }
 
 SUB_STREAMS = {
-    'ecommerce_orders': 'ecommerce_order_products'
+    'ecommerce_orders': 'ecommerce_order_products',
+    'contacts-contact_automations': 'contact_automations',
+    'contacts-contact_custom_field_values': 'contact_custom_field_values',
+    'contacts-contact_tags': 'contact_tags',
+    'contacts-contact_lists': 'contact_lists',
+    'contacts-bounce_logs': 'bounce_logs',
+    'contacts-activities': 'activities'
 }
 
 def flatten_streams():