Skip to content

Commit

Permalink
TDL-17002: Make the logger messages more descriptive (#46)
Browse files Browse the repository at this point in the history 
* added logger messages

* removed unnecessary loggers

* removed unnecessary loggers

* updated the code to use stream name instead of parent stream name
  • Loading branch information
@hpatel41
hpatel41 authored Jan 5, 2022
1 parent 3993fd7 commit 602f913
Show file tree
Hide file tree
Showing 2 changed files with 35 additions and 2 deletions.
2 changes: 2 additions & 0 deletions tap_intercom/client.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -205,6 +205,8 @@ def request(self, method, path=None, url=None, **kwargs):
if not url and path:
url = '{}/{}'.format(self.base_url, path)

LOGGER.info("URL: {} {}, Params: {}, JSON Body: {}".format(method, url, kwargs.get("params"), kwargs.get("json")))

if 'endpoint' in kwargs:
endpoint = kwargs['endpoint']
del kwargs['endpoint']
Expand Down
35 changes: 33 additions & 2 deletions tap_intercom/streams.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,7 @@ def sync(self,
self.replication_key,
config['start_date'])

LOGGER.info("Stream: {}, initial max_bookmark_value: {}".format(self.tap_stream_id, start_date))
bookmark_datetime = singer.utils.strptime_to_utc(start_date)
max_datetime = bookmark_datetime

Expand All @@ -131,7 +132,9 @@ def sync(self,
counter.increment()
max_datetime = max(record_datetime, max_datetime)
bookmark_date = singer.utils.strftime(max_datetime)
LOGGER.info("FINISHED Syncing: {}, total_records: {}.".format(self.tap_stream_id, counter.value))

LOGGER.info("Stream: {}, writing final bookmark".format(self.tap_stream_id))
state = singer.write_bookmark(state,
self.tap_stream_id,
self.replication_key,
Expand Down Expand Up @@ -181,6 +184,8 @@ def sync(self,
singer.write_record(self.tap_stream_id, transformed_record, time_extracted=singer.utils.now())
counter.increment()

LOGGER.info("FINISHED Syncing: {}, total_records: {}.".format(self.tap_stream_id, counter.value))

return state


Expand Down Expand Up @@ -220,6 +225,7 @@ class Admins(FullTableStream):
parent = AdminList

def get_records(self, bookmark_datetime=None, is_parent=False) -> Iterator[list]:
LOGGER.info("Syncing: {}".format(self.tap_stream_id))
admins = []

for record in self.get_parent_data():
Expand Down Expand Up @@ -247,22 +253,25 @@ class Companies(IncrementalStream):
def get_records(self, bookmark_datetime=None, is_parent=False) -> Iterator[list]:
scrolling = True
params = {}
LOGGER.info("Syncing: {}".format(self.tap_stream_id))

while scrolling:
response = self.client.get(self.path, params=params)

if response.get(self.data_key) is None:
LOGGER.warning('response is empty for "{}" stream'.format(self.tap_stream_id))

records = transform_json(response, self.tap_stream_id, self.data_key)
LOGGER.info("Synced: {}, records: {}".format(self.tap_stream_id, len(records)))

# stop scrolling if 'data' array is empty
if 'scroll_param' in response and response.get(self.data_key):
scroll_param = response.get('scroll_param')
params = {'scroll_param': scroll_param}
LOGGER.info("Syncing next page")
else:
scrolling = False

records = transform_json(response, self.tap_stream_id, self.data_key)

yield from records


Expand All @@ -281,13 +290,16 @@ class CompanyAttributes(FullTableStream):
def get_records(self, bookmark_datetime=None, is_parent=False) -> Iterator[list]:
paging = True
next_page = None
LOGGER.info("Syncing: {}".format(self.tap_stream_id))

while paging:
response = self.client.get(self.path, url=next_page, params=self.params)

LOGGER.info("Synced: {}, records: {}".format(self.tap_stream_id, len(response.get(self.data_key, []))))
if 'pages' in response and response.get('pages', {}).get('next'):
next_page = response.get('pages', {}).get('next')
self.path = None
LOGGER.info("Syncing next page")
else:
paging = False

Expand All @@ -314,13 +326,16 @@ class CompnaySegments(IncrementalStream):
def get_records(self, bookmark_datetime=None, is_parent=False) -> Iterator[list]:
paging = True
next_page = None
LOGGER.info("Syncing: {}".format(self.tap_stream_id))

while paging:
response = self.client.get(self.path, url=next_page, params=self.params)

LOGGER.info("Synced: {}, records: {}".format(self.tap_stream_id, len(response.get(self.data_key, []))))
if 'pages' in response and response.get('pages', {}).get('next'):
next_page = response.get('pages', {}).get('next')
self.path = None
LOGGER.info("Syncing next page")
else:
paging = False

Expand Down Expand Up @@ -367,6 +382,7 @@ def get_records(self, bookmark_datetime=None, is_parent=False) -> Iterator[list]
"order": "ascending"
}
}
LOGGER.info("Syncing: {}".format(self.tap_stream_id))

while paging:
response = self.client.post(self.path, json=search_query)
Expand All @@ -378,6 +394,7 @@ def get_records(self, bookmark_datetime=None, is_parent=False) -> Iterator[list]
paging = False

records = transform_json(response, self.tap_stream_id, self.data_key)
LOGGER.info("Synced: {} for page: {}, records: {}".format(self.tap_stream_id, response.get('pages', {}).get('page'), len(records)))

if is_parent:
for record in records:
Expand Down Expand Up @@ -484,15 +501,18 @@ class ContactAttributes(FullTableStream):
data_key = 'data'

def get_records(self, bookmark_datetime=None, is_parent=False) -> Iterator[list]:
LOGGER.info("Syncing: {}".format(self.tap_stream_id))
paging = True
next_page = None

while paging:
response = self.client.get(self.path, url=next_page, params=self.params)

LOGGER.info("Synced: {}, records: {}".format(self.tap_stream_id, len(response.get(self.data_key, []))))
if 'pages' in response and response.get('pages', {}).get('next'):
next_page = response.get('pages', {}).get('next')
self.path = None
LOGGER.info("Syncing next page")
else:
paging = False

Expand Down Expand Up @@ -539,6 +559,7 @@ def get_records(self, bookmark_datetime=None, is_parent=False) -> Iterator[list]
'order': 'ascending'
}
}
LOGGER.info("Syncing: {}".format(self.tap_stream_id))

while paging:
response = self.client.post(self.path, json=search_query)
Expand All @@ -550,6 +571,7 @@ def get_records(self, bookmark_datetime=None, is_parent=False) -> Iterator[list]
paging = False

records = transform_json(response, self.tap_stream_id, self.data_key)
LOGGER.info("Synced: {} for page: {}, records: {}".format(self.tap_stream_id, response.get('pages', {}).get('page'), len(records)))

yield from records

Expand All @@ -571,13 +593,16 @@ class Segments(IncrementalStream):
def get_records(self, bookmark_datetime=None, is_parent=False) -> Iterator[list]:
paging = True
next_page = None
LOGGER.info("Syncing: {}".format(self.tap_stream_id))

while paging:
response = self.client.get(self.path, url=next_page, params=self.params)

LOGGER.info("Synced: {}, records: {}".format(self.tap_stream_id, len(response.get(self.data_key, []))))
if 'pages' in response and response.get('pages', {}).get('next'):
next_page = response.get('pages', {}).get('next')
self.path = None
LOGGER.info("Syncing next page")
else:
paging = False

Expand All @@ -598,13 +623,16 @@ class Tags(FullTableStream):
def get_records(self, bookmark_datetime=None, is_parent=False) -> Iterator[list]:
paging = True
next_page = None
LOGGER.info("Syncing: {}".format(self.tap_stream_id))

while paging:
response = self.client.get(self.path, url=next_page, params=self.params)

LOGGER.info("Synced: {}, records: {}".format(self.tap_stream_id, len(response.get(self.data_key, []))))
if 'pages' in response and response.get('pages', {}).get('next'):
next_page = response.get('pages', {}).get('next')
self.path = None
LOGGER.info("Syncing next page")
else:
paging = False

Expand All @@ -625,13 +653,16 @@ class Teams(FullTableStream):
def get_records(self, bookmark_datetime=None, is_parent=False) -> Iterator[list]:
paging = True
next_page = None
LOGGER.info("Syncing: {}".format(self.tap_stream_id))

while paging:
response = self.client.get(self.path, url=next_page, params=self.params)

LOGGER.info("Synced: {}, records: {}".format(self.tap_stream_id, len(response.get(self.data_key, []))))
if 'pages' in response and response.get('pages', {}).get('next'):
next_page = response.get('pages', {}).get('next')
self.path = None
LOGGER.info("Syncing next page")
else:
paging = False

Expand Down

0 comments on commit 602f913

Please sign in to comment.