From bb081e132574cee7f93e6ce38f3a8582d14d39d4 Mon Sep 17 00:00:00 2001 From: Rushikesh Todkar <98420315+RushiT0122@users.noreply.github.com> Date: Tue, 23 May 2023 15:30:23 +0530 Subject: [PATCH] update behaviour for 1 second window size --- tap_zendesk/streams.py | 4 +++ test/unittests/test_user_infinite_loop.py | 32 +++++++++++++++-------- 2 files changed, 25 insertions(+), 11 deletions(-) diff --git a/tap_zendesk/streams.py b/tap_zendesk/streams.py index 5c9e4af..08232fe 100644 --- a/tap_zendesk/streams.py +++ b/tap_zendesk/streams.py @@ -77,6 +77,10 @@ def __init__(self, client=None, config=None): else: self.request_timeout = REQUEST_TIMEOUT # If value is 0,"0","" or not passed then it set default to 300 seconds. + # To avoid infinite loop behavior we should not configure search window less than 2 + if config.get('search_window_size') and int(config.get('search_window_size')) < 2: + raise ValueError('Search window size cannot be less than 2') + def get_bookmark(self, state): return utils.strptime_with_tz(singer.get_bookmark(state, self.name, self.replication_key)) diff --git a/test/unittests/test_user_infinite_loop.py b/test/unittests/test_user_infinite_loop.py index 8b2a368..db6d916 100644 --- a/test/unittests/test_user_infinite_loop.py +++ b/test/unittests/test_user_infinite_loop.py @@ -7,14 +7,15 @@ import datetime from dateutil import tz + class MockSearch: def __init__(self): self.count = 1001 self.updated_at = "test" - + def __iter__(self): return (self for x in range(4)) - + def search(self, test, updated_after, updated_before, type="user" ): # For window size less than 2 return less than or equal to 1000 records and for larger window return greater than 1000 records if (singer.strptime(updated_before) - singer.strptime(updated_after)).seconds < 2: @@ -30,31 +31,40 @@ def test_many_records_in_one_seconds_for_user(self, mocked_now): Reproduce infinite looping behavior for Users stream when user have many record in single seconds """ user_obj = Users(MockSearch(), {}) - + with self.assertRaises(Exception) as e: l = list(user_obj.sync({'bookmarks': {'users': {'updated_at': '2022-03-30T08:45:21.000000Z'}}})) - + self.assertEqual(str(e.exception), 'users - Unable to get all users within minimum window of a single second (2022-03-30T08:45:21Z), found 1001 users within this timestamp. Zendesk can only provide a maximum of 1000 users per request. See: https://develop.zendesk.com/hc/en-us/articles/360022563994--BREAKING-New-Search-API-Result-Limits') - + def test_many_records_in_one_seconds_for_user_with_3_sec_window(self): """ To verify that if user give 3 seconds window then also we don't get infinite loop behavior """ user_obj = Users(client=MockSearch(), config={'search_window_size': 3}) - + with self.assertRaises(Exception) as e: l = list(user_obj.sync({'bookmarks': {'users': {'updated_at': '2022-03-30T08:45:21.000000Z'}}})) - + self.assertEqual(str(e.exception), 'users - Unable to get all users within minimum window of a single second (2022-03-30T08:45:21Z), found 1001 users within this timestamp. Zendesk can only provide a maximum of 1000 users per request. See: https://develop.zendesk.com/hc/en-us/articles/360022563994--BREAKING-New-Search-API-Result-Limits') - + def test_many_records_in_one_seconds_for_user_with_2_sec_window(self): """ To verify that if user give 2 seconds window then also we don't get infinite loop behavior """ user_obj = Users(client=MockSearch(), config={'search_window_size': 2}) - + with self.assertRaises(Exception) as e: l = list(user_obj.sync({'bookmarks': {'users': {'updated_at': '2022-03-30T08:45:21.000000Z'}}})) - + self.assertEqual(str(e.exception), 'users - Unable to get all users within minimum window of a single second (2022-03-30T08:45:21Z), found 1001 users within this timestamp. Zendesk can only provide a maximum of 1000 users per request. See: https://develop.zendesk.com/hc/en-us/articles/360022563994--BREAKING-New-Search-API-Result-Limits') - \ No newline at end of file + + def test_search_window_size_equals_1_sec(self): + """ + To verify that search window size 1 second cannot be configured + """ + + with self.assertRaises(ValueError) as e: + Users(client=MockSearch(), config={'search_window_size': 1}) + + self.assertEqual(str(e.exception), 'Search window size cannot be less than 2')