Updated backoff for 503 error code (#95)

singer-io · Dec 2, 2021 · 29e3ea3 · 29e3ea3
1 parent 02b5264
commit 29e3ea3
Show file tree

Hide file tree

Showing 2 changed files with 21 additions and 48 deletions.
diff --git a/tap_zendesk/http.py b/tap_zendesk/http.py
@@ -101,17 +101,11 @@ class ZendeskServiceUnavailableError(ZendeskBackoffError):
 def is_fatal(exception):
     status_code = exception.response.status_code
 
-    if status_code in [429, 503]:
-        # If status_code is 429 or 503 then checking whether response header has 'Retry-After' attribute or not.
-        # If response header has 'Retry-After' attribute then retry the error otherwise raise the error directly.
-        retry_after = exception.response.headers.get('Retry-After')
-        if retry_after:
-            sleep_time = int(retry_after)
-            LOGGER.info("Caught HTTP %s, retrying request in %s seconds", status_code, sleep_time)
-            sleep(sleep_time)
-            return False
-        else:
-            return True
+    if status_code == 429:
+        sleep_time = int(exception.response.headers['Retry-After'])
+        LOGGER.info("Caught HTTP 429, retrying request in %s seconds", sleep_time)
+        sleep(sleep_time)
+        return False
 
     return 400 <=status_code < 500
 

diff --git a/test/unittests/test_http.py b/test/unittests/test_http.py
@@ -218,43 +218,6 @@ def test_get_cursor_based_handles_502(self,mock_get, mock_sleep):
 
         #Verify the request retry 10 times
         self.assertEqual(mock_get.call_count, 10)
-
-    @patch('requests.get',
-           side_effect=[
-               mocked_get(status_code=503, headers={'Retry-After': '1'}, json={"key3": "val3", **SINGLE_RESPONSE}),
-               mocked_get(status_code=503, headers={'retry-after': 1}, json={"key2": "val2", **SINGLE_RESPONSE}),
-               mocked_get(status_code=200, json={"key1": "val1", **SINGLE_RESPONSE}),
-           ])
-    def test_get_cursor_based_handles_503(self,mock_get, mock_sleep):
-        """Test that the tap:
-        - can handle 503s
-        - requests uses a case insensitive dict for the `headers`
-        - can handle either a string or an integer for the retry header
-        """
-        responses = [response for response in http.get_cursor_based(url='some_url',
-                                                                    access_token='some_token', request_timeout=300)]
-        actual_response = responses[0]
-        self.assertDictEqual({"key1": "val1", **SINGLE_RESPONSE},
-                             actual_response)
-
-        #Verify the request calls only 3 times
-        self.assertEqual(mock_get.call_count, 3)
-
-    @patch('requests.get',
-           side_effect=[mocked_get(status_code=503)])
-    def test_get_cursor_based_handles_503_without_retry_after(self,mock_get, mock_sleep):
-        """Test that the tap can handle 503 without retry-after headers
-        """
-        try:
-            responses = [response for response in http.get_cursor_based(url='some_url',
-                                                                    access_token='some_token', request_timeout=300)]
-        except http.ZendeskServiceUnavailableError as e:
-            expected_error_message =  'HTTP-error-code: 503, Error: API service is currently unavailable.'
-            # Verify the message formed for the custom exception
-            self.assertEqual(str(e), expected_error_message)
-
-        #Verify the request calls only 1 time
-        self.assertEqual(mock_get.call_count, 1)
 
     @patch('requests.get')
     def test_get_cursor_based_handles_444(self,mock_get, mock_sleep):
@@ -338,5 +301,21 @@ def test_get_cursor_based_handles_520(self,mock_get, mock_sleep):
             # Verify the message formed for the custom exception
             self.assertEqual(str(e), expected_error_message)
 
+        #Verify the request retry 10 times
+        self.assertEqual(mock_get.call_count, 10)
+
+    @patch('requests.get',side_effect=10*[mocked_get(status_code=503, json={"key1": "val1"})])
+    def test_get_cursor_based_handles_503(self,mock_get, mock_sleep):
+        """
+        Test that the tap can handle 503 error and retry it 10 times
+        """
+        try:
+            responses = [response for response in http.get_cursor_based(url='some_url',
+                                                                    access_token='some_token', request_timeout=300)]
+        except http.ZendeskServiceUnavailableError as e:
+            expected_error_message = "HTTP-error-code: 503, Error: API service is currently unavailable."
+            # Verify the message formed for the custom exception
+            self.assertEqual(str(e), expected_error_message)
+
         #Verify the request retry 10 times
         self.assertEqual(mock_get.call_count, 10)