Improve back-off client

gspread/exceptions.py

@@ -6,7 +6,7 @@
 
 """
 
-from typing import Dict, Optional, Union
+from typing import Any, Dict, Mapping, Optional, Union
 
 from requests import Response
 
@@ -40,15 +40,12 @@ class APIError(GSpreadException):
     such as when we attempt to retrieve things that don't exist."""
 
     def __init__(self, response: Response):
-        super().__init__(self._extract_text(response))
+        super().__init__(self._extract_error(response))
         self.response: Response = response
+        self.error: Mapping[str, Any] = response.json()["error"]
+        self.code: int = self.error["code"]
 
-    def _extract_text(
-        self, response: Response
-    ) -> Union[Dict[str, Union[int, str]], str]:
-        return self._text_from_detail(response) or response.text
-
-    def _text_from_detail(
+    def _extract_error(
         self, response: Response
     ) -> Optional[Dict[str, Union[int, str]]]:
         try:
@@ -57,6 +54,14 @@ def _text_from_detail(
         except (AttributeError, KeyError, ValueError):
             return None
 
+    def __str__(self) -> str:
+        return "{}: [{}]: {}".format(
+            self.__class__.__name__, self.code, self.error["message"]
+        )
+
+    def __repr__(self) -> str:
+        return self.__str__()
+
 
 class SpreadsheetNotFound(GSpreadException):
     """Trying to open non-existent or inaccessible spreadsheet."""

gspread/http_client.py

@@ -6,6 +6,7 @@
 Google API.
 
 """
+import time
 from http import HTTPStatus
 from typing import (
     IO,
@@ -493,39 +494,57 @@ class BackOffHTTPClient(HTTPClient):
           for api rate limit exceeded."""
 
     _HTTP_ERROR_CODES: List[HTTPStatus] = [
-        HTTPStatus.FORBIDDEN,  # Drive API return a 403 Forbidden on usage rate limit exceeded
         HTTPStatus.REQUEST_TIMEOUT,  # in case of a timeout
         HTTPStatus.TOO_MANY_REQUESTS,  # sheet API usage rate limit exceeded
     ]
     _NR_BACKOFF: int = 0
     _MAX_BACKOFF: int = 128  # arbitrary maximum backoff
-    _MAX_BACKOFF_REACHED: bool = False  # Stop after reaching _MAX_BACKOFF
 
     def request(self, *args: Any, **kwargs: Any) -> Response:
+        # Check if we should retry the request
+        def _should_retry(
+            code: int,
+            error: Mapping[str, Any],
+            wait: int,
+        ) -> bool:
+            # Drive API return a dict object 'errors', the sheet API does not
+            if "errors" in error:
+                # Drive API returns a code 403 when reaching quotas/usage limits
+                if (
+                    code == HTTPStatus.FORBIDDEN
+                    and error["errors"][0]["domain"] == "usageLimits"
+                ):
+                    return True
+
+            # We retry if:
+            #   - the return code is one of:
+            #     - 429: too many requests
+            #     - 408: request timeout
+            #     - >= 500: some server error
+            #   - AND we did not reach the max retry limit
+            return (
+                code in self._HTTP_ERROR_CODES
+                or code >= HTTPStatus.INTERNAL_SERVER_ERROR
+            ) and wait <= self._MAX_BACKOFF
+
         try:
             return super().request(*args, **kwargs)
         except APIError as err:
-            data = err.response.json()
-            code = data["error"]["code"]
-
-            # check if error should retry
-            if code in self._HTTP_ERROR_CODES and self._MAX_BACKOFF_REACHED is False:
-                self._NR_BACKOFF += 1
-                wait = min(2**self._NR_BACKOFF, self._MAX_BACKOFF)
+            code = err.code
+            error = err.error
 
-                if wait >= self._MAX_BACKOFF:
-                    self._MAX_BACKOFF_REACHED = True
-
-                import time
+            self._NR_BACKOFF += 1
+            wait = min(2**self._NR_BACKOFF, self._MAX_BACKOFF)
 
+            # check if error should retry
+            if _should_retry(code, error, wait) is True:
                 time.sleep(wait)
 
                 # make the request again
                 response = self.request(*args, **kwargs)
 
                 # reset counters for next time
                 self._NR_BACKOFF = 0
-                self._MAX_BACKOFF_REACHED = False
 
                 return response