[CIVIS-8768] ENH customize API call retrying behavior (#495)

* REF move _BufferedPartialReader and _retry to _files.py

* REF move maybe_get_random_name to under civis.io

* DOC List[Response] not List[dict] for the returned object

* ENH user can provide their own tenacity.Retrying

* Revert "DOC List[Response] not List[dict] for the returned object"

This reverts commit fe17a91.

* MAINT update changelog

* SEC nosec for eval() call

* DOC explain why define default retrying as code string

* MAINT update twine version to fix importlib-metadata issue

* RM drop no-longer-needed MAX_RETRIES

* MAINT add 'python.exe -m' to windows build config

* DOC update client docs

* DOC more updates for docs

.circleci/config.yml

@@ -63,9 +63,9 @@ jobs:
           shell: bash.exe
           command: |
             python --version && \
-            pip install --upgrade pip setuptools wheel && \
-            pip install ".[dev-core,dev-civisml]" && \
-            pip list && \
+            python.exe -m pip install --upgrade pip setuptools wheel && \
+            python.exe -m pip install ".[dev-core,dev-civisml]" && \
+            python.exe -m pip list && \
             CIVIS_API_KEY=foobar pytest
 
 workflows:

CHANGELOG.md

@@ -7,6 +7,9 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 ## Unreleased
 
 ### Added
+- The new kwarg `retries` has been added to `civis.APIClient` so that
+  a `tenacity.Retrying` instance can be provided to customize retries. (#495)
+
 ### Changed
 ### Deprecated
 ### Removed

README.rst

@@ -162,26 +162,6 @@ See the `documentation <https://civis-python.readthedocs.io>`_ for a more
 complete user guide.
 
 
-.. start-include-marker-retries-section
-
-Retries
--------
-
-The API client will automatically retry for certain API error responses.
-
-If the error is one of [413, 429, 503] and the API client is told how long it needs
-to wait before it's safe to retry (this is always the case with 429s, which are
-rate limit errors), then the client will wait the specified amount of time
-before retrying the request.
-
-If the error is one of [429, 502, 503, 504] and the request is not a ``patch*`` or ``post*``
-method, then the API client will retry the request several times, with an exponential delay,
-to see if it will succeed. If the request is of type ``post*`` it will retry with the same parameters
-for error codes [429, 503].
-
-.. end-include-marker-retires-section
-
-
 Build Documentation Locally
 ---------------------------
 

docs/source/client.rst

@@ -3,7 +3,7 @@ API Client
 
 :class:`~civis.APIClient` is a class for handling requests to the Civis API.
 An instantiated :class:`~civis.APIClient` contains a set of resources
-(listed below) where each resource is an object with methods. By convention,
+(listed in :ref:`api_resources`) where each resource is an object with methods. By convention,
 an instantiated :class:`~civis.APIClient` object is named ``client`` and API
 requests are made with the following syntax:
 
@@ -12,6 +12,17 @@ requests are made with the following syntax:
    client = civis.APIClient()
    response = client.resource.method(params)
 
+
+.. toctree::
+   :maxdepth: 1
+
+   api_resources
+   responses
+
+
+Dynamically Created Resources and Methods
+-----------------------------------------
+
 The methods on :class:`~civis.APIClient` are created dynamically at runtime
 by parsing an :class:`python:collections.OrderedDict` representation of the
 Civis API specification.
@@ -84,13 +95,33 @@ specification has been saved.
        json.dump(spec, f)
    client = civis.APIClient(local_api_spec='local_api_spec.json')
 
+
+.. _retries:
+
+Retries
+-------
+
+The API client will automatically retry for certain API error responses.
+
+If the error is one of [413, 429, 503] and the API client is told how long it needs
+to wait before it's safe to retry (this is always the case with 429s, which are
+rate limit errors), then the client will wait the specified amount of time
+before retrying the request.
+
+If the error is one of [429, 502, 503, 504] and the request is not a ``patch*`` or ``post*``
+method, then the API client will retry the request several times, with an exponential delay,
+to see if it will succeed. If the request is of type ``post*`` it will retry with the same parameters
+for error codes [429, 503].
+
+While the conditions under which retries are attempted are set as described above,
+the behavior of the retries is customizable by passing in a :class:`tenacity.Retrying` instance
+to the ``retries`` kwarg of :class:`civis.APIClient`.
+
+
+Object Reference
+----------------
+
 .. currentmodule:: civis
 
 .. autoclass:: civis.APIClient
    :members:
-
-.. toctree::
-   :maxdepth: 1
-
-   responses
-   api_resources

docs/source/conf.py

@@ -43,6 +43,7 @@
     "requests": ("https://docs.python-requests.org/en/latest/", None),
     "sklearn": ("https://scikit-learn.org/stable", None),
     "joblib": ("https://joblib.readthedocs.io/en/latest/", None),
+    "tenacity": ("https://tenacity.readthedocs.io/en/latest/", None),
 }
 
 # Add any paths that contain templates here, relative to this directory.
@@ -361,12 +362,14 @@ def _write_resources_rst(class_names, filename, civis_module):
             "API Resources\n"
             "=============\n\n"
             ".. note::\n\n"
-            "   As the Civis API is updated from time to time, "
-            "   the API resources available on a :class:`civis.APIClient` "
-            "   instance may differ from what's documented below. "
-            "   While we strive to keep this documentation up-to-date, the Civis API "
-            "   is officially documented at https://api.civisanalytics.com "
-            "   (Civis Platform login required).\n\n"
+            "   The API resources listed in this documentation are those for "
+            "   a standard Civis Platform user. "
+            "   Particular users may have access to resources and/or "
+            "   methods that are featured-flagged or still under development and "
+            "   therefore do not appear in this documentation. "
+            "   For the exact API resources and methods available to a given "
+            "   Civis Platform user, log on as that user and go to "
+            "   https://api.civisanalytics.com.\n\n"
             ".. toctree::\n"
             "   :titlesonly:\n\n"
         )

docs/source/index.rst

@@ -31,11 +31,6 @@ User Guide
 For a more detailed walkthrough, see the :ref:`user_guide`.
 
 
-.. include:: ../../README.rst
-  :start-after: start-include-marker-retries-section
-  :end-before: end-include-marker-retires-section
-
-
 Table of Contents
 -----------------
 

docs/source/responses.rst

@@ -1,5 +1,7 @@
-API Responses
-=============
+.. _responses:
+
+Responses
+=========
 
 A Civis API call from ``client.<endpoint>.<method>`` returns a :class:`civis.response.Response` object
 (or a :class:`civis.response.PaginatedResponse` object, if ``<method>`` is a "list" call):

docs/source/user_guide.rst

@@ -223,6 +223,7 @@ For endpoints that support pagination when the `iterator` kwarg is specified,
 a :class:`civis.response.PaginatedResponse` object is returned.
 To facilitate working with :class:`civis.response.Response` objects,
 the helper functions :func:`civis.find` and :func:`civis.find_one` are defined.
+For more notes, see :ref:`responses`.
 
 
 Testing Your Code

pyproject.toml

@@ -45,12 +45,12 @@ dev-core = [
     "bandit",  # Install the latest version.
     "black == 24.4.2",
     "build == 1.2.1",
-    "flake8 == 7.0.0",
+    "flake8 == 7.1.0",
     "pandas == 2.2.2",
     "pip-audit",  # Install the latest version.
     "pytest == 8.2.2",
     "pytest-cov == 5.0.0",
-    "twine == 5.1.0",
+    "twine == 5.1.1",
 ]
 dev-civisml = [
     "feather-format == 0.4.1",

src/civis/_utils.py

@@ -1,32 +1,34 @@
 import logging
 import os
-import time
-import uuid
-from random import random
-
-from tenacity import (
-    Retrying,
-    retry_if_result,
-    stop_after_attempt,
-    stop_after_delay,
-    wait_random_exponential,
-)
+
+import tenacity
 from tenacity.wait import wait_base
 
 
 log = logging.getLogger(__name__)
 
-MAX_RETRIES = 10
-
 _RETRY_CODES = [429, 502, 503, 504]
 _RETRY_VERBS = ["HEAD", "TRACE", "GET", "PUT", "OPTIONS", "DELETE"]
 _POST_RETRY_CODES = [429, 503]
 
 
-def maybe_get_random_name(name):
-    if not name:
-        name = uuid.uuid4().hex
-    return name
+# Defining the default tenacity.Retrying as a user-friendly code string
+# so that it can be shown in civis.APIClient's docstring.
+DEFAULT_RETRYING_STR = """
+tenacity.Retrying(
+    wait=tenacity.wait_random_exponential(multiplier=2, max=60),
+    stop=(tenacity.stop_after_delay(600) | tenacity.stop_after_attempt(10)),
+    retry_error_callback=lambda retry_state: retry_state.outcome.result(),
+)
+"""
+
+# Explicitly set the available globals and locals
+# to mitigate risk of unwanted code execution
+DEFAULT_RETRYING = eval(  # nosec
+    DEFAULT_RETRYING_STR,
+    {"tenacity": tenacity, "__builtins__": {}},  # globals
+    {},  # locals
+)
 
 
 def get_api_key(api_key):
@@ -44,112 +46,34 @@ def get_api_key(api_key):
     return api_key
 
 
-def retry_request(method, prepared_req, session, max_retries=10):
+def retry_request(method, prepared_req, session, retrying=None):
     retry_conditions = None
+    retrying = retrying if retrying else DEFAULT_RETRYING
 
     def _make_request(req, sess):
         """send the prepared session request"""
         response = sess.send(req)
         return response
 
-    def _return_last_value(retry_state):
-        """return the result of the last call attempt
-        and let code pick up the error"""
-        return retry_state.outcome.result()
-
     if method.upper() == "POST":
-        retry_conditions = retry_if_result(
+        retry_conditions = tenacity.retry_if_result(
             lambda res: res.status_code in _POST_RETRY_CODES
         )
     elif method.upper() in _RETRY_VERBS:
-        retry_conditions = retry_if_result(lambda res: res.status_code in _RETRY_CODES)
+        retry_conditions = tenacity.retry_if_result(
+            lambda res: res.status_code in _RETRY_CODES
+        )
 
     if retry_conditions:
-        retry_config = Retrying(
-            retry=retry_conditions,
-            wait=wait_for_retry_after_header(
-                fallback=wait_random_exponential(multiplier=2, max=60)
-            ),
-            stop=(stop_after_delay(600) | stop_after_attempt(max_retries)),
-            retry_error_callback=_return_last_value,
-        )
-        response = retry_config(_make_request, prepared_req, session)
+        retrying.retry = retry_conditions
+        retrying.wait = wait_for_retry_after_header(fallback=retrying.wait)
+        response = retrying(_make_request, prepared_req, session)
         return response
 
     response = _make_request(prepared_req, session)
     return response
 
 
-def retry(exceptions, retries=5, delay=0.5, backoff=2):
-    """
-    Retry decorator
-
-    Parameters
-    ----------
-    exceptions: Exception
-        exceptions to trigger retry
-    retries: int, optional
-        number of retries to perform
-    delay: float, optional
-        delay before next retry
-    backoff: int, optional
-        factor used to calculate the exponential increase
-        delay after each retry
-
-    Returns
-    -------
-    retry decorator
-
-    Raises
-    ------
-    exception raised by decorator function
-    """
-
-    def deco_retry(f):
-        def f_retry(*args, **kwargs):
-            n_failed = 0
-            new_delay = delay
-            while True:
-                try:
-                    return f(*args, **kwargs)
-                except exceptions as exc:
-                    if n_failed < retries:
-                        n_failed += 1
-                        msg = "%s, Retrying in %d seconds..." % (str(exc), new_delay)
-                        log.debug(msg)
-                        time.sleep(new_delay)
-                        new_delay = min(
-                            (pow(2, n_failed) / 4) * (random() + backoff),  # nosec
-                            50 + 10 * random(),  # nosec
-                        )
-                    else:
-                        raise exc
-
-        return f_retry
-
-    return deco_retry
-
-
-class BufferedPartialReader(object):
-    def __init__(self, buf, max_bytes):
-        self.buf = buf
-        self.max_bytes = max_bytes
-        self.bytes_read = 0
-        self.len = max_bytes
-
-    def read(self, size=-1):
-        if self.bytes_read >= self.max_bytes:
-            return b""
-        bytes_left = self.max_bytes - self.bytes_read
-        if size < 0:
-            bytes_to_read = bytes_left
-        else:
-            bytes_to_read = min(size, bytes_left)
-        data = self.buf.read(bytes_to_read)
-        self.bytes_read += len(data)
-        return data
-
-
 class wait_for_retry_after_header(wait_base):
     """Wait strategy that first looks for Retry-After header. If not
     present it uses the fallback strategy as the wait param"""

src/civis/base.py

@@ -10,7 +10,7 @@
 
 import civis
 from civis.response import PaginatedResponse, convert_response_data_type
-from civis._utils import retry_request, MAX_RETRIES
+from civis._utils import retry_request, DEFAULT_RETRYING
 
 FINISHED = ["success", "succeeded"]
 FAILED = ["failed"]
@@ -123,12 +123,17 @@ def _make_request(self, method, path=None, params=None, data=None, **kwargs):
         url = self._build_path(path)
 
         with self._lock:
+            if self._client._retrying is None:
+                retrying = self._session_kwargs.pop("retrying", None)
+                self._client._retrying = retrying if retrying else DEFAULT_RETRYING
             with open_session(**self._session_kwargs) as sess:
                 request = requests.Request(
                     method, url, json=data, params=params, **kwargs
                 )
                 pre_request = sess.prepare_request(request)
-                response = retry_request(method, pre_request, sess, MAX_RETRIES)
+                response = retry_request(
+                    method, pre_request, sess, self._client._retrying
+                )
 
         if response.status_code == 401:
             auth_error = response.headers["www-authenticate"]