Merge pull request #73 from fiaas/client-retry

Add retries to HTTP client

k8s/client.py

@@ -21,6 +21,8 @@
 
 import requests
 from requests import RequestException
+from requests.adapters import HTTPAdapter
+from requests.packages.urllib3.util.retry import Retry
 
 from . import config
 
@@ -72,12 +74,28 @@ class ClientError(K8sClientException):
     """The client made a bad request"""
 
 
+def _session_factory():
+    """Retry on errors from the API-server. Retry-After header will be respected first, which
+    we expect to be set for too_many_requests, and for other errors it will back-off exponentially up
+    to the 120s maximum"""
+    session = requests.Session()
+    retry_statuses = [requests.codes.too_many_requests,
+                      requests.codes.internal_server_error,
+                      requests.codes.bad_gateway,
+                      requests.codes.service_unavailable,
+                      requests.codes.gateway_timeout]
+    retries = Retry(total=10, backoff_factor=1, status_forcelist=retry_statuses, method_whitelist=False)
+    session.mount('http://', HTTPAdapter(max_retries=retries))
+    session.mount('https://', HTTPAdapter(max_retries=retries))
+    return session
+
+
 class Client(object):
-    _session = requests.Session()
+    _session = _session_factory()
 
     @classmethod
     def clear_session(cls):
-        cls._session = requests.Session()
+        cls._session = _session_factory()
 
     @classmethod
     def init_session(cls):

tests/k8s/test_client.py

@@ -21,7 +21,9 @@
 
 from k8s import config
 from k8s.base import Model, Field
-from k8s.client import Client, SENSITIVE_HEADERS
+from k8s.client import Client, SENSITIVE_HEADERS, _session_factory
+
+import requests
 
 
 @pytest.mark.usefixtures("k8s_config")
@@ -50,6 +52,14 @@ def url(self):
     def explicit_timeout(self):
         return 60
 
+    @pytest.mark.parametrize("url", ["http://api.k8s.example.com", "https://api.k8s.example.com"])
+    def test_session_configured_for_retry(self, url):
+        session = _session_factory()
+        adapter = session.get_adapter(url)
+        assert adapter.max_retries.total > 0
+        assert requests.codes.too_many_requests in adapter.max_retries.status_forcelist
+        assert requests.codes.ok not in adapter.max_retries.status_forcelist
+
     def test_get_should_use_default_timeout(self, session, client, url):
         client.get(url)
         session.request.assert_called_once_with("GET", _absolute_url(url), json=None, timeout=config.timeout)