Add retries to HTTP client

This uses the urllib3 Retry to add retries with back-off to requests
to the API-server that error. It will retry errors with the listed
statuses, for all HTTP methods. If the response contains a Retry-After
header (which we expect to be the case for 429/Too Many Requests), the
delay it specifies will be respected. For other cases, it will back
off exponentially (after one immediate retry, doubling) up to 10 times,
with the library's maximum delay (120s).

Fixes #72

k8s/client.py

@@ -21,6 +21,8 @@
 
 import requests
 from requests import RequestException
+from requests.adapters import HTTPAdapter
+from requests.packages.urllib3.util.retry import Retry
 
 from . import config
 
@@ -72,12 +74,28 @@ class ClientError(K8sClientException):
     """The client made a bad request"""
 
 
+def _session_factory():
+    """Retry on errors from the API-server. Retry-After header will be respected first, which
+    we expect to be set for too_many_requests, and for other errors it will back-off exponentially up
+    to the 120s maximum"""
+    session = requests.Session()
+    retry_statuses = [requests.codes.too_many_requests,
+                      requests.codes.internal_server_error,
+                      requests.codes.bad_gateway,
+                      requests.codes.service_unavailable,
+                      requests.codes.gateway_timeout]
+    retries = Retry(total=10, backoff_factor=1, status_forcelist=retry_statuses, method_whitelist=False)
+    session.mount('http://', HTTPAdapter(max_retries=retries))
+    session.mount('https://', HTTPAdapter(max_retries=retries))
+    return session
+
+
 class Client(object):
-    _session = requests.Session()
+    _session = _session_factory()
 
     @classmethod
     def clear_session(cls):
-        cls._session = requests.Session()
+        cls._session = _session_factory()
 
     @classmethod
     def init_session(cls):

tests/k8s/test_client.py

@@ -21,7 +21,9 @@
 
 from k8s import config
 from k8s.base import Model, Field
-from k8s.client import Client, SENSITIVE_HEADERS
+from k8s.client import Client, SENSITIVE_HEADERS, _session_factory
+
+import requests
 
 
 @pytest.mark.usefixtures("k8s_config")
@@ -50,6 +52,14 @@ def url(self):
     def explicit_timeout(self):
         return 60
 
+    @pytest.mark.parametrize("url", ["http://api.k8s.example.com", "https://api.k8s.example.com"])
+    def test_session_configured_for_retry(self, url):
+        session = _session_factory()
+        adapter = session.get_adapter(url)
+        assert adapter.max_retries.total > 0
+        assert requests.codes.too_many_requests in adapter.max_retries.status_forcelist
+        assert requests.codes.ok not in adapter.max_retries.status_forcelist
+
     def test_get_should_use_default_timeout(self, session, client, url):
         client.get(url)
         session.request.assert_called_once_with("GET", _absolute_url(url), json=None, timeout=config.timeout)