diff --git a/k8s/client.py b/k8s/client.py index 4c0a055..f923a57 100644 --- a/k8s/client.py +++ b/k8s/client.py @@ -21,6 +21,8 @@ import requests from requests import RequestException +from requests.adapters import HTTPAdapter +from requests.packages.urllib3.util.retry import Retry from . import config @@ -72,12 +74,28 @@ class ClientError(K8sClientException): """The client made a bad request""" +def _session_factory(): + """Retry on errors from the API-server. Retry-After header will be respected first, which + we expect to be set for too_many_requests, and for other errors it will back-off exponentially up + to the 120s maximum""" + session = requests.Session() + retry_statuses = [requests.codes.too_many_requests, + requests.codes.internal_server_error, + requests.codes.bad_gateway, + requests.codes.service_unavailable, + requests.codes.gateway_timeout] + retries = Retry(total=10, backoff_factor=1, status_forcelist=retry_statuses, method_whitelist=False) + session.mount('http://', HTTPAdapter(max_retries=retries)) + session.mount('https://', HTTPAdapter(max_retries=retries)) + return session + + class Client(object): - _session = requests.Session() + _session = _session_factory() @classmethod def clear_session(cls): - cls._session = requests.Session() + cls._session = _session_factory() @classmethod def init_session(cls): diff --git a/tests/k8s/test_client.py b/tests/k8s/test_client.py index b93cf48..a0dc552 100644 --- a/tests/k8s/test_client.py +++ b/tests/k8s/test_client.py @@ -21,7 +21,9 @@ from k8s import config from k8s.base import Model, Field -from k8s.client import Client, SENSITIVE_HEADERS +from k8s.client import Client, SENSITIVE_HEADERS, _session_factory + +import requests @pytest.mark.usefixtures("k8s_config") @@ -50,6 +52,14 @@ def url(self): def explicit_timeout(self): return 60 + @pytest.mark.parametrize("url", ["http://api.k8s.example.com", "https://api.k8s.example.com"]) + def test_session_configured_for_retry(self, url): + session = _session_factory() + adapter = session.get_adapter(url) + assert adapter.max_retries.total > 0 + assert requests.codes.too_many_requests in adapter.max_retries.status_forcelist + assert requests.codes.ok not in adapter.max_retries.status_forcelist + def test_get_should_use_default_timeout(self, session, client, url): client.get(url) session.request.assert_called_once_with("GET", _absolute_url(url), json=None, timeout=config.timeout)