From 4777d97e627c9da0d7b8cff99a5ca3181d797641 Mon Sep 17 00:00:00 2001 From: Greg Jones Date: Wed, 6 Nov 2019 11:12:24 +0100 Subject: [PATCH] Add retries to HTTP client This uses the urllib3 Retry to add retries with back-off to requests to the API-server that error. It will retry errors with the listed statuses, for all HTTP methods. If the response contains a Retry-After header (which we expect to be the case for 429/Too Many Requests), the delay it specifies will be respected. For other cases, it will back off exponentially (after one immediate retry, doubling) up to 10 times, with the library's maximum delay (120s). Fixes #72 --- k8s/client.py | 22 ++++++++++++++++++++-- tests/k8s/test_client.py | 12 +++++++++++- 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/k8s/client.py b/k8s/client.py index 4c0a055..f923a57 100644 --- a/k8s/client.py +++ b/k8s/client.py @@ -21,6 +21,8 @@ import requests from requests import RequestException +from requests.adapters import HTTPAdapter +from requests.packages.urllib3.util.retry import Retry from . import config @@ -72,12 +74,28 @@ class ClientError(K8sClientException): """The client made a bad request""" +def _session_factory(): + """Retry on errors from the API-server. Retry-After header will be respected first, which + we expect to be set for too_many_requests, and for other errors it will back-off exponentially up + to the 120s maximum""" + session = requests.Session() + retry_statuses = [requests.codes.too_many_requests, + requests.codes.internal_server_error, + requests.codes.bad_gateway, + requests.codes.service_unavailable, + requests.codes.gateway_timeout] + retries = Retry(total=10, backoff_factor=1, status_forcelist=retry_statuses, method_whitelist=False) + session.mount('http://', HTTPAdapter(max_retries=retries)) + session.mount('https://', HTTPAdapter(max_retries=retries)) + return session + + class Client(object): - _session = requests.Session() + _session = _session_factory() @classmethod def clear_session(cls): - cls._session = requests.Session() + cls._session = _session_factory() @classmethod def init_session(cls): diff --git a/tests/k8s/test_client.py b/tests/k8s/test_client.py index b93cf48..a0dc552 100644 --- a/tests/k8s/test_client.py +++ b/tests/k8s/test_client.py @@ -21,7 +21,9 @@ from k8s import config from k8s.base import Model, Field -from k8s.client import Client, SENSITIVE_HEADERS +from k8s.client import Client, SENSITIVE_HEADERS, _session_factory + +import requests @pytest.mark.usefixtures("k8s_config") @@ -50,6 +52,14 @@ def url(self): def explicit_timeout(self): return 60 + @pytest.mark.parametrize("url", ["http://api.k8s.example.com", "https://api.k8s.example.com"]) + def test_session_configured_for_retry(self, url): + session = _session_factory() + adapter = session.get_adapter(url) + assert adapter.max_retries.total > 0 + assert requests.codes.too_many_requests in adapter.max_retries.status_forcelist + assert requests.codes.ok not in adapter.max_retries.status_forcelist + def test_get_should_use_default_timeout(self, session, client, url): client.get(url) session.request.assert_called_once_with("GET", _absolute_url(url), json=None, timeout=config.timeout)