diff --git a/numismatic/feeds/base.py b/numismatic/feeds/base.py index 3919965..6b7d498 100644 --- a/numismatic/feeds/base.py +++ b/numismatic/feeds/base.py @@ -249,7 +249,12 @@ def __requester_validator(self, attribute, value): raise ValueError(f'{attribute.name}: {value}') def _make_request(self, api_url, params=None, headers=None, raw=False): - response = self.requester.get(api_url, params=params, headers=headers) + try: + response = self.requester.get(api_url, params=params, headers=headers) + except Exception as ex: + logger.error(ex) + logger.error(packet) + raise ex if not raw: data = response.json() else: diff --git a/numismatic/numismatic.ini b/numismatic/numismatic.ini index deb5bb7..85b6d6a 100755 --- a/numismatic/numismatic.ini +++ b/numismatic/numismatic.ini @@ -4,6 +4,12 @@ assets = BTC currencies = USD channels = trades +[REQUESTER] +retries = 3 +backoff_factor = 0.3 +status_forcelist=500,502,503,504 +timeout=5 + [BitfinexFeed] [BraveNewCoinFeed] diff --git a/numismatic/requesters.py b/numismatic/requesters.py index f052adb..e4647f7 100644 --- a/numismatic/requesters.py +++ b/numismatic/requesters.py @@ -1,5 +1,7 @@ import logging import requests +from requests.adapters import HTTPAdapter +from requests.packages.urllib3.util.retry import Retry import time from pathlib import Path from functools import partial @@ -9,6 +11,7 @@ import attr from appdirs import user_cache_dir +from .config import config_item_getter log = logging.getLogger(__name__) @@ -21,6 +24,20 @@ class Requester: "Basic Requester using requests and blocking calls" + retries = attr.ib(default=attr.Factory( + config_item_getter('REQUESTER', 'retries')), + convert=int) + timeout = attr.ib(default=attr.Factory( + config_item_getter('REQUESTER', 'timeout')), + convert=int) + backoff_factor = attr.ib(default=attr.Factory( + config_item_getter('REQUESTER', 'backoff_factor')), + convert=float) + status_forcelist = attr.ib(default=attr.Factory( + config_item_getter('REQUESTER', 'status_forcelist')), + convert=lambda val: tuple(map(int, val.split(','))), + validator=attr.validators.instance_of(tuple)) + @classmethod def factory(cls, requester, **kwargs): requester = '' if requester is None else requester.lower() @@ -36,7 +53,18 @@ def factory(cls, requester, **kwargs): return subcls(**kwds) def get(self, url, params=None, headers=None): - response = requests.get(url, params=params, headers=headers) + session = requests.Session() + retry = Retry( + total=self.retries, + read=self.retries, + connect=self.retries, + backoff_factor=self.backoff_factor, + status_forcelist=self.status_forcelist, + ) + adapter = HTTPAdapter(max_retries=retry) + session.mount('http://', adapter) + session.mount('https://', adapter) + response = session.get(url, params=params, headers=headers, timeout=self.timeout) return response