From 30d29af74a98ad83a200a9054f083a1560479d5c Mon Sep 17 00:00:00 2001 From: Yuichiro Ueno Date: Fri, 16 Dec 2022 16:09:37 +0900 Subject: [PATCH 1/6] Add error handling --- pfio/cache/http_cache.py | 48 +++++++++++++++++++++++++++++----------- 1 file changed, 35 insertions(+), 13 deletions(-) diff --git a/pfio/cache/http_cache.py b/pfio/cache/http_cache.py index 916476d3..5ceea824 100644 --- a/pfio/cache/http_cache.py +++ b/pfio/cache/http_cache.py @@ -1,11 +1,16 @@ import os import pickle import time +import logging import urllib3 +import urllib3.exceptions from pfio.cache import Cache +logger = logging.getLogger(__name__) +logger.addHandler(logging.StreamHandler()) + class HTTPCache(Cache): """HTTP-based cache system @@ -86,27 +91,44 @@ def put(self, i, data): if self.do_pickle: data = pickle.dumps(data) - res = self.conn.urlopen("PUT", - url=self._url(i), - headers=self._header_with_token(), - body=data) - return res.status == 201 + try: + res = self.conn.urlopen("PUT", + url=self._url(i), + headers=self._header_with_token(), + body=data) + except urllib3.exceptions.RequestError as e: + logger.warning("put: {}".format(e)) + return False + + if res.status == 201: + return True + + logger.warning("put: unexpected status code {}".format(res.status)) + return False def get(self, i): if i < 0 or self.length <= i: raise IndexError("index {} out of range ([0, {}])" .format(i, self.length - 1)) - res = self.conn.urlopen("GET", - url=self._url(i), - headers=self._header_with_token()) - if res.status != 200 or res.data is None: + try: + res = self.conn.urlopen("GET", + url=self._url(i), + headers=self._header_with_token()) + except urllib3.exceptions.RequestError as e: + logger.warning("get: {}".format(e)) return None - if self.do_pickle: - return pickle.loads(res.data) - else: - return res.data + if res.status == 200: + if self.do_pickle: + return pickle.loads(res.data) + else: + return res.data + elif res.status == 404: + return None + + logger.warning("get: unexpected status code {}".format(res.status)) + return None def _url(self, i) -> str: return self.url + str(i) From 6f09a673d8c6c62c4ac5098dc7a79c4a172463e2 Mon Sep 17 00:00:00 2001 From: Yuichiro Ueno Date: Fri, 16 Dec 2022 16:09:52 +0900 Subject: [PATCH 2/6] Allow redirect or retry once --- pfio/cache/http_cache.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pfio/cache/http_cache.py b/pfio/cache/http_cache.py index 5ceea824..797acdd0 100644 --- a/pfio/cache/http_cache.py +++ b/pfio/cache/http_cache.py @@ -71,7 +71,8 @@ def __init__(self, self.do_pickle = do_pickle - self.conn = urllib3.poolmanager.PoolManager() + # Allow redirect or retry once + self.conn = urllib3.poolmanager.PoolManager(retries=1) def __len__(self): return self.length From 0667f9f5efc7fa0fe23208de7f045d38d6f901c4 Mon Sep 17 00:00:00 2001 From: Yuichiro Ueno Date: Fri, 16 Dec 2022 17:47:01 +0900 Subject: [PATCH 3/6] Shorter timeout --- pfio/cache/http_cache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pfio/cache/http_cache.py b/pfio/cache/http_cache.py index 797acdd0..03cb6e06 100644 --- a/pfio/cache/http_cache.py +++ b/pfio/cache/http_cache.py @@ -72,7 +72,7 @@ def __init__(self, self.do_pickle = do_pickle # Allow redirect or retry once - self.conn = urllib3.poolmanager.PoolManager(retries=1) + self.conn = urllib3.poolmanager.PoolManager(retries=1, timeout=3) def __len__(self): return self.length From 132da57d5b9412839dfa2e2ad1d48eed569b1f6b Mon Sep 17 00:00:00 2001 From: Yuichiro Ueno Date: Mon, 19 Dec 2022 14:25:41 +0900 Subject: [PATCH 4/6] Don't pickle conn in HTTPCache --- pfio/cache/http_cache.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/pfio/cache/http_cache.py b/pfio/cache/http_cache.py index 03cb6e06..43eff7ba 100644 --- a/pfio/cache/http_cache.py +++ b/pfio/cache/http_cache.py @@ -71,9 +71,32 @@ def __init__(self, self.do_pickle = do_pickle + self.conn = None + self._prepare_conn() + + self.pid = os.getpid() + + @property + def is_forked(self): + return self.pid != os.getpid() + + def _checkfork(self): + if self.is_forked: + self._prepare_conn() + self.pid = os.getpid() + + def _prepare_conn(self): # Allow redirect or retry once self.conn = urllib3.poolmanager.PoolManager(retries=1, timeout=3) + def __getstate__(self): + state = self.__dict__.copy() + state['conn'] = None + return state + + def __setstate__(self, state): + self.__dict__ = state + def __len__(self): return self.length @@ -86,6 +109,7 @@ def multithread_safe(self): return True def put(self, i, data): + self._checkfork() if i < 0 or self.length <= i: raise IndexError("index {} out of range ([0, {}])" .format(i, self.length - 1)) @@ -108,6 +132,7 @@ def put(self, i, data): return False def get(self, i): + self._checkfork() if i < 0 or self.length <= i: raise IndexError("index {} out of range ([0, {}])" .format(i, self.length - 1)) From e99563245c9a4a4b557e17b2b9f8a9db2f72b296 Mon Sep 17 00:00:00 2001 From: Yuichiro Ueno Date: Mon, 19 Dec 2022 19:39:32 +0900 Subject: [PATCH 5/6] Make isort happy --- pfio/cache/http_cache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pfio/cache/http_cache.py b/pfio/cache/http_cache.py index 43eff7ba..04dc4d2d 100644 --- a/pfio/cache/http_cache.py +++ b/pfio/cache/http_cache.py @@ -1,7 +1,7 @@ +import logging import os import pickle import time -import logging import urllib3 import urllib3.exceptions From 42a8e161670437adfb3e1c0e92db84181cd5d358 Mon Sep 17 00:00:00 2001 From: Yuichiro Ueno Date: Tue, 20 Dec 2022 12:54:36 +0900 Subject: [PATCH 6/6] Check self.conn is None --- pfio/cache/http_cache.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pfio/cache/http_cache.py b/pfio/cache/http_cache.py index 04dc4d2d..6447c64f 100644 --- a/pfio/cache/http_cache.py +++ b/pfio/cache/http_cache.py @@ -80,8 +80,8 @@ def __init__(self, def is_forked(self): return self.pid != os.getpid() - def _checkfork(self): - if self.is_forked: + def _checkconn(self): + if self.is_forked or self.conn is None: self._prepare_conn() self.pid = os.getpid() @@ -109,7 +109,7 @@ def multithread_safe(self): return True def put(self, i, data): - self._checkfork() + self._checkconn() if i < 0 or self.length <= i: raise IndexError("index {} out of range ([0, {}])" .format(i, self.length - 1)) @@ -132,7 +132,7 @@ def put(self, i, data): return False def get(self, i): - self._checkfork() + self._checkconn() if i < 0 or self.length <= i: raise IndexError("index {} out of range ([0, {}])" .format(i, self.length - 1))