From 1503bffc89852ab2b8868e1db3e0902dff1ee5a0 Mon Sep 17 00:00:00 2001 From: sshuair Date: Fri, 26 Feb 2021 14:24:04 +0800 Subject: [PATCH 1/7] add HTTP backend --- mmcv/fileio/file_client.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/mmcv/fileio/file_client.py b/mmcv/fileio/file_client.py index 26a86a6cf2..c43abb670a 100644 --- a/mmcv/fileio/file_client.py +++ b/mmcv/fileio/file_client.py @@ -191,6 +191,20 @@ def get_text(self, filepath): return value_buf +class HTTPBackend(BaseStorageBackend): + """HTTP and HTTPS storage bachend.""" + + def get(self, filepath): + from urllib.request import urlopen + value_buf = urlopen(filepath).read() + return value_buf + + def get_text(self, filepath): + from urllib.request import urlopen + value_buf = urlopen(filepath).read() + return value_buf.decode('utf-8') + + class FileClient: """A general file client to access files in different backend. @@ -210,6 +224,7 @@ class FileClient: 'memcached': MemcachedBackend, 'lmdb': LmdbBackend, 'petrel': PetrelBackend, + 'http': HTTPBackend, } def __init__(self, backend='disk', **kwargs): From 6259c23ba377744e6218be69b2f99838eafdc062 Mon Sep 17 00:00:00 2001 From: sshuair Date: Tue, 2 Mar 2021 19:58:27 +0800 Subject: [PATCH 2/7] add HTTPbackend --- mmcv/fileio/file_client.py | 3 +-- tests/test_fileclient.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/mmcv/fileio/file_client.py b/mmcv/fileio/file_client.py index c43abb670a..2b15453fb8 100644 --- a/mmcv/fileio/file_client.py +++ b/mmcv/fileio/file_client.py @@ -1,5 +1,6 @@ import inspect from abc import ABCMeta, abstractmethod +from urllib.request import urlopen class BaseStorageBackend(metaclass=ABCMeta): @@ -195,12 +196,10 @@ class HTTPBackend(BaseStorageBackend): """HTTP and HTTPS storage bachend.""" def get(self, filepath): - from urllib.request import urlopen value_buf = urlopen(filepath).read() return value_buf def get_text(self, filepath): - from urllib.request import urlopen value_buf = urlopen(filepath).read() return value_buf.decode('utf-8') diff --git a/tests/test_fileclient.py b/tests/test_fileclient.py index 37b4eee03e..27d1cc3efc 100644 --- a/tests/test_fileclient.py +++ b/tests/test_fileclient.py @@ -182,6 +182,34 @@ def test_lmdb_backend(self): img = mmcv.imfrombytes(img_bytes) assert img.shape == (120, 125, 3) + def test_http_backend(self): + def get_bytes(filepath): + with open(filepath, 'rb') as f: + content = f.read() + return content + http_backend = FileClient('http') + + # input is path or Path object + with pytest.raises(Exception): + http_backend.get(self.img_path) + with pytest.raises(Exception): + http_backend.get(str(self.img_path)) + with pytest.raises(Exception): + http_backend.get_text(self.text_path) + with pytest.raises(Exception): + http_backend.get_text(str(self.text_path)) + + # input url is http image + http_backend.get = MagicMock(return_value=get_bytes(self.img_path)) + img_bytes = http_backend.get(self.img_path) + img = mmcv.imfrombytes(img_bytes) + assert img.shape == self.img_shape + + # input url is http text + http_backend.get_text = MagicMock(return_value=get_bytes(self.text_path)) + value_buf = http_backend.get_text(self.text_path) + assert self.text_path.open('r').read() == value_buf.decode('utf-8') + def test_register_backend(self): # name must be a string From e54d26297bac2470afc734b0963529f85e346e35 Mon Sep 17 00:00:00 2001 From: sshuair Date: Tue, 2 Mar 2021 22:29:58 +0800 Subject: [PATCH 3/7] fix linting --- mmcv/fileio/file_client.py | 2 +- tests/test_fileclient.py | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/mmcv/fileio/file_client.py b/mmcv/fileio/file_client.py index 2b15453fb8..efb8d74b9e 100644 --- a/mmcv/fileio/file_client.py +++ b/mmcv/fileio/file_client.py @@ -198,7 +198,7 @@ class HTTPBackend(BaseStorageBackend): def get(self, filepath): value_buf = urlopen(filepath).read() return value_buf - + def get_text(self, filepath): value_buf = urlopen(filepath).read() return value_buf.decode('utf-8') diff --git a/tests/test_fileclient.py b/tests/test_fileclient.py index 27d1cc3efc..e7f0dc6076 100644 --- a/tests/test_fileclient.py +++ b/tests/test_fileclient.py @@ -200,13 +200,15 @@ def get_bytes(filepath): http_backend.get_text(str(self.text_path)) # input url is http image - http_backend.get = MagicMock(return_value=get_bytes(self.img_path)) + http_backend.get = MagicMock(return_value = get_bytes(self.img_path)) img_bytes = http_backend.get(self.img_path) img = mmcv.imfrombytes(img_bytes) assert img.shape == self.img_shape # input url is http text - http_backend.get_text = MagicMock(return_value=get_bytes(self.text_path)) + http_backend.get_text = MagicMock( + return_value = get_bytes(self.text_path) + ) value_buf = http_backend.get_text(self.text_path) assert self.text_path.open('r').read() == value_buf.decode('utf-8') From ef98c206112d5003b2b1591c0ba7fa3f00807fe4 Mon Sep 17 00:00:00 2001 From: sshuair Date: Tue, 2 Mar 2021 22:34:53 +0800 Subject: [PATCH 4/7] fix linting --- tests/test_fileclient.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_fileclient.py b/tests/test_fileclient.py index e7f0dc6076..acd42c2285 100644 --- a/tests/test_fileclient.py +++ b/tests/test_fileclient.py @@ -200,14 +200,14 @@ def get_bytes(filepath): http_backend.get_text(str(self.text_path)) # input url is http image - http_backend.get = MagicMock(return_value = get_bytes(self.img_path)) + http_backend.get = MagicMock(return_value=get_bytes(self.img_path)) img_bytes = http_backend.get(self.img_path) img = mmcv.imfrombytes(img_bytes) assert img.shape == self.img_shape # input url is http text http_backend.get_text = MagicMock( - return_value = get_bytes(self.text_path) + return_value=get_bytes(self.text_path) ) value_buf = http_backend.get_text(self.text_path) assert self.text_path.open('r').read() == value_buf.decode('utf-8') From 862dc2e8e66c20d6bea6cb2009d3ea57e9bc970d Mon Sep 17 00:00:00 2001 From: sshuair Date: Tue, 2 Mar 2021 22:49:54 +0800 Subject: [PATCH 5/7] add docs attributes http --- mmcv/fileio/file_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mmcv/fileio/file_client.py b/mmcv/fileio/file_client.py index efb8d74b9e..f496f6ee4d 100644 --- a/mmcv/fileio/file_client.py +++ b/mmcv/fileio/file_client.py @@ -213,7 +213,7 @@ class FileClient: Attributes: backend (str): The storage backend type. Options are "disk", "ceph", - "memcached" and "lmdb". + "memcached", "lmdb" and "http". client (:obj:`BaseStorageBackend`): The backend object. """ From d40888004cb88ff8d8e54d054707fcc12f389d66 Mon Sep 17 00:00:00 2001 From: sshuair Date: Tue, 2 Mar 2021 22:56:57 +0800 Subject: [PATCH 6/7] fix format --- tests/test_fileclient.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/test_fileclient.py b/tests/test_fileclient.py index acd42c2285..31e4dcd226 100644 --- a/tests/test_fileclient.py +++ b/tests/test_fileclient.py @@ -183,10 +183,12 @@ def test_lmdb_backend(self): assert img.shape == (120, 125, 3) def test_http_backend(self): + def get_bytes(filepath): with open(filepath, 'rb') as f: content = f.read() return content + http_backend = FileClient('http') # input is path or Path object @@ -207,8 +209,7 @@ def get_bytes(filepath): # input url is http text http_backend.get_text = MagicMock( - return_value=get_bytes(self.text_path) - ) + return_value=get_bytes(self.text_path)) value_buf = http_backend.get_text(self.text_path) assert self.text_path.open('r').read() == value_buf.decode('utf-8') From 96a63aaef0faa18a7b38fb2e022fe8335aa73c75 Mon Sep 17 00:00:00 2001 From: sshuair Date: Wed, 3 Mar 2021 14:30:20 +0800 Subject: [PATCH 7/7] add real http link test for http backend --- tests/test_fileclient.py | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/tests/test_fileclient.py b/tests/test_fileclient.py index 31e4dcd226..80357cf31d 100644 --- a/tests/test_fileclient.py +++ b/tests/test_fileclient.py @@ -183,13 +183,11 @@ def test_lmdb_backend(self): assert img.shape == (120, 125, 3) def test_http_backend(self): - - def get_bytes(filepath): - with open(filepath, 'rb') as f: - content = f.read() - return content - http_backend = FileClient('http') + img_url = 'https://raw.githubusercontent.com/open-mmlab/mmcv/' \ + 'master/tests/data/color.jpg' + text_url = 'https://raw.githubusercontent.com/open-mmlab/mmcv/' \ + 'master/tests/data/filelist.txt' # input is path or Path object with pytest.raises(Exception): @@ -202,16 +200,13 @@ def get_bytes(filepath): http_backend.get_text(str(self.text_path)) # input url is http image - http_backend.get = MagicMock(return_value=get_bytes(self.img_path)) - img_bytes = http_backend.get(self.img_path) + img_bytes = http_backend.get(img_url) img = mmcv.imfrombytes(img_bytes) assert img.shape == self.img_shape # input url is http text - http_backend.get_text = MagicMock( - return_value=get_bytes(self.text_path)) - value_buf = http_backend.get_text(self.text_path) - assert self.text_path.open('r').read() == value_buf.decode('utf-8') + value_buf = http_backend.get_text(text_url) + assert self.text_path.open('r').read() == value_buf def test_register_backend(self):