Skip to content

Commit

Permalink
fileio related interfaces should keep bc (open-mmlab#2539)
Browse files Browse the repository at this point in the history 
* fileio related interfaces should keep bc

* fix format

* update unit tests of transforms

* update format

* lazily infer client until calling get function of fileio
  • Loading branch information
@zhouzaida @akozlov-outrider
zhouzaida authored and akozlov-outrider committed May 8, 2023
1 parent 636fb8d commit 505f4fb
Show file tree
Hide file tree
Showing 4 changed files with 175 additions and 17 deletions.
46 changes: 44 additions & 2 deletions mmcv/image/io.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -145,6 +145,7 @@ def imread(img_or_path: Union[np.ndarray, str, Path],
flag: str = 'color',
channel_order: str = 'bgr',
backend: Optional[str] = None,
file_client_args: Optional[dict] = None,
*,
backend_args: Optional[dict] = None) -> np.ndarray:
"""Read an image.
Expand All @@ -166,12 +167,18 @@ def imread(img_or_path: Union[np.ndarray, str, Path],
`cv2`, `pillow`, `turbojpeg`, `tifffile`, `None`.
If backend is None, the global imread_backend specified by
``mmcv.use_backend()`` will be used. Default: None.
file_client_args (dict, optional): Arguments to instantiate a
FileClient. See :class:`mmengine.fileio.FileClient` for details.
Default: None. It will be deprecated in future. Please use
``backend_args`` instead.
Deprecated in version 2.0.0rc4.
backend_args (dict, optional): Instantiates the corresponding file
backend. It may contain `backend` key to specify the file
backend. If it contains, the file backend corresponding to this
value will be used and initialized with the remaining values,
otherwise the corresponding file backend will be selected
based on the prefix of the file path. Defaults to None.
New in version 2.0.0rc4.
Returns:
ndarray: Loaded image array.
Expand All @@ -195,13 +202,27 @@ def imread(img_or_path: Union[np.ndarray, str, Path],
>>> img = mmcv.imread(http_img_path, backend_args={
... 'backend': 'http'})
"""
if file_client_args is not None:
warnings.warn(
'"file_client_args" will be deprecated in future. '
'Please use "backend_args" instead', DeprecationWarning)
if backend_args is not None:
raise ValueError(
'"file_client_args" and "backend_args" cannot be set at the '
'same time.')

if isinstance(img_or_path, Path):
img_or_path = str(img_or_path)

if isinstance(img_or_path, np.ndarray):
return img_or_path
elif is_str(img_or_path):
img_bytes = fileio.get(img_or_path, backend_args=backend_args)
if file_client_args is not None:
file_client = fileio.FileClient.infer_client(
file_client_args, img_or_path)
img_bytes = file_client.get(img_or_path)
else:
img_bytes = fileio.get(img_or_path, backend_args=backend_args)
return imfrombytes(img_bytes, flag, channel_order, backend)
else:
raise TypeError('"img" must be a numpy array or a str or '
Expand Down Expand Up @@ -271,6 +292,7 @@ def imwrite(img: np.ndarray,
file_path: str,
params: Optional[list] = None,
auto_mkdir: Optional[bool] = None,
file_client_args: Optional[dict] = None,
*,
backend_args: Optional[dict] = None) -> bool:
"""Write image to file.
Expand All @@ -285,12 +307,18 @@ def imwrite(img: np.ndarray,
params (None or list): Same as opencv :func:`imwrite` interface.
auto_mkdir (bool): If the parent folder of `file_path` does not exist,
whether to create it automatically. It will be deprecated.
file_client_args (dict, optional): Arguments to instantiate a
FileClient. See :class:`mmengine.fileio.FileClient` for details.
Default: None. It will be deprecated in future. Please use
``backend_args`` instead.
Deprecated in version 2.0.0rc4.
backend_args (dict, optional): Instantiates the corresponding file
backend. It may contain `backend` key to specify the file
backend. If it contains, the file backend corresponding to this
value will be used and initialized with the remaining values,
otherwise the corresponding file backend will be selected
based on the prefix of the file path. Defaults to None.
New in version 2.0.0rc4.
Returns:
bool: Successful or not.
Expand All @@ -304,6 +332,15 @@ def imwrite(img: np.ndarray,
>>> ret = mmcv.imwrite(img, 's3://bucket/img.jpg', backend_args={
... 'backend': 'petrel'})
"""
if file_client_args is not None:
warnings.warn(
'"file_client_args" will be deprecated in future. '
'Please use "backend_args" instead', DeprecationWarning)
if backend_args is not None:
raise ValueError(
'"file_client_args" and "backend_args" cannot be set at the '
'same time.')

assert is_filepath(file_path)
file_path = str(file_path)
if auto_mkdir is not None:
Expand All @@ -317,6 +354,11 @@ def imwrite(img: np.ndarray,
# format is '.jpg'.
flag, img_buff = cv2.imencode(img_ext, img, params)

fileio.put(img_buff.tobytes(), file_path, backend_args=backend_args)
if file_client_args is not None:
file_client = fileio.FileClient.infer_client(file_client_args,
file_path)
file_client.put(img_buff.tobytes(), file_path)
else:
fileio.put(img_buff.tobytes(), file_path, backend_args=backend_args)

return flag
80 changes: 67 additions & 13 deletions mmcv/transforms/loading.py
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
# Copyright (c) OpenMMLab. All rights reserved.
import warnings
from typing import Optional

import mmengine.fileio as fileio
Expand Down Expand Up @@ -33,6 +34,11 @@ class LoadImageFromFile(BaseTransform):
argument for :func:`mmcv.imfrombytes`.
See :func:`mmcv.imfrombytes` for details.
Defaults to 'cv2'.
file_client_args (dict, optional): Arguments to instantiate a
FileClient. See :class:`mmengine.fileio.FileClient` for details.
Defaults to None. It will be deprecated in future. Please use
``backend_args`` instead.
Deprecated in version 2.0.0rc4.
ignore_empty (bool): Whether to allow loading empty image or file path
not existent. Defaults to False.
backend_args (dict, optional): Instantiates the corresponding file
Expand All @@ -41,20 +47,36 @@ class LoadImageFromFile(BaseTransform):
value will be used and initialized with the remaining values,
otherwise the corresponding file backend will be selected
based on the prefix of the file path. Defaults to None.
New in version 2.0.0rc4.
"""

def __init__(self,
to_float32: bool = False,
color_type: str = 'color',
imdecode_backend: str = 'cv2',
file_client_args: Optional[dict] = None,
ignore_empty: bool = False,
*,
backend_args: Optional[dict] = None) -> None:
self.ignore_empty = ignore_empty
self.to_float32 = to_float32
self.color_type = color_type
self.imdecode_backend = imdecode_backend
self.backend_args = backend_args

self.file_client_args: Optional[dict] = None
self.backend_args: Optional[dict] = None
if file_client_args is not None:
warnings.warn(
'"file_client_args" will be deprecated in future. '
'Please use "backend_args" instead', DeprecationWarning)
if backend_args is not None:
raise ValueError(
'"file_client_args" and "backend_args" cannot be set '
'at the same time.')

self.file_client_args = file_client_args.copy()
if backend_args is not None:
self.backend_args = backend_args.copy()

def transform(self, results: dict) -> Optional[dict]:
"""Functions to load image.
Expand All @@ -69,7 +91,13 @@ def transform(self, results: dict) -> Optional[dict]:

filename = results['img_path']
try:
img_bytes = fileio.get(filename, backend_args=self.backend_args)
if self.file_client_args is not None:
file_client = fileio.FileClient.infer_client(
self.file_client_args, filename)
img_bytes = file_client.get(filename)
else:
img_bytes = fileio.get(
filename, backend_args=self.backend_args)
img = mmcv.imfrombytes(
img_bytes, flag=self.color_type, backend=self.imdecode_backend)
except Exception as e:
Expand All @@ -90,12 +118,12 @@ def __repr__(self):
f'ignore_empty={self.ignore_empty}, '
f'to_float32={self.to_float32}, '
f"color_type='{self.color_type}', "
f"imdecode_backend='{self.imdecode_backend}'")
f"imdecode_backend='{self.imdecode_backend}', ")

if self.backend_args is not None:
repr_str += f', backend_args={self.backend_args})'
if self.file_client_args is not None:
repr_str += f'file_client_args={self.file_client_args})'
else:
repr_str += ')'
repr_str += f'backend_args={self.backend_args})'

return repr_str

Expand Down Expand Up @@ -177,12 +205,18 @@ class LoadAnnotations(BaseTransform):
argument for :func:`mmcv.imfrombytes`.
See :func:`mmcv.imfrombytes` for details.
Defaults to 'cv2'.
file_client_args (dict, optional): Arguments to instantiate a
FileClient. See :class:`mmengine.fileio.FileClient` for details.
Defaults to None. It will be deprecated in future. Please use
``backend_args`` instead.
Deprecated in version 2.0.0rc4.
backend_args (dict, optional): Instantiates the corresponding file
backend. It may contain `backend` key to specify the file
backend. If it contains, the file backend corresponding to this
value will be used and initialized with the remaining values,
otherwise the corresponding file backend will be selected
based on the prefix of the file path. Defaults to None.
New in version 2.0.0rc4.
"""

def __init__(
Expand All @@ -192,6 +226,7 @@ def __init__(
with_seg: bool = False,
with_keypoints: bool = False,
imdecode_backend: str = 'cv2',
file_client_args: Optional[dict] = None,
*,
backend_args: Optional[dict] = None,
) -> None:
Expand All @@ -201,7 +236,21 @@ def __init__(
self.with_seg = with_seg
self.with_keypoints = with_keypoints
self.imdecode_backend = imdecode_backend
self.backend_args = backend_args

self.file_client_args: Optional[dict] = None
self.backend_args: Optional[dict] = None
if file_client_args is not None:
warnings.warn(
'"file_client_args" will be deprecated in future. '
'Please use "backend_args" instead', DeprecationWarning)
if backend_args is not None:
raise ValueError(
'"file_client_args" and "backend_args" cannot be set '
'at the same time.')

self.file_client_args = file_client_args.copy()
if backend_args is not None:
self.backend_args = backend_args.copy()

def _load_bboxes(self, results: dict) -> None:
"""Private function to load bounding box annotations.
Expand Down Expand Up @@ -245,9 +294,14 @@ def _load_seg_map(self, results: dict) -> None:
Returns:
dict: The dict contains loaded semantic segmentation annotations.
"""
if self.file_client_args is not None:
file_client = fileio.FileClient.infer_client(
self.file_client_args, results['seg_map_path'])
img_bytes = file_client.get(results['seg_map_path'])
else:
img_bytes = fileio.get(
results['seg_map_path'], backend_args=self.backend_args)

img_bytes = fileio.get(
results['seg_map_path'], backend_args=self.backend_args)
results['gt_seg_map'] = mmcv.imfrombytes(
img_bytes, flag='unchanged',
backend=self.imdecode_backend).squeeze()
Expand Down Expand Up @@ -296,11 +350,11 @@ def __repr__(self) -> str:
repr_str += f'with_label={self.with_label}, '
repr_str += f'with_seg={self.with_seg}, '
repr_str += f'with_keypoints={self.with_keypoints}, '
repr_str += f"imdecode_backend='{self.imdecode_backend}'"
repr_str += f"imdecode_backend='{self.imdecode_backend}', "

if self.backend_args is not None:
repr_str += f', backend_args={self.backend_args})'
if self.file_client_args is not None:
repr_str += f'file_client_args={self.file_client_args})'
else:
repr_str += ')'
repr_str += f'backend_args={self.backend_args})'

return repr_str
45 changes: 45 additions & 0 deletions tests/test_image/test_io.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,15 @@ def test_imread(self):
# backend cv2
mmcv.use_backend('cv2')

# file_client_args and backend_args can not be both set
with pytest.raises(
ValueError,
match='"file_client_args" and "backend_args" cannot be set'):
mmcv.imread(
self.img_path,
file_client_args={'backend': 'disk'},
backend_args={'backend': 'disk'})

# HardDiskBackend
img_cv2_color_bgr = mmcv.imread(self.img_path)
assert img_cv2_color_bgr.shape == (300, 400, 3)
Expand Down Expand Up @@ -95,6 +104,16 @@ def test_imread(self):
PetrelBackend, 'get',
return_value=img_cv2_color_bgr) as mock_method:
img_cv2_color_bgr_petrel = mmcv.imread(self.s3_path, backend='cv2')
img_cv2_color_bgr_petrel_with_args = mmcv.imread(
self.s3_path,
backend='cv2',
file_client_args={'backend': 'petrel'})
mock_method.assert_called()
assert_array_equal(img_cv2_color_bgr_petrel,
img_cv2_color_bgr_petrel_with_args)

mock_method.reset_mock()

img_cv2_color_bgr_petrel_with_args = mmcv.imread(
self.s3_path,
backend='cv2',
Expand All @@ -109,6 +128,16 @@ def test_imread(self):
HTTPBackend, 'get',
return_value=img_cv2_color_bgr) as mock_method:
img_cv2_color_bgr_http = mmcv.imread(self.http_path, backend='cv2')
img_cv2_color_bgr_http_with_args = mmcv.imread(
self.http_path,
backend='cv2',
file_client_args={'backend': 'http'})
mock_method.assert_called()
assert_array_equal(img_cv2_color_bgr_http,
img_cv2_color_bgr_http_with_args)

mock_method.reset_mock()

img_cv2_color_bgr_http_with_args = mmcv.imread(
self.http_path,
backend='cv2',
Expand Down Expand Up @@ -358,6 +387,16 @@ def test_imwrite(self):
img = mmcv.imread(self.img_path)
out_file = osp.join(tempfile.gettempdir(), 'mmcv_test.jpg')

# file_client_args and backend_args can not be both set
with pytest.raises(
ValueError,
match='"file_client_args" and "backend_args" cannot be set'):
mmcv.imwrite(
img,
out_file,
file_client_args={'backend': 'disk'},
backend_args={'backend': 'disk'})

mmcv.imwrite(img, out_file)
rewrite_img = mmcv.imread(out_file)
os.remove(out_file)
Expand All @@ -367,7 +406,13 @@ def test_imwrite(self):
with patch.object(
PetrelBackend, 'put', return_value=None) as mock_method:
ret = mmcv.imwrite(img, self.s3_path)
ret_with_args = mmcv.imwrite(
img, self.s3_path, file_client_args={'backend': 'petrel'})
assert ret
assert ret_with_args
mock_method.assert_called()

mock_method.reset_mock()

ret_with_args = mmcv.imwrite(
img, self.s3_path, backend_args={'backend': 'petrel'})
Expand Down
Loading

0 comments on commit 505f4fb

Please sign in to comment.