Introduce a base store Class

Unconditionally close store in tests.

All the tested stores should now have a `close()` method we can call and
will be no-op if the stores do not need closing.

Turn UserWarnings into errors

And turn then back into only warnings into relevant tests.
This ensure that we are not using deprecated functionalities, except
when testing for it.

initially based on 318eddcd, and later 1249f35 and 0f89a96

docs/tutorial.rst

@@ -176,7 +176,7 @@ print some diagnostics, e.g.::
     Read-only          : False
     Compressor         : Blosc(cname='zstd', clevel=3, shuffle=BITSHUFFLE,
                        : blocksize=0)
-    Store type         : builtins.dict
+    Store type         : zarr.storage.KVStore
     No. bytes          : 400000000 (381.5M)
     No. bytes stored   : 3379344 (3.2M)
     Storage ratio      : 118.4
@@ -268,7 +268,7 @@ Here is an example using a delta filter with the Blosc compressor::
     Read-only          : False
     Filter [0]         : Delta(dtype='<i4')
     Compressor         : Blosc(cname='zstd', clevel=1, shuffle=SHUFFLE, blocksize=0)
-    Store type         : builtins.dict
+    Store type         : zarr.storage.KVStore
     No. bytes          : 400000000 (381.5M)
     No. bytes stored   : 1290562 (1.2M)
     Storage ratio      : 309.9
@@ -795,8 +795,10 @@ Here is an example using S3Map to read an array created previously::
     Order              : C
     Read-only          : False
     Compressor         : Blosc(cname='lz4', clevel=5, shuffle=SHUFFLE, blocksize=0)
-    Store type         : fsspec.mapping.FSMap
+    Store type         : zarr.storage.KVStore
     No. bytes          : 21
+    No. bytes stored   : 382
+    Storage ratio      : 0.1
     Chunks initialized : 3/3
     >>> z[:]
     array([b'H', b'e', b'l', b'l', b'o', b' ', b'f', b'r', b'o', b'm', b' ',
@@ -1262,7 +1264,7 @@ ratios, depending on the correlation structure within the data. E.g.::
     Order              : C
     Read-only          : False
     Compressor         : Blosc(cname='lz4', clevel=5, shuffle=SHUFFLE, blocksize=0)
-    Store type         : builtins.dict
+    Store type         : zarr.storage.KVStore
     No. bytes          : 400000000 (381.5M)
     No. bytes stored   : 6696010 (6.4M)
     Storage ratio      : 59.7
@@ -1276,7 +1278,7 @@ ratios, depending on the correlation structure within the data. E.g.::
     Order              : F
     Read-only          : False
     Compressor         : Blosc(cname='lz4', clevel=5, shuffle=SHUFFLE, blocksize=0)
-    Store type         : builtins.dict
+    Store type         : zarr.storage.KVStore
     No. bytes          : 400000000 (381.5M)
     No. bytes stored   : 4684636 (4.5M)
     Storage ratio      : 85.4

mypy.ini

@@ -1,4 +1,4 @@
 [mypy]
-python_version = 3.6
+python_version = 3.8
 ignore_missing_imports = True
 follow_imports = silent

pytest.ini

@@ -3,4 +3,6 @@ doctest_optionflags = NORMALIZE_WHITESPACE ELLIPSIS IGNORE_EXCEPTION_DETAIL
 addopts = --durations=10
 filterwarnings =
     error::DeprecationWarning:zarr.*
+    error::UserWarning:zarr.*
     ignore:PY_SSIZE_T_CLEAN will be required.*:DeprecationWarning
+    ignore:The loop argument is deprecated since Python 3.8.*:DeprecationWarning

zarr/convenience.py

@@ -12,17 +12,21 @@
 from zarr.hierarchy import group as _create_group
 from zarr.hierarchy import open_group
 from zarr.meta import json_dumps, json_loads
-from zarr.storage import contains_array, contains_group
+from zarr.storage import contains_array, contains_group, Store
 from zarr.util import TreeViewer, buffer_size, normalize_storage_path
 
+from typing import Union
+
+StoreLike = Union[Store, str, None]
+
 
 # noinspection PyShadowingBuiltins
-def open(store=None, mode='a', **kwargs):
+def open(store: StoreLike = None, mode: str = "a", **kwargs):
     """Convenience function to open a group or array using file-mode-like semantics.
 
     Parameters
     ----------
-    store : MutableMapping or string, optional
+    store : Store or string, optional
         Store or path to directory in file system or name of zip file.
     mode : {'r', 'r+', 'a', 'w', 'w-'}, optional
         Persistence mode: 'r' means read only (must exist); 'r+' means
@@ -75,32 +79,33 @@ def open(store=None, mode='a', **kwargs):
     clobber = mode == 'w'
     # we pass storage options explicitly, since normalize_store_arg might construct
     # a store if the input is a fsspec-compatible URL
-    store = normalize_store_arg(store, clobber=clobber,
-                                storage_options=kwargs.pop("storage_options", {}))
+    _store: Store = normalize_store_arg(
+        store, clobber=clobber, storage_options=kwargs.pop("storage_options", {})
+    )
     path = normalize_storage_path(path)
 
     if mode in {'w', 'w-', 'x'}:
         if 'shape' in kwargs:
-            return open_array(store, mode=mode, **kwargs)
+            return open_array(_store, mode=mode, **kwargs)
         else:
-            return open_group(store, mode=mode, **kwargs)
+            return open_group(_store, mode=mode, **kwargs)
 
     elif mode == "a":
-        if "shape" in kwargs or contains_array(store, path):
-            return open_array(store, mode=mode, **kwargs)
+        if "shape" in kwargs or contains_array(_store, path):
+            return open_array(_store, mode=mode, **kwargs)
         else:
-            return open_group(store, mode=mode, **kwargs)
+            return open_group(_store, mode=mode, **kwargs)
 
     else:
-        if contains_array(store, path):
-            return open_array(store, mode=mode, **kwargs)
-        elif contains_group(store, path):
-            return open_group(store, mode=mode, **kwargs)
+        if contains_array(_store, path):
+            return open_array(_store, mode=mode, **kwargs)
+        elif contains_group(_store, path):
+            return open_group(_store, mode=mode, **kwargs)
         else:
             raise PathNotFoundError(path)
 
 
-def save_array(store, arr, **kwargs):
+def save_array(store: StoreLike, arr, **kwargs):
     """Convenience function to save a NumPy array to the local file system, following a
     similar API to the NumPy save() function.
 
@@ -132,16 +137,16 @@ def save_array(store, arr, **kwargs):
 
     """
     may_need_closing = isinstance(store, str)
-    store = normalize_store_arg(store, clobber=True)
+    _store: Store = normalize_store_arg(store, clobber=True)
     try:
-        _create_array(arr, store=store, overwrite=True, **kwargs)
+        _create_array(arr, store=_store, overwrite=True, **kwargs)
     finally:
-        if may_need_closing and hasattr(store, 'close'):
+        if may_need_closing:
             # needed to ensure zip file records are written
-            store.close()
+            _store.close()
 
 
-def save_group(store, *args, **kwargs):
+def save_group(store: StoreLike, *args, **kwargs):
     """Convenience function to save several NumPy arrays to the local file system, following a
     similar API to the NumPy savez()/savez_compressed() functions.
 
@@ -203,21 +208,21 @@ def save_group(store, *args, **kwargs):
         raise ValueError('at least one array must be provided')
     # handle polymorphic store arg
     may_need_closing = isinstance(store, str)
-    store = normalize_store_arg(store, clobber=True)
+    _store: Store = normalize_store_arg(store, clobber=True)
     try:
-        grp = _create_group(store, overwrite=True)
+        grp = _create_group(_store, overwrite=True)
         for i, arr in enumerate(args):
             k = 'arr_{}'.format(i)
             grp.create_dataset(k, data=arr, overwrite=True)
         for k, arr in kwargs.items():
             grp.create_dataset(k, data=arr, overwrite=True)
     finally:
-        if may_need_closing and hasattr(store, 'close'):
+        if may_need_closing:
             # needed to ensure zip file records are written
-            store.close()
+            _store.close()
 
 
-def save(store, *args, **kwargs):
+def save(store: StoreLike, *args, **kwargs):
     """Convenience function to save an array or group of arrays to the local file system.
 
     Parameters
@@ -327,7 +332,7 @@ def __repr__(self):
         return r
 
 
-def load(store):
+def load(store: StoreLike):
     """Load data from an array or group into memory.
 
     Parameters
@@ -353,11 +358,11 @@ def load(store):
 
     """
     # handle polymorphic store arg
-    store = normalize_store_arg(store)
-    if contains_array(store, path=None):
-        return Array(store=store, path=None)[...]
-    elif contains_group(store, path=None):
-        grp = Group(store=store, path=None)
+    _store = normalize_store_arg(store)
+    if contains_array(_store, path=None):
+        return Array(store=_store, path=None)[...]
+    elif contains_group(_store, path=None):
+        grp = Group(store=_store, path=None)
         return LazyLoader(grp)
 
 
@@ -1073,7 +1078,7 @@ def copy_all(source, dest, shallow=False, without_attrs=False, log=None,
     return n_copied, n_skipped, n_bytes_copied
 
 
-def consolidate_metadata(store, metadata_key='.zmetadata'):
+def consolidate_metadata(store: Store, metadata_key=".zmetadata"):
     """
     Consolidate all metadata for groups and arrays within the given store
     into a single resource and put it under the given key.
@@ -1124,7 +1129,7 @@ def is_zarr_key(key):
     return open_consolidated(store, metadata_key=metadata_key)
 
 
-def open_consolidated(store, metadata_key='.zmetadata', mode='r+', **kwargs):
+def open_consolidated(store: Store, metadata_key=".zmetadata", mode="r+", **kwargs):
     """Open group using metadata previously consolidated into a single key.
 
     This is an optimised method for opening a Zarr group, where instead of

zarr/core.py

@@ -9,6 +9,8 @@
 import numpy as np
 from numcodecs.compat import ensure_bytes, ensure_ndarray
 
+from collections.abc import MutableMapping
+
 from zarr.attrs import Attributes
 from zarr.codecs import AsType, get_codec
 from zarr.errors import ArrayNotFoundError, ReadOnlyError, ArrayIndexError
@@ -29,7 +31,7 @@
     pop_fields,
 )
 from zarr.meta import decode_array_metadata, encode_array_metadata
-from zarr.storage import array_meta_key, attrs_key, getsize, listdir
+from zarr.storage import array_meta_key, attrs_key, getsize, listdir, Store
 from zarr.util import (
     InfoReporter,
     check_array_shape,
@@ -129,7 +131,7 @@ class Array:
 
     def __init__(
         self,
-        store,
+        store: Store,
         path=None,
         read_only=False,
         chunk_store=None,
@@ -141,6 +143,9 @@ def __init__(
         # N.B., expect at this point store is fully initialized with all
         # configuration metadata fully specified and normalized
 
+        store = Store._ensure_store(store)
+        chunk_store = Store._ensure_store(chunk_store)
+
         self._store = store
         self._chunk_store = chunk_store
         self._path = normalize_storage_path(path)
@@ -1955,7 +1960,7 @@ def _encode_chunk(self, chunk):
             cdata = chunk
 
         # ensure in-memory data is immutable and easy to compare
-        if isinstance(self.chunk_store, dict):
+        if isinstance(self.chunk_store, MutableMapping):
             cdata = ensure_bytes(cdata)
 
         return cdata
@@ -1988,10 +1993,10 @@ def info(self):
         Order              : C
         Read-only          : False
         Compressor         : Blosc(cname='lz4', clevel=5, shuffle=SHUFFLE, blocksize=0)
-        Store type         : builtins.dict
+        Store type         : zarr.storage.KVStore
         No. bytes          : 4000000 (3.8M)
-        No. bytes stored   : ...
-        Storage ratio      : ...
+        No. bytes stored   : 320
+        Storage ratio      : 12500.0
         Chunks initialized : 0/10
 
         """

zarr/creation.py

@@ -2,6 +2,7 @@
 
 import numpy as np
 from numcodecs.registry import codec_registry
+from collections.abc import MutableMapping
 
 from zarr.core import Array
 from zarr.errors import (
@@ -10,9 +11,18 @@
     ContainsGroupError,
 )
 from zarr.n5 import N5Store
-from zarr.storage import (DirectoryStore, ZipStore, contains_array,
-                          contains_group, default_compressor, init_array,
-                          normalize_storage_path, FSStore)
+from zarr.storage import (
+    DirectoryStore,
+    ZipStore,
+    KVStore,
+    contains_array,
+    contains_group,
+    default_compressor,
+    init_array,
+    normalize_storage_path,
+    FSStore,
+    Store,
+)
 
 
 def create(shape, chunks=True, dtype=None, compressor='default',
@@ -129,9 +139,9 @@ def create(shape, chunks=True, dtype=None, compressor='default',
     return z
 
 
-def normalize_store_arg(store, clobber=False, storage_options=None, mode='w'):
+def normalize_store_arg(store, clobber=False, storage_options=None, mode="w") -> Store:
     if store is None:
-        return dict()
+        return Store._ensure_store(dict())
     elif isinstance(store, str):
         mode = mode if clobber else "r"
         if "://" in store or "::" in store:
@@ -145,6 +155,8 @@ def normalize_store_arg(store, clobber=False, storage_options=None, mode='w'):
         else:
             return DirectoryStore(store)
     else:
+        if not isinstance(store, Store) and isinstance(store, MutableMapping):
+            store = KVStore(store)
         return store