diff --git a/zarr/_storage/store.py b/zarr/_storage/store.py index a779a4e26a..6f5bf78e28 100644 --- a/zarr/_storage/store.py +++ b/zarr/_storage/store.py @@ -1,6 +1,7 @@ from collections.abc import MutableMapping from typing import Any, List, Optional, Union +from zarr.meta import Metadata2 from zarr.util import normalize_storage_path # v2 store keys @@ -32,6 +33,8 @@ class BaseStore(MutableMapping): _writeable = True _erasable = True _listable = True + _store_version = 2 + _metadata_class = Metadata2 def is_readable(self): return self._readable @@ -114,6 +117,7 @@ class Store(BaseStore): .. added: 2.11.0 """ + def listdir(self, path: str = "") -> List[str]: path = normalize_storage_path(path) return _listdir_from_keys(self, path) diff --git a/zarr/attrs.py b/zarr/attrs.py index ec01dbe04f..eff1237db1 100644 --- a/zarr/attrs.py +++ b/zarr/attrs.py @@ -1,6 +1,5 @@ from collections.abc import MutableMapping -from zarr.meta import parse_metadata from zarr._storage.store import Store from zarr.util import json_dumps @@ -40,7 +39,7 @@ def _get_nosync(self): except KeyError: d = dict() else: - d = parse_metadata(data) + d = self.store._metadata_class.parse_metadata(data) return d def asdict(self): diff --git a/zarr/core.py b/zarr/core.py index 56b22ead8d..d366139423 100644 --- a/zarr/core.py +++ b/zarr/core.py @@ -31,7 +31,6 @@ is_scalar, pop_fields, ) -from zarr.meta import decode_array_metadata, encode_array_metadata from zarr.storage import array_meta_key, attrs_key, getsize, listdir, BaseStore from zarr.util import ( all_equal, @@ -210,7 +209,7 @@ def _load_metadata_nosync(self): else: # decode and store metadata as instance members - meta = decode_array_metadata(meta_bytes) + meta = self._store._metadata_class.decode_array_metadata(meta_bytes) self._meta = meta self._shape = meta['shape'] self._chunks = meta['chunks'] @@ -267,7 +266,7 @@ def _flush_metadata_nosync(self): compressor=compressor_config, fill_value=self._fill_value, order=self._order, filters=filters_config) mkey = self._key_prefix + array_meta_key - self._store[mkey] = encode_array_metadata(meta) + self._store[mkey] = self._store._metadata_class.encode_array_metadata(meta) @property def store(self): diff --git a/zarr/hierarchy.py b/zarr/hierarchy.py index 402b8dd976..763a5f1631 100644 --- a/zarr/hierarchy.py +++ b/zarr/hierarchy.py @@ -14,7 +14,6 @@ GroupNotFoundError, ReadOnlyError, ) -from zarr.meta import decode_group_metadata from zarr.storage import ( BaseStore, MemoryStore, @@ -134,8 +133,7 @@ def __init__(self, store, path=None, read_only=False, chunk_store=None, except KeyError: raise GroupNotFoundError(path) else: - meta = decode_group_metadata(meta_bytes) - self._meta = meta + self._meta = self._store._metadata_class.decode_group_metadata(meta_bytes) # setup attributes akey = self._key_prefix + attrs_key diff --git a/zarr/meta.py b/zarr/meta.py index 8a7d760c0f..c292b09a14 100644 --- a/zarr/meta.py +++ b/zarr/meta.py @@ -11,220 +11,231 @@ ZARR_FORMAT = 2 -def parse_metadata(s: Union[MappingType, str]) -> MappingType[str, Any]: +class Metadata2: + ZARR_FORMAT = ZARR_FORMAT - # Here we allow that a store may return an already-parsed metadata object, - # or a string of JSON that we will parse here. We allow for an already-parsed - # object to accommodate a consolidated metadata store, where all the metadata for - # all groups and arrays will already have been parsed from JSON. + @classmethod + def parse_metadata(cls, s: Union[MappingType, str]) -> MappingType[str, Any]: - if isinstance(s, Mapping): - # assume metadata has already been parsed into a mapping object - meta = s + # Here we allow that a store may return an already-parsed metadata object, + # or a string of JSON that we will parse here. We allow for an already-parsed + # object to accommodate a consolidated metadata store, where all the metadata for + # all groups and arrays will already have been parsed from JSON. - else: - # assume metadata needs to be parsed as JSON - meta = json_loads(s) + if isinstance(s, Mapping): + # assume metadata has already been parsed into a mapping object + meta = s - return meta + else: + # assume metadata needs to be parsed as JSON + meta = json_loads(s) + return meta -def decode_array_metadata(s: Union[MappingType, str]) -> MappingType[str, Any]: - meta = parse_metadata(s) + @classmethod + def decode_array_metadata(cls, s: Union[MappingType, str]) -> MappingType[str, Any]: + meta = cls.parse_metadata(s) - # check metadata format - zarr_format = meta.get('zarr_format', None) - if zarr_format != ZARR_FORMAT: - raise MetadataError('unsupported zarr format: %s' % zarr_format) + # check metadata format + zarr_format = meta.get("zarr_format", None) + if zarr_format != cls.ZARR_FORMAT: + raise MetadataError("unsupported zarr format: %s" % zarr_format) - # extract array metadata fields - try: - dtype = decode_dtype(meta['dtype']) + # extract array metadata fields + try: + dtype = cls.decode_dtype(meta["dtype"]) + if dtype.hasobject: + import numcodecs + object_codec = numcodecs.get_codec(meta['filters'][0]) + else: + object_codec = None + + dimension_separator = meta.get("dimension_separator", None) + fill_value = cls.decode_fill_value(meta['fill_value'], dtype, object_codec) + meta = dict( + zarr_format=meta["zarr_format"], + shape=tuple(meta["shape"]), + chunks=tuple(meta["chunks"]), + dtype=dtype, + compressor=meta["compressor"], + fill_value=fill_value, + order=meta["order"], + filters=meta["filters"], + ) + if dimension_separator: + meta['dimension_separator'] = dimension_separator + except Exception as e: + raise MetadataError("error decoding metadata") from e + else: + return meta + @classmethod + def encode_array_metadata(cls, meta: MappingType[str, Any]) -> bytes: + dtype = meta["dtype"] + sdshape = () + if dtype.subdtype is not None: + dtype, sdshape = dtype.subdtype + + dimension_separator = meta.get("dimension_separator") if dtype.hasobject: import numcodecs object_codec = numcodecs.get_codec(meta['filters'][0]) else: object_codec = None - dimension_separator = meta.get('dimension_separator', None) - fill_value = decode_fill_value(meta['fill_value'], dtype, object_codec) meta = dict( - zarr_format=meta['zarr_format'], - shape=tuple(meta['shape']), - chunks=tuple(meta['chunks']), - dtype=dtype, - compressor=meta['compressor'], - fill_value=fill_value, - order=meta['order'], - filters=meta['filters'], + zarr_format=cls.ZARR_FORMAT, + shape=meta["shape"] + sdshape, + chunks=meta["chunks"], + dtype=cls.encode_dtype(dtype), + compressor=meta["compressor"], + fill_value=cls.encode_fill_value(meta["fill_value"], dtype, object_codec), + order=meta["order"], + filters=meta["filters"], ) if dimension_separator: meta['dimension_separator'] = dimension_separator - except Exception as e: - raise MetadataError('error decoding metadata: %s' % e) - else: - return meta + if dimension_separator: + meta["dimension_separator"] = dimension_separator + return json_dumps(meta) -def encode_array_metadata(meta: MappingType[str, Any]) -> bytes: - dtype = meta['dtype'] - sdshape = () - if dtype.subdtype is not None: - dtype, sdshape = dtype.subdtype - - dimension_separator = meta.get('dimension_separator') - if dtype.hasobject: - import numcodecs - object_codec = numcodecs.get_codec(meta['filters'][0]) - else: - object_codec = None - meta = dict( - zarr_format=ZARR_FORMAT, - shape=meta['shape'] + sdshape, - chunks=meta['chunks'], - dtype=encode_dtype(dtype), - compressor=meta['compressor'], - fill_value=encode_fill_value(meta['fill_value'], dtype, object_codec), - order=meta['order'], - filters=meta['filters'], - ) - - if dimension_separator: - meta['dimension_separator'] = dimension_separator - - return json_dumps(meta) - - -def encode_dtype(d: np.dtype): - if d.fields is None: - return d.str - else: - return d.descr - - -def _decode_dtype_descr(d) -> List[Any]: - # need to convert list of lists to list of tuples - if isinstance(d, list): - # recurse to handle nested structures - d = [(k[0], _decode_dtype_descr(k[1])) + tuple(k[2:]) for k in d] - return d - - -def decode_dtype(d) -> np.dtype: - d = _decode_dtype_descr(d) - return np.dtype(d) - - -def decode_group_metadata(s: Union[MappingType, str]) -> MappingType[str, Any]: - meta = parse_metadata(s) - - # check metadata format version - zarr_format = meta.get('zarr_format', None) - if zarr_format != ZARR_FORMAT: - raise MetadataError('unsupported zarr format: %s' % zarr_format) - - meta = dict(zarr_format=zarr_format) - return meta - - -# N.B., keep `meta` parameter as a placeholder for future -# noinspection PyUnusedLocal -def encode_group_metadata(meta=None) -> bytes: - meta = dict( - zarr_format=ZARR_FORMAT, - ) - return json_dumps(meta) - - -FLOAT_FILLS = { - 'NaN': np.nan, - 'Infinity': np.PINF, - '-Infinity': np.NINF -} - - -def decode_fill_value(v, dtype, object_codec=None): - # early out - if v is None: - return v - if dtype.kind == 'V' and dtype.hasobject: - if object_codec is None: - raise ValueError('missing object_codec for object array') - v = base64.standard_b64decode(v) - v = object_codec.decode(v) - v = np.array(v, dtype=dtype)[()] - return v - if dtype.kind == 'f': - if v == 'NaN': - return np.nan - elif v == 'Infinity': - return np.PINF - elif v == '-Infinity': - return np.NINF + @classmethod + def encode_dtype(cls, d: np.dtype): + if d.fields is None: + return d.str else: + return d.descr + + @classmethod + def _decode_dtype_descr(cls, d) -> List[Any]: + # need to convert list of lists to list of tuples + if isinstance(d, list): + # recurse to handle nested structures + d = [(k[0], cls._decode_dtype_descr(k[1])) + tuple(k[2:]) for k in d] + return d + + @classmethod + def decode_dtype(cls, d) -> np.dtype: + d = cls._decode_dtype_descr(d) + return np.dtype(d) + + @classmethod + def decode_group_metadata(cls, s: Union[MappingType, str]) -> MappingType[str, Any]: + meta = cls.parse_metadata(s) + + # check metadata format version + zarr_format = meta.get("zarr_format", None) + if zarr_format != cls.ZARR_FORMAT: + raise MetadataError("unsupported zarr format: %s" % zarr_format) + + meta = dict(zarr_format=zarr_format) + return meta + + # N.B., keep `meta` parameter as a placeholder for future + # noinspection PyUnusedLocal + @classmethod + def encode_group_metadata(cls, meta=None) -> bytes: + meta = dict(zarr_format=cls.ZARR_FORMAT) + return json_dumps(meta) + + @classmethod + def decode_fill_value(cls, v: Any, dtype: np.dtype, object_codec: Any = None) -> Any: + # early out + if v is None: + return v + if dtype.kind == 'V' and dtype.hasobject: + if object_codec is None: + raise ValueError('missing object_codec for object array') + v = base64.standard_b64decode(v) + v = object_codec.decode(v) + v = np.array(v, dtype=dtype)[()] + return v + if dtype.kind == "f": + if v == "NaN": + return np.nan + elif v == "Infinity": + return np.PINF + elif v == "-Infinity": + return np.NINF + else: + return np.array(v, dtype=dtype)[()] + elif dtype.kind in "c": + v = ( + cls.decode_fill_value(v[0], dtype.type().real.dtype), + cls.decode_fill_value(v[1], dtype.type().imag.dtype), + ) + v = v[0] + 1j * v[1] return np.array(v, dtype=dtype)[()] - elif dtype.kind in 'c': - v = (decode_fill_value(v[0], dtype.type().real.dtype), - decode_fill_value(v[1], dtype.type().imag.dtype)) - v = v[0] + 1j * v[1] - return np.array(v, dtype=dtype)[()] - elif dtype.kind == 'S': - # noinspection PyBroadException - try: + elif dtype.kind == "S": + # noinspection PyBroadException + try: + v = base64.standard_b64decode(v) + except Exception: + # be lenient, allow for other values that may have been used before base64 + # encoding and may work as fill values, e.g., the number 0 + pass + v = np.array(v, dtype=dtype)[()] + return v + elif dtype.kind == "V": v = base64.standard_b64decode(v) - except Exception: - # be lenient, allow for other values that may have been used before base64 - # encoding and may work as fill values, e.g., the number 0 - pass - v = np.array(v, dtype=dtype)[()] - return v - elif dtype.kind == 'V': - v = base64.standard_b64decode(v) - v = np.array(v, dtype=dtype.str).view(dtype)[()] - return v - elif dtype.kind == 'U': - # leave as-is - return v - else: - return np.array(v, dtype=dtype)[()] - - -def encode_fill_value(v: Any, dtype: np.dtype, object_codec: Any = None) -> Any: - # early out - if v is None: - return v - if dtype.kind == 'V' and dtype.hasobject: - if object_codec is None: - raise ValueError('missing object_codec for object array') - v = object_codec.encode(v) - v = str(base64.standard_b64encode(v), 'ascii') - return v - if dtype.kind == 'f': - if np.isnan(v): - return 'NaN' - elif np.isposinf(v): - return 'Infinity' - elif np.isneginf(v): - return '-Infinity' + v = np.array(v, dtype=dtype.str).view(dtype)[()] + return v + elif dtype.kind == "U": + # leave as-is + return v + else: + return np.array(v, dtype=dtype)[()] + + @classmethod + def encode_fill_value(cls, v: Any, dtype: np.dtype, object_codec: Any = None) -> Any: + # early out + if v is None: + return v + if dtype.kind == 'V' and dtype.hasobject: + if object_codec is None: + raise ValueError('missing object_codec for object array') + v = object_codec.encode(v) + v = str(base64.standard_b64encode(v), 'ascii') + return v + if dtype.kind == "f": + if np.isnan(v): + return "NaN" + elif np.isposinf(v): + return "Infinity" + elif np.isneginf(v): + return "-Infinity" + else: + return float(v) + elif dtype.kind in "ui": + return int(v) + elif dtype.kind == "b": + return bool(v) + elif dtype.kind in "c": + c = cast(np.complex128, np.dtype(complex).type()) + v = (cls.encode_fill_value(v.real, c.real.dtype, object_codec), + cls.encode_fill_value(v.imag, c.imag.dtype, object_codec)) + return v + elif dtype.kind in "SV": + v = str(base64.standard_b64encode(v), "ascii") + return v + elif dtype.kind == "U": + return v + elif dtype.kind in "mM": + return int(v.view("i8")) else: - return float(v) - elif dtype.kind in 'ui': - return int(v) - elif dtype.kind == 'b': - return bool(v) - elif dtype.kind in 'c': - c = cast(np.complex128, np.dtype(complex).type()) - v = (encode_fill_value(v.real, c.real.dtype, object_codec), - encode_fill_value(v.imag, c.imag.dtype, object_codec)) - return v - elif dtype.kind in 'SV': - v = str(base64.standard_b64encode(v), 'ascii') - return v - elif dtype.kind == 'U': - return v - elif dtype.kind in 'mM': - return int(v.view('i8')) - else: - return v + return v + + +# expose class methods for backwards compatibility +parse_metadata = Metadata2.parse_metadata +decode_array_metadata = Metadata2.decode_array_metadata +encode_array_metadata = Metadata2.encode_array_metadata +encode_dtype = Metadata2.encode_dtype +_decode_dtype_descr = Metadata2._decode_dtype_descr +decode_dtype = Metadata2.decode_dtype +decode_group_metadata = Metadata2.decode_group_metadata +encode_group_metadata = Metadata2.encode_group_metadata +decode_fill_value = Metadata2.decode_fill_value +encode_fill_value = Metadata2.encode_fill_value diff --git a/zarr/storage.py b/zarr/storage.py index 901011c9d2..7170eeaf23 100644 --- a/zarr/storage.py +++ b/zarr/storage.py @@ -231,8 +231,8 @@ def init_array( fill_value=None, order: str = "C", overwrite: bool = False, - path: Path = None, - chunk_store: StoreLike = None, + path: Optional[Path] = None, + chunk_store: Optional[StoreLike] = None, filters=None, object_codec=None, dimension_separator=None, @@ -357,7 +357,7 @@ def init_array( def _init_array_metadata( - store, + store: StoreLike, shape, chunks=None, dtype=None, @@ -366,7 +366,7 @@ def _init_array_metadata( order="C", overwrite=False, path: Optional[str] = None, - chunk_store=None, + chunk_store: Optional[StoreLike] = None, filters=None, object_codec=None, dimension_separator=None, @@ -446,7 +446,10 @@ def _init_array_metadata( order=order, filters=filters_config, dimension_separator=dimension_separator) key = _path_to_prefix(path) + array_meta_key - store[key] = encode_array_metadata(meta) + if hasattr(store, '_metadata_class'): + store[key] = store._metadata_class.encode_array_metadata(meta) # type: ignore + else: + store[key] = encode_array_metadata(meta) # backwards compatibility @@ -511,7 +514,10 @@ def _init_group_metadata( # be in future meta = dict() # type: ignore key = _path_to_prefix(path) + group_meta_key - store[key] = encode_group_metadata(meta) + if hasattr(store, '_metadata_class'): + store[key] = store._metadata_class.encode_group_metadata(meta) # type: ignore + else: + store[key] = encode_group_metadata(meta) def _dict_store_keys(d: Dict, prefix="", cls=dict): @@ -568,7 +574,7 @@ def __eq__(self, other): class MemoryStore(Store): - """Store class that uses a hierarchy of :class:`dict` objects, thus all data + """Store class that uses a hierarchy of :class:`KVStore` objects, thus all data will be held in main memory. Examples @@ -581,7 +587,7 @@ class MemoryStore(Store): <class 'zarr.storage.MemoryStore'> Note that the default class when creating an array is the built-in - :class:`dict` class, i.e.:: + :class:`KVStore` class, i.e.:: >>> z = zarr.zeros(100) >>> type(z.store) @@ -1685,7 +1691,10 @@ def migrate_1to2(store): del meta['compression_opts'] # store migrated metadata - store[array_meta_key] = encode_array_metadata(meta) + if hasattr(store, '_metadata_class'): + store[array_meta_key] = store._metadata_class.encode_array_metadata(meta) + else: + store[array_meta_key] = encode_array_metadata(meta) # migrate user attributes store[attrs_key] = store['attrs'] @@ -2099,8 +2108,8 @@ class LRUStoreCache(Store): """ - def __init__(self, store: Store, max_size: int): - self._store = Store._ensure_store(store) + def __init__(self, store: StoreLike, max_size: int): + self._store: BaseStore = BaseStore._ensure_store(store) self._max_size = max_size self._current_size = 0 self._keys_cache = None diff --git a/zarr/tests/test_storage.py b/zarr/tests/test_storage.py index 889fa80043..3438e60691 100644 --- a/zarr/tests/test_storage.py +++ b/zarr/tests/test_storage.py @@ -20,9 +20,7 @@ from zarr.codecs import BZ2, AsType, Blosc, Zlib from zarr.errors import MetadataError from zarr.hierarchy import group -from zarr.meta import (ZARR_FORMAT, decode_array_metadata, - decode_group_metadata, encode_array_metadata, - encode_group_metadata) +from zarr.meta import ZARR_FORMAT, decode_array_metadata from zarr.n5 import N5Store, N5FSStore from zarr.storage import (ABSStore, ConsolidatedMetadataStore, DBMStore, DictStore, DirectoryStore, KVStore, LMDBStore, @@ -427,7 +425,7 @@ def test_init_array(self, dimension_separator_fixture): # check metadata assert array_meta_key in store - meta = decode_array_metadata(store[array_meta_key]) + meta = store._metadata_class.decode_array_metadata(store[array_meta_key]) assert ZARR_FORMAT == meta['zarr_format'] assert (1000,) == meta['shape'] assert (100,) == meta['chunks'] @@ -460,7 +458,7 @@ def test_init_group_overwrite_chunk_store(self): def _test_init_array_overwrite(self, order): # setup store = self.create_store() - store[array_meta_key] = encode_array_metadata( + store[array_meta_key] = store._metadata_class.encode_array_metadata( dict(shape=(2000,), chunks=(200,), dtype=np.dtype('u1'), @@ -482,7 +480,7 @@ def _test_init_array_overwrite(self, order): pass else: assert array_meta_key in store - meta = decode_array_metadata(store[array_meta_key]) + meta = store._metadata_class.decode_array_metadata(store[array_meta_key]) assert ZARR_FORMAT == meta['zarr_format'] assert (1000,) == meta['shape'] assert (100,) == meta['chunks'] @@ -498,7 +496,7 @@ def test_init_array_path(self): # check metadata key = path + '/' + array_meta_key assert key in store - meta = decode_array_metadata(store[key]) + meta = store._metadata_class.decode_array_metadata(store[key]) assert ZARR_FORMAT == meta['zarr_format'] assert (1000,) == meta['shape'] assert (100,) == meta['chunks'] @@ -519,8 +517,8 @@ def _test_init_array_overwrite_path(self, order): fill_value=0, order=order, filters=None) - store[array_meta_key] = encode_array_metadata(meta) - store[path + '/' + array_meta_key] = encode_array_metadata(meta) + store[array_meta_key] = store._metadata_class.encode_array_metadata(meta) + store[path + '/' + array_meta_key] = store._metadata_class.encode_array_metadata(meta) # don't overwrite with pytest.raises(ValueError): @@ -537,7 +535,7 @@ def _test_init_array_overwrite_path(self, order): assert array_meta_key not in store assert (path + '/' + array_meta_key) in store # should have been overwritten - meta = decode_array_metadata(store[path + '/' + array_meta_key]) + meta = store._metadata_class.decode_array_metadata(store[path + '/' + array_meta_key]) assert ZARR_FORMAT == meta['zarr_format'] assert (1000,) == meta['shape'] assert (100,) == meta['chunks'] @@ -549,7 +547,7 @@ def test_init_array_overwrite_group(self): # setup path = 'foo/bar' store = self.create_store() - store[path + '/' + group_meta_key] = encode_group_metadata() + store[path + '/' + group_meta_key] = store._metadata_class.encode_group_metadata() # don't overwrite with pytest.raises(ValueError): @@ -564,7 +562,7 @@ def test_init_array_overwrite_group(self): else: assert (path + '/' + group_meta_key) not in store assert (path + '/' + array_meta_key) in store - meta = decode_array_metadata(store[path + '/' + array_meta_key]) + meta = store._metadata_class.decode_array_metadata(store[path + '/' + array_meta_key]) assert ZARR_FORMAT == meta['zarr_format'] assert (1000,) == meta['shape'] assert (100,) == meta['chunks'] @@ -576,7 +574,7 @@ def _test_init_array_overwrite_chunk_store(self, order): # setup store = self.create_store() chunk_store = self.create_store() - store[array_meta_key] = encode_array_metadata( + store[array_meta_key] = store._metadata_class.encode_array_metadata( dict(shape=(2000,), chunks=(200,), dtype=np.dtype('u1'), @@ -600,7 +598,7 @@ def _test_init_array_overwrite_chunk_store(self, order): pass else: assert array_meta_key in store - meta = decode_array_metadata(store[array_meta_key]) + meta = store._metadata_class.decode_array_metadata(store[array_meta_key]) assert ZARR_FORMAT == meta['zarr_format'] assert (1000,) == meta['shape'] assert (100,) == meta['chunks'] @@ -614,7 +612,7 @@ def _test_init_array_overwrite_chunk_store(self, order): def test_init_array_compat(self): store = self.create_store() init_array(store, shape=1000, chunks=100, compressor='none') - meta = decode_array_metadata(store[array_meta_key]) + meta = store._metadata_class.decode_array_metadata(store[array_meta_key]) assert meta['compressor'] is None store.close() @@ -625,7 +623,7 @@ def test_init_group(self): # check metadata assert group_meta_key in store - meta = decode_group_metadata(store[group_meta_key]) + meta = store._metadata_class.decode_group_metadata(store[group_meta_key]) assert ZARR_FORMAT == meta['zarr_format'] store.close() @@ -633,7 +631,7 @@ def test_init_group(self): def _test_init_group_overwrite(self, order): # setup store = self.create_store() - store[array_meta_key] = encode_array_metadata( + store[array_meta_key] = store._metadata_class.encode_array_metadata( dict(shape=(2000,), chunks=(200,), dtype=np.dtype('u1'), @@ -655,7 +653,7 @@ def _test_init_group_overwrite(self, order): else: assert array_meta_key not in store assert group_meta_key in store - meta = decode_group_metadata(store[group_meta_key]) + meta = store._metadata_class.decode_group_metadata(store[group_meta_key]) assert ZARR_FORMAT == meta['zarr_format'] # don't overwrite group @@ -675,8 +673,8 @@ def _test_init_group_overwrite_path(self, order): fill_value=0, order=order, filters=None) - store[array_meta_key] = encode_array_metadata(meta) - store[path + '/' + array_meta_key] = encode_array_metadata(meta) + store[array_meta_key] = store._metadata_class.encode_array_metadata(meta) + store[path + '/' + array_meta_key] = store._metadata_class.encode_array_metadata(meta) # don't overwrite with pytest.raises(ValueError): @@ -693,7 +691,7 @@ def _test_init_group_overwrite_path(self, order): assert (path + '/' + array_meta_key) not in store assert (path + '/' + group_meta_key) in store # should have been overwritten - meta = decode_group_metadata(store[path + '/' + group_meta_key]) + meta = store._metadata_class.decode_group_metadata(store[path + '/' + group_meta_key]) assert ZARR_FORMAT == meta['zarr_format'] store.close() @@ -702,7 +700,7 @@ def _test_init_group_overwrite_chunk_store(self, order): # setup store = self.create_store() chunk_store = self.create_store() - store[array_meta_key] = encode_array_metadata( + store[array_meta_key] = store._metadata_class.encode_array_metadata( dict(shape=(2000,), chunks=(200,), dtype=np.dtype('u1'), @@ -726,7 +724,7 @@ def _test_init_group_overwrite_chunk_store(self, order): else: assert array_meta_key not in store assert group_meta_key in store - meta = decode_group_metadata(store[group_meta_key]) + meta = store._metadata_class.decode_group_metadata(store[group_meta_key]) assert ZARR_FORMAT == meta['zarr_format'] assert 'foo' not in chunk_store assert 'baz' not in chunk_store @@ -941,7 +939,7 @@ def test_init_array(self): # check metadata assert array_meta_key in store - meta = decode_array_metadata(store[array_meta_key]) + meta = store._metadata_class.decode_array_metadata(store[array_meta_key]) assert ZARR_FORMAT == meta['zarr_format'] assert (1000,) == meta['shape'] assert (100,) == meta['chunks'] @@ -1191,7 +1189,7 @@ def test_init_array(self): # check metadata assert array_meta_key in store - meta = decode_array_metadata(store[array_meta_key]) + meta = store._metadata_class.decode_array_metadata(store[array_meta_key]) assert ZARR_FORMAT == meta['zarr_format'] assert (1000,) == meta['shape'] assert (100,) == meta['chunks'] @@ -1267,7 +1265,7 @@ def test_init_array(self): # check metadata assert array_meta_key in store - meta = decode_array_metadata(store[array_meta_key]) + meta = store._metadata_class.decode_array_metadata(store[array_meta_key]) assert ZARR_FORMAT == meta['zarr_format'] assert (1000,) == meta['shape'] assert (100,) == meta['chunks'] @@ -1287,7 +1285,7 @@ def test_init_array_path(self): # check metadata key = path + '/' + array_meta_key assert key in store - meta = decode_array_metadata(store[key]) + meta = store._metadata_class.decode_array_metadata(store[key]) assert ZARR_FORMAT == meta['zarr_format'] assert (1000,) == meta['shape'] assert (100,) == meta['chunks'] @@ -1301,7 +1299,7 @@ def test_init_array_path(self): def test_init_array_compat(self): store = self.create_store() init_array(store, shape=1000, chunks=100, compressor='none') - meta = decode_array_metadata(store[array_meta_key]) + meta = store._metadata_class.decode_array_metadata(store[array_meta_key]) # N5Store wraps the actual compressor compressor_config = meta['compressor']['compressor_config'] assert compressor_config is None @@ -1332,7 +1330,7 @@ def test_init_group(self): assert group_meta_key in store assert group_meta_key in store.listdir() assert group_meta_key in store.listdir('') - meta = decode_group_metadata(store[group_meta_key]) + meta = store._metadata_class.decode_group_metadata(store[group_meta_key]) assert ZARR_FORMAT == meta['zarr_format'] def test_filters(self): @@ -1387,7 +1385,7 @@ def test_init_array(self): # check metadata assert array_meta_key in store - meta = decode_array_metadata(store[array_meta_key]) + meta = store._metadata_class.decode_array_metadata(store[array_meta_key]) assert ZARR_FORMAT == meta['zarr_format'] assert (1000,) == meta['shape'] assert (100,) == meta['chunks'] @@ -1407,7 +1405,7 @@ def test_init_array_path(self): # check metadata key = path + '/' + array_meta_key assert key in store - meta = decode_array_metadata(store[key]) + meta = store._metadata_class.decode_array_metadata(store[key]) assert ZARR_FORMAT == meta['zarr_format'] assert (1000,) == meta['shape'] assert (100,) == meta['chunks'] @@ -1421,7 +1419,7 @@ def test_init_array_path(self): def test_init_array_compat(self): store = self.create_store() init_array(store, shape=1000, chunks=100, compressor='none') - meta = decode_array_metadata(store[array_meta_key]) + meta = store._metadata_class.decode_array_metadata(store[array_meta_key]) # N5Store wraps the actual compressor compressor_config = meta['compressor']['compressor_config'] assert compressor_config is None @@ -1929,14 +1927,15 @@ def test_getsize(): assert -1 == getsize(store) -def test_migrate_1to2(): +@pytest.mark.parametrize('dict_store', [False, True]) +def test_migrate_1to2(dict_store): from zarr import meta_v1 # N.B., version 1 did not support hierarchies, so we only have to be # concerned about migrating a single array at the root of the store # setup - store = KVStore(dict()) + store = dict() if dict_store else KVStore(dict()) meta = dict( shape=(100,), chunks=(10,), @@ -1974,7 +1973,7 @@ def test_migrate_1to2(): assert meta_migrated['compressor'] == Zlib(1).get_config() # check dict compression_opts - store = KVStore(dict()) + store = dict() if dict_store else KVStore(dict()) meta['compression'] = 'blosc' meta['compression_opts'] = dict(cname='lz4', clevel=5, shuffle=1) meta_json = meta_v1.encode_metadata(meta) @@ -1988,7 +1987,7 @@ def test_migrate_1to2(): Blosc(cname='lz4', clevel=5, shuffle=1).get_config()) # check 'none' compression is migrated to None (null in JSON) - store = KVStore(dict()) + store = dict() if dict_store else KVStore(dict()) meta['compression'] = 'none' meta_json = meta_v1.encode_metadata(meta) store['meta'] = meta_json