Merge branch 'hdf5_reader' of https://github.com/TomNicholas/Virtuali…

…Zarr into hdf5_reader

virtualizarr/readers/hdf.py

@@ -36,15 +36,11 @@ def _dataset_chunk_manifest(path: str, dataset: h5py.Dataset) -> ChunkManifest:
  key_list = [0] * (len(dataset.shape) or 1)
  key = ".".join(map(str, key_list))
  chunk_entry = ChunkEntry(
- path=path,
- offset=dsid.get_offset(),
- length=dsid.get_storage_size()
+ path=path, offset=dsid.get_offset(), length=dsid.get_storage_size()
  )
  chunk_key = ChunkKey(key)
  chunk_entries = {chunk_key: chunk_entry}
- chunk_manifest = ChunkManifest(
- entries=chunk_entries
- )
+ chunk_manifest = ChunkManifest(entries=chunk_entries)
  return chunk_manifest
  else:
  num_chunks = dsid.get_num_chunks()
@@ -60,9 +56,7 @@ def get_key(blob):
 
  def store_chunk_entry(blob):
  chunk_entries[get_key(blob)] = ChunkEntry(
- path=path,
- offset=blob.byte_offset,
- length=blob.size
+ path=path, offset=blob.byte_offset, length=blob.size
  )
 
  has_chunk_iter = callable(getattr(dsid, "chunk_iter", None))
@@ -72,9 +66,7 @@ def store_chunk_entry(blob):
  for index in range(num_chunks):
  store_chunk_entry(dsid.get_chunk_info(index))
 
- chunk_manifest = ChunkManifest(
- entries=chunk_entries
- )
+ chunk_manifest = ChunkManifest(entries=chunk_entries)
  return chunk_manifest
 
 

virtualizarr/readers/hdf_filters.py

@@ -6,9 +6,7 @@
 from numcodecs.abc import Codec
 from pydantic import BaseModel, validator
 
-_non_standard_filters = {
- "gzip": "zlib"
-}
+_non_standard_filters = {"gzip": "zlib"}
 
 
 class BloscProperties(BaseModel):
@@ -20,12 +18,15 @@ class BloscProperties(BaseModel):
  @validator("cname", pre=True)
  def get_cname_from_code(cls, v):
  blosc_compressor_codes = {
- value: key for key, value in hdf5plugin._filters.Blosc._Blosc__COMPRESSIONS.items()
+ value: key
+ for key, value in hdf5plugin._filters.Blosc._Blosc__COMPRESSIONS.items()
  }
  return blosc_compressor_codes[v]
 
 
-def _filter_to_codec(filter_id: str, filter_properties: Union[int, None, Tuple] = None) -> Codec:
+def _filter_to_codec(
+ filter_id: str, filter_properties: Union[int, None, Tuple] = None
+) -> Codec:
  id_int = None
  id_str = None
  try:
@@ -40,15 +41,20 @@ def _filter_to_codec(filter_id: str, filter_properties: Union[int, None, Tuple]
  id = id_str
  conf = {"id": id}
  if id == "zlib":
- conf["level"] = filter_properties # type: ignore[assignment]
+ conf["level"] = filter_properties  # type: ignore[assignment]
  if id_int:
  filter = hdf5plugin.get_filters(id_int)[0]
  id = filter.filter_name
  if id == "blosc" and isinstance(filter_properties, tuple):
- blosc_props = BloscProperties(**{k: v for k, v in
- zip(BloscProperties.__fields__.keys(),
- filter_properties[-4:])})
- conf = blosc_props.model_dump() # type: ignore[assignment]
+ blosc_props = BloscProperties(
+ **{
+ k: v
+ for k, v in zip(
+ BloscProperties.__fields__.keys(), filter_properties[-4:]
+ )
+ }
+ )
+ conf = blosc_props.model_dump() # type: ignore[assignment]
  conf["id"] = id
 
  codec = registry.get_codec(conf)

virtualizarr/tests/test_readers/conftest.py

@@ -138,12 +138,15 @@ def filter_encoded_netcdf4_file(tmpdir, np_uncompressed, request):
  filepath = f"{tmpdir}/{request.param}.nc"
  f = h5py.File(filepath, "w")
  if request.param == "gzip":
- f.create_dataset(name="data", data=np_uncompressed, compression="gzip", compression_opts=1)
+ f.create_dataset(
+ name="data", data=np_uncompressed, compression="gzip", compression_opts=1
+ )
  if request.param == "blosc":
- f.create_dataset(name="data", data=np_uncompressed,
- **hdf5plugin.Blosc(
- cname="lz4", clevel=9, shuffle=hdf5plugin.Blosc.SHUFFLE
- ))
+ f.create_dataset(
+ name="data",
+ data=np_uncompressed,
+ **hdf5plugin.Blosc(cname="lz4", clevel=9, shuffle=hdf5plugin.Blosc.SHUFFLE),
+ )
  return filepath
 
 
@@ -152,10 +155,7 @@ def filter_encoded_xarray_netcdf4_files(tmpdir, request):
  ds = xr.tutorial.open_dataset("air_temperature")
  encoding = {}
  if request.param == "gzip":
- encoding_config = {
- "zlib": True,
- "complevel": 1
- }
+ encoding_config = {"zlib": True, "complevel": 1}
 
  for var_name in ds.variables:
  encoding[var_name] = encoding_config

virtualizarr/tests/test_readers/test_hdf.py

@@ -105,8 +105,5 @@ def test_groups_not_implemented(self, group_netcdf4_file):
  virtual_vars_from_hdf(group_netcdf4_file)
 
  def test_drop_variables(self, multiple_datasets_netcdf4_file):
- variables = virtual_vars_from_hdf(
- multiple_datasets_netcdf4_file,
- ["data2"]
- )
+ variables = virtual_vars_from_hdf(multiple_datasets_netcdf4_file, ["data2"])
  assert "data2" not in variables.keys()

virtualizarr/tests/test_readers/test_hdf_filters.py

@@ -36,7 +36,7 @@ def test_numcodec_decoding(self, np_uncompressed, filter_encoded_netcdf4_file):
  ds = f["data"]
  chunk_info = ds.id.get_chunk_info(0)
  codecs = codecs_from_dataset(ds)
- with open(filter_encoded_netcdf4_file, 'rb') as file:
+ with open(filter_encoded_netcdf4_file, "rb") as file:
  file.seek(chunk_info.byte_offset)
  bytes_read = file.read(chunk_info.size)
  decoded = codecs[0].decode(bytes_read)

virtualizarr/tests/test_readers/test_hdf_integration.py

@@ -7,11 +7,9 @@
 
 
 class TestIntegration:
- def test_filters_end_to_end(self, tmpdir,
- filter_encoded_xarray_netcdf4_files):
+ def test_filters_end_to_end(self, tmpdir, filter_encoded_xarray_netcdf4_files):
  virtual_ds = virtualizarr.open_virtual_dataset(
- filter_encoded_xarray_netcdf4_files,
- filetype=FileType("netcdf4")
+ filter_encoded_xarray_netcdf4_files, filetype=FileType("netcdf4")
  )
  kerchunk_file = f"{tmpdir}/kerchunk.json"
  virtual_ds.virtualize.to_kerchunk(kerchunk_file, format="json")

virtualizarr/xarray.py

@@ -20,8 +20,8 @@
  _automatically_determine_filetype,
 )
 from virtualizarr.manifests import ChunkManifest, ManifestArray
-from virtualizarr.utils import _fsspec_openfile_from_filepath
 from virtualizarr.readers.hdf import attrs_from_root_group, virtual_vars_from_hdf
+from virtualizarr.utils import _fsspec_openfile_from_filepath
 from virtualizarr.zarr import (
  attrs_from_zarr_group_json,
  dataset_to_zarr,
@@ -109,8 +109,7 @@ def open_virtual_dataset(
  if filetype.name.lower() == "netcdf4":
  print("wat")
  virtual_vars = virtual_vars_from_hdf(
- path=filepath,
- drop_variables=drop_variables
+ path=filepath, drop_variables=drop_variables
  )
  ds_attrs = attrs_from_root_group(path=filepath)
  if filetype == "zarr_v3":