Zarr reader

conftest.py

@@ -35,6 +35,24 @@ def netcdf4_file(tmpdir):
     return filepath
 
 
+@pytest.fixture()
+def zarr_v2_store(tmpdir):
+    # Set up example xarray dataset
+    ds = xr.tutorial.open_dataset("air_temperature", chunks={})
+    # grabbing a piece and making sure there are multiple chunks present (2): Frozen({'time': (5, 5), 'lat': (9,), 'lon': (18,)})
+    chunked_subset = ds.isel(
+        time=slice(0, 10), lat=slice(0, 9), lon=slice(0, 18)
+    ).chunk({"time": 5})
+
+    # Save it to disk as netCDF (in temporary directory)
+    filepath = f"{tmpdir}/air.zarr"
+
+    chunked_subset.to_zarr(filepath, zarr_format=2)
+    ds.close()
+
+    return filepath
+
+
 @pytest.fixture
 def netcdf4_virtual_dataset(netcdf4_file):
     from virtualizarr import open_virtual_dataset

virtualizarr/backend.py

@@ -17,14 +17,16 @@
     KerchunkVirtualBackend,
     NetCDF3VirtualBackend,
     TIFFVirtualBackend,
-    ZarrV3VirtualBackend,
+    ZarrV3ChunkManifestVirtualBackend,  # If this is kept, we should incorporate it into ZarrVirtualBackend
+    ZarrVirtualBackend,
 )
 from virtualizarr.utils import _FsspecFSFromFilepath, check_for_collisions
 
 # TODO add entrypoint to allow external libraries to add to this mapping
 VIRTUAL_BACKENDS = {
     "kerchunk": KerchunkVirtualBackend,
-    "zarr_v3": ZarrV3VirtualBackend,
+    "zarr_v3": ZarrV3ChunkManifestVirtualBackend,
+    "zarr": ZarrVirtualBackend,
     "dmrpp": DMRPPVirtualBackend,
     # all the below call one of the kerchunk backends internally (https://fsspec.github.io/kerchunk/reference.html#file-format-backends)
     "netcdf3": NetCDF3VirtualBackend,
@@ -71,8 +73,7 @@ def automatically_determine_filetype(
 
     # TODO how do we handle kerchunk json / parquet here?
     if Path(filepath).suffix == ".zarr":
-        # TODO we could imagine opening an existing zarr store, concatenating it, and writing a new virtual one...
-        raise NotImplementedError()
+        return FileType.zarr
 
     # Read magic bytes from local or remote file
     fpath = _FsspecFSFromFilepath(

virtualizarr/readers/__init__.py

@@ -4,7 +4,10 @@
 from virtualizarr.readers.kerchunk import KerchunkVirtualBackend
 from virtualizarr.readers.netcdf3 import NetCDF3VirtualBackend
 from virtualizarr.readers.tiff import TIFFVirtualBackend
-from virtualizarr.readers.zarr_v3 import ZarrV3VirtualBackend
+from virtualizarr.readers.zarr import (
+    ZarrV3ChunkManifestVirtualBackend,
+    ZarrVirtualBackend,
+)
 
 __all__ = [
     "DMRPPVirtualBackend",
@@ -13,5 +16,6 @@
     "KerchunkVirtualBackend",
     "NetCDF3VirtualBackend",
     "TIFFVirtualBackend",
-    "ZarrV3VirtualBackend",
+    "ZarrVirtualBackend",
+    "ZarrV3ChunkManifestVirtualBackend",
 ]

virtualizarr/readers/common.py

@@ -60,18 +60,27 @@ def open_loadable_vars_and_indexes(
         # TODO we are reading a bunch of stuff we know we won't need here, e.g. all of the data variables...
         # TODO it would also be nice if we could somehow consolidate this with the reading of the kerchunk references
         # TODO really we probably want a dedicated xarray backend that iterates over all variables only once
-        fpath = _FsspecFSFromFilepath(
-            filepath=filepath, reader_options=reader_options
-        ).open_file()
 
         # fpath can be `Any` thanks to fsspec.filesystem(...).open() returning Any.
         # We'll (hopefully safely) cast it to what xarray is expecting, but this might let errors through.
+        fpath = _FsspecFSFromFilepath(filepath=filepath, reader_options=reader_options)
+
+        # Update the xarray open_dataset kwargs if Zarr
+
+        if fpath.filepath.suffix == ".zarr":
+            engine = "zarr"
+            xr_input = fpath.filepath
+
+        else:
+            engine = None
+            xr_input = fpath.open_file()
 
         ds = open_dataset(
-            cast(XArrayOpenT, fpath),
+            cast(XArrayOpenT, xr_input),
             drop_variables=drop_variables,
             group=group,
             decode_times=decode_times,
+            engine=engine,
         )
 
         if indexes is None:
@@ -87,7 +96,6 @@ def open_loadable_vars_and_indexes(
             raise NotImplementedError()
         else:
             indexes = dict(**indexes)  # for type hinting: to allow mutation
-
         # TODO we should drop these earlier by using drop_variables
         loadable_vars = {
             str(name): var