Merge branch 'main' into parsedims

jhamman · web-flow · commit 9e0f312ec7de · 2022-11-28T10:56:32.000-08:00
diff --git a/.github/workflows/upstream-dev-ci.yaml b/.github/workflows/upstream-dev-ci.yaml
@@ -80,6 +80,7 @@ jobs:
         if: success()
         id: status
         run: |
+          export ZARR_V3_EXPERIMENTAL_API=1
           python -m pytest --timeout=60 -rf \
             --report-log output-${{ matrix.python-version }}-log.jsonl
       - name: Generate and publish the report
diff --git a/ci/requirements/all-but-dask.yml b/ci/requirements/all-but-dask.yml
@@ -30,7 +30,6 @@ dependencies:
   - pip
   - pseudonetcdf
   - pydap
-  # - pynio  # not compatible with netCDF4>1.5.3, see #4491
   - pytest
   - pytest-cov
   - pytest-env
diff --git a/ci/requirements/environment-windows.yml b/ci/requirements/environment-windows.yml
@@ -30,7 +30,6 @@ dependencies:
   - pre-commit
   - pseudonetcdf
   - pydap
-  # - pynio  # Not available on Windows
   - pytest
   - pytest-cov
   - pytest-env
diff --git a/ci/requirements/environment.yml b/ci/requirements/environment.yml
@@ -34,7 +34,6 @@ dependencies:
   - pre-commit
   - pseudonetcdf
   - pydap
-  # - pynio  # not compatible with netCDF4>1.5.3, see #4491
   - pytest
   - pytest-cov
   - pytest-env
diff --git a/ci/requirements/min-all-deps.yml b/ci/requirements/min-all-deps.yml
@@ -40,7 +40,6 @@ dependencies:
   - pip
   - pseudonetcdf=3.2
   - pydap=3.2
-  # - pynio=1.5.5  # see: https://github.com/pydata/xarray/issues/4491
   - pytest
   - pytest-cov
   - pytest-env
diff --git a/doc/user-guide/io.rst b/doc/user-guide/io.rst
@@ -1209,6 +1209,10 @@ We recommend installing cfgrib via conda::
 Formats supported by PyNIO
 --------------------------
 
+.. warning::
+
+    The PyNIO backend is deprecated_. PyNIO is no longer maintained_. See
+
 Xarray can also read GRIB, HDF4 and other file formats supported by PyNIO_,
 if PyNIO is installed. To use PyNIO to read such files, supply
 ``engine='pynio'`` to :py:func:`open_dataset`.
@@ -1217,12 +1221,9 @@ We recommend installing PyNIO via conda::
 
     conda install -c conda-forge pynio
 
-.. warning::
-
-    PyNIO is no longer actively maintained and conflicts with netcdf4 > 1.5.3.
-    The PyNIO backend may be moved outside of xarray in the future.
-
 .. _PyNIO: https://www.pyngl.ucar.edu/Nio.shtml
+.. _deprecated: https://github.com/pydata/xarray/issues/4491
+.. _maintained: https://github.com/NCAR/pynio/issues/53
 
 .. _io.PseudoNetCDF:
 
diff --git a/doc/whats-new.rst b/doc/whats-new.rst
@@ -14,7 +14,6 @@ What's New
 
     np.random.seed(123456)
 
-
 .. _whats-new.2022.11.1:
 
 v2022.11.1 (unreleased)
@@ -23,6 +22,8 @@ v2022.11.1 (unreleased)
 New Features
 ~~~~~~~~~~~~
 
+- Add experimental support for Zarr's in-progress V3 specification. (:pull:`6475`).
+  By `Gregory Lee  <https://github.com/grlee77>`_ and `Joe Hamman <https://github.com/jhamman>`_.
 
 Breaking changes
 ~~~~~~~~~~~~~~~~
@@ -49,13 +50,16 @@ Breaking changes
 
 Deprecations
 ~~~~~~~~~~~~
-
+- The PyNIO backend has been deprecated (:issue:`4491`, :pull:`7301`).
+  By `Joe Hamman <https://github.com/jhamman>`_.
 
 Bug fixes
 ~~~~~~~~~
 
 - Import ``nc_time_axis`` when needed (:issue:`7275`, :pull:`7276`).
   By `Michael Niklas <https://github.com/headtr1ck>`_.
+- Fix static typing of :py:meth:`xr.polyval` (:issue:`7312`, :pull:`7315`).
+  By `Michael Niklas <https://github.com/headtr1ck>`_.
 
 Documentation
 ~~~~~~~~~~~~~
diff --git a/pyproject.toml b/pyproject.toml
@@ -40,8 +40,6 @@ module = [
   "cfgrib.*",
   "cftime.*",
   "cupy.*",
-  "dask.*",
-  "distributed.*",
   "fsspec.*",
   "h5netcdf.*",
   "h5py.*",
@@ -52,11 +50,9 @@ module = [
   "Nio.*",
   "nc_time_axis.*",
   "numbagg.*",
-  "numpy.*",
   "netCDF4.*",
   "netcdftime.*",
   "pandas.*",
-  "pint.*",
   "pooch.*",
   "PseudoNetCDF.*",
   "pydap.*",
@@ -70,8 +66,6 @@ module = [
   "zarr.*",
 ]
 
-# version spanning code is hard to type annotate (and most of this module will
-# be going away soon anyways)
 [[tool.mypy.overrides]]
 ignore_errors = true
-module = "xarray.core.pycompat"
+module = []
diff --git a/setup.cfg b/setup.cfg
@@ -147,11 +147,12 @@ markers =
 
 [flake8]
 ignore =
-    E203 # whitespace before ':' - doesn't work well with black
-    E402 # module level import not at top of file
-    E501 # line too long - let black worry about that
-    E731 # do not assign a lambda expression, use a def
-    W503 # line break before binary operator
+    # E203: whitespace before ':' - doesn't work well with black
+    # E402: module level import not at top of file
+    # E501: line too long - let black worry about that
+    # E731: do not assign a lambda expression, use a def
+    # W503: line break before binary operator
+    E203, E402, E501, E731, W503
 exclude =
     .eggs
     doc
diff --git a/xarray/backends/api.py b/xarray/backends/api.py
@@ -1504,6 +1504,7 @@ def to_zarr(
     region: Mapping[str, slice] | None = None,
     safe_chunks: bool = True,
     storage_options: dict[str, str] | None = None,
+    zarr_version: int | None = None,
 ) -> backends.ZarrStore:
     ...
 
@@ -1525,6 +1526,7 @@ def to_zarr(
     region: Mapping[str, slice] | None = None,
     safe_chunks: bool = True,
     storage_options: dict[str, str] | None = None,
+    zarr_version: int | None = None,
 ) -> Delayed:
     ...
 
@@ -1543,6 +1545,7 @@ def to_zarr(
     region: Mapping[str, slice] | None = None,
     safe_chunks: bool = True,
     storage_options: dict[str, str] | None = None,
+    zarr_version: int | None = None,
 ) -> backends.ZarrStore | Delayed:
     """This function creates an appropriate datastore for writing a dataset to
     a zarr ztore
@@ -1609,6 +1612,13 @@ def to_zarr(
                 f"``region`` with to_zarr(), got {append_dim} in both"
             )
 
+    if zarr_version is None:
+        # default to 2 if store doesn't specify it's version (e.g. a path)
+        zarr_version = int(getattr(store, "_store_version", 2))
+
+    if consolidated is None and zarr_version > 2:
+        consolidated = False
+
     if mode == "r+":
         already_consolidated = consolidated
         consolidate_on_close = False
@@ -1627,6 +1637,7 @@ def to_zarr(
         write_region=region,
         safe_chunks=safe_chunks,
         stacklevel=4,  # for Dataset.to_zarr()
+        zarr_version=zarr_version,
     )
 
     if mode in ["a", "r+"]:
diff --git a/xarray/backends/pynio_.py b/xarray/backends/pynio_.py
@@ -1,5 +1,7 @@
 from __future__ import annotations
 
+import warnings
+
 import numpy as np
 
 from ..core import indexing
@@ -55,6 +57,12 @@ class NioDataStore(AbstractDataStore):
     def __init__(self, filename, mode="r", lock=None, **kwargs):
         import Nio
 
+        warnings.warn(
+            "The PyNIO backend is Deprecated and will be removed from Xarray in a future release. "
+            "See https://github.com/pydata/xarray/issues/4491 for more information",
+            DeprecationWarning,
+        )
+
         if lock is None:
             lock = PYNIO_LOCK
         self.lock = ensure_lock(lock)
@@ -94,6 +102,15 @@ def close(self):
 
 
 class PynioBackendEntrypoint(BackendEntrypoint):
+    """
+    PyNIO backend
+
+        .. deprecated:: 0.20.0
+
+        Deprecated as PyNIO is no longer supported. See
+        https://github.com/pydata/xarray/issues/4491 for more information
+    """
+
     available = module_available("Nio")
 
     def open_dataset(
diff --git a/xarray/backends/zarr.py b/xarray/backends/zarr.py
@@ -353,19 +353,37 @@ def open_group(
         write_region=None,
         safe_chunks=True,
         stacklevel=2,
+        zarr_version=None,
     ):
         import zarr
 
         # zarr doesn't support pathlib.Path objects yet. zarr-python#601
         if isinstance(store, os.PathLike):
             store = os.fspath(store)
 
+        if zarr_version is None:
+            # default to 2 if store doesn't specify it's version (e.g. a path)
+            zarr_version = getattr(store, "_store_version", 2)
+
         open_kwargs = dict(
             mode=mode,
             synchronizer=synchronizer,
             path=group,
         )
         open_kwargs["storage_options"] = storage_options
+        if zarr_version > 2:
+            open_kwargs["zarr_version"] = zarr_version
+
+            if consolidated or consolidate_on_close:
+                raise ValueError(
+                    "consolidated metadata has not been implemented for zarr "
+                    f"version {zarr_version} yet. Set consolidated=False for "
+                    f"zarr version {zarr_version}. See also "
+                    "https://github.com/zarr-developers/zarr-specs/issues/136"
+                )
+
+            if consolidated is None:
+                consolidated = False
 
         if chunk_store:
             open_kwargs["chunk_store"] = chunk_store
@@ -440,6 +458,11 @@ def open_store_variable(self, name, zarr_array):
             zarr_array, DIMENSION_KEY, try_nczarr
         )
         attributes = dict(attributes)
+
+        # TODO: this should not be needed once
+        # https://github.com/zarr-developers/zarr-python/issues/1269 is resolved.
+        attributes.pop("filters", None)
+
         encoding = {
             "chunks": zarr_array.chunks,
             "preferred_chunks": dict(zip(dimensions, zarr_array.chunks)),
@@ -668,6 +691,7 @@ def open_zarr(
     storage_options=None,
     decode_timedelta=None,
     use_cftime=None,
+    zarr_version=None,
     **kwargs,
 ):
     """Load and decode a dataset from a Zarr store.
@@ -725,6 +749,9 @@ def open_zarr(
         capability. Only works for stores that have already been consolidated.
         By default (`consolidate=None`), attempts to read consolidated metadata,
         falling back to read non-consolidated metadata if that fails.
+
+        When the experimental ``zarr_version=3``, ``consolidated`` must be
+        either be ``None`` or ``False``.
     chunk_store : MutableMapping, optional
         A separate Zarr store only for chunk data.
     storage_options : dict, optional
@@ -745,6 +772,10 @@ def open_zarr(
         represented using ``np.datetime64[ns]`` objects.  If False, always
         decode times to ``np.datetime64[ns]`` objects; if this is not possible
         raise an error.
+    zarr_version : int or None, optional
+        The desired zarr spec version to target (currently 2 or 3). The default
+        of None will attempt to determine the zarr version from ``store`` when
+        possible, otherwise defaulting to 2.
 
     Returns
     -------
@@ -782,6 +813,7 @@ def open_zarr(
         "chunk_store": chunk_store,
         "storage_options": storage_options,
         "stacklevel": 4,
+        "zarr_version": zarr_version,
     }
 
     ds = open_dataset(
@@ -798,6 +830,7 @@ def open_zarr(
         backend_kwargs=backend_kwargs,
         decode_timedelta=decode_timedelta,
         use_cftime=use_cftime,
+        zarr_version=zarr_version,
     )
     return ds
 
@@ -842,6 +875,7 @@ def open_dataset(
         chunk_store=None,
         storage_options=None,
         stacklevel=3,
+        zarr_version=None,
     ):
 
         filename_or_obj = _normalize_path(filename_or_obj)
@@ -855,6 +889,7 @@ def open_dataset(
             chunk_store=chunk_store,
             storage_options=storage_options,
             stacklevel=stacklevel + 1,
+            zarr_version=zarr_version,
         )
 
         store_entrypoint = StoreBackendEntrypoint()
diff --git a/xarray/core/computation.py b/xarray/core/computation.py
@@ -1876,22 +1876,30 @@ def where(cond, x, y, keep_attrs=None):
 
 
 @overload
-def polyval(coord: DataArray, coeffs: DataArray, degree_dim: Hashable) -> DataArray:
+def polyval(
+    coord: DataArray, coeffs: DataArray, degree_dim: Hashable = "degree"
+) -> DataArray:
     ...
 
 
 @overload
-def polyval(coord: DataArray, coeffs: Dataset, degree_dim: Hashable) -> Dataset:
+def polyval(
+    coord: DataArray, coeffs: Dataset, degree_dim: Hashable = "degree"
+) -> Dataset:
     ...
 
 
 @overload
-def polyval(coord: Dataset, coeffs: DataArray, degree_dim: Hashable) -> Dataset:
+def polyval(
+    coord: Dataset, coeffs: DataArray, degree_dim: Hashable = "degree"
+) -> Dataset:
     ...
 
 
 @overload
-def polyval(coord: Dataset, coeffs: Dataset, degree_dim: Hashable) -> Dataset:
+def polyval(
+    coord: Dataset, coeffs: Dataset, degree_dim: Hashable = "degree"
+) -> Dataset:
     ...
 
 
diff --git a/xarray/core/dataset.py b/xarray/core/dataset.py
diff --git a/xarray/core/pycompat.py b/xarray/core/pycompat.py
diff --git a/xarray/tests/test_backends.py b/xarray/tests/test_backends.py
