From 25fab8ec3a56b41dea95b8d20a8785ceef70e445 Mon Sep 17 00:00:00 2001 From: Justus Magin Date: Fri, 10 Nov 2023 17:53:08 +0100 Subject: [PATCH 1/9] remove `cdms2` from all environments --- ci/requirements/all-but-dask.yml | 2 -- ci/requirements/environment-windows.yml | 1 - ci/requirements/environment.yml | 1 - ci/requirements/min-all-deps.yml | 1 - 4 files changed, 5 deletions(-) diff --git a/ci/requirements/all-but-dask.yml b/ci/requirements/all-but-dask.yml index 4645be08b83..0161810df4b 100644 --- a/ci/requirements/all-but-dask.yml +++ b/ci/requirements/all-but-dask.yml @@ -3,13 +3,11 @@ channels: - conda-forge - nodefaults dependencies: - - python=3.10 - black - aiobotocore - boto3 - bottleneck - cartopy - - cdms2 - cftime - coveralls - flox diff --git a/ci/requirements/environment-windows.yml b/ci/requirements/environment-windows.yml index efa9ccb5a9a..021929b03f5 100644 --- a/ci/requirements/environment-windows.yml +++ b/ci/requirements/environment-windows.yml @@ -5,7 +5,6 @@ dependencies: - boto3 - bottleneck - cartopy - # - cdms2 # Not available on Windows - cftime - dask-core - distributed diff --git a/ci/requirements/environment.yml b/ci/requirements/environment.yml index 6e93ab7a946..d1e6f9115ba 100644 --- a/ci/requirements/environment.yml +++ b/ci/requirements/environment.yml @@ -7,7 +7,6 @@ dependencies: - boto3 - bottleneck - cartopy - - cdms2 - cftime - dask-core - distributed diff --git a/ci/requirements/min-all-deps.yml b/ci/requirements/min-all-deps.yml index 8400270ce1b..00fd750b31b 100644 --- a/ci/requirements/min-all-deps.yml +++ b/ci/requirements/min-all-deps.yml @@ -11,7 +11,6 @@ dependencies: - boto3=1.24 - bottleneck=1.3 - cartopy=0.20 - - cdms2=3.1 - cftime=1.6 - coveralls - dask-core=2022.7 From 280c330b1ea62d95d17c18dfe5fcacae924e4e17 Mon Sep 17 00:00:00 2001 From: Justus Magin Date: Fri, 10 Nov 2023 18:15:57 +0100 Subject: [PATCH 2/9] remove any code related to `cdms2` --- xarray/convert.py | 88 --------------------------- xarray/core/dataarray.py | 46 -------------- xarray/tests/test_dataarray.py | 107 --------------------------------- 3 files changed, 241 deletions(-) diff --git a/xarray/convert.py b/xarray/convert.py index 5863352ae41..aeb746f4a9c 100644 --- a/xarray/convert.py +++ b/xarray/convert.py @@ -3,7 +3,6 @@ from collections import Counter import numpy as np -import pandas as pd from xarray.coding.times import CFDatetimeCoder, CFTimedeltaCoder from xarray.conventions import decode_cf @@ -12,7 +11,6 @@ from xarray.core.dtypes import get_fill_value from xarray.core.pycompat import array_type -cdms2_ignored_attrs = {"name", "tileIndex"} iris_forbidden_keys = { "standard_name", "long_name", @@ -60,92 +58,6 @@ def _filter_attrs(attrs, ignored_attrs): return {k: v for k, v in attrs.items() if k not in ignored_attrs} -def from_cdms2(variable): - """Convert a cdms2 variable into an DataArray""" - values = np.asarray(variable) - name = variable.id - dims = variable.getAxisIds() - coords = {} - for axis in variable.getAxisList(): - coords[axis.id] = DataArray( - np.asarray(axis), - dims=[axis.id], - attrs=_filter_attrs(axis.attributes, cdms2_ignored_attrs), - ) - grid = variable.getGrid() - if grid is not None: - ids = [a.id for a in grid.getAxisList()] - for axis in grid.getLongitude(), grid.getLatitude(): - if axis.id not in variable.getAxisIds(): - coords[axis.id] = DataArray( - np.asarray(axis[:]), - dims=ids, - attrs=_filter_attrs(axis.attributes, cdms2_ignored_attrs), - ) - attrs = _filter_attrs(variable.attributes, cdms2_ignored_attrs) - dataarray = DataArray(values, dims=dims, coords=coords, name=name, attrs=attrs) - return decode_cf(dataarray.to_dataset())[dataarray.name] - - -def to_cdms2(dataarray, copy=True): - """Convert a DataArray into a cdms2 variable""" - # we don't want cdms2 to be a hard dependency - import cdms2 - - def set_cdms2_attrs(var, attrs): - for k, v in attrs.items(): - setattr(var, k, v) - - # 1D axes - axes = [] - for dim in dataarray.dims: - coord = encode(dataarray.coords[dim]) - axis = cdms2.createAxis(coord.values, id=dim) - set_cdms2_attrs(axis, coord.attrs) - axes.append(axis) - - # Data - var = encode(dataarray) - cdms2_var = cdms2.createVariable( - var.values, axes=axes, id=dataarray.name, mask=pd.isnull(var.values), copy=copy - ) - - # Attributes - set_cdms2_attrs(cdms2_var, var.attrs) - - # Curvilinear and unstructured grids - if dataarray.name not in dataarray.coords: - cdms2_axes = {} - for coord_name in set(dataarray.coords.keys()) - set(dataarray.dims): - coord_array = dataarray.coords[coord_name].to_cdms2() - - cdms2_axis_cls = ( - cdms2.coord.TransientAxis2D - if coord_array.ndim - else cdms2.auxcoord.TransientAuxAxis1D - ) - cdms2_axis = cdms2_axis_cls(coord_array) - if cdms2_axis.isLongitude(): - cdms2_axes["lon"] = cdms2_axis - elif cdms2_axis.isLatitude(): - cdms2_axes["lat"] = cdms2_axis - - if "lon" in cdms2_axes and "lat" in cdms2_axes: - if len(cdms2_axes["lon"].shape) == 2: - cdms2_grid = cdms2.hgrid.TransientCurveGrid( - cdms2_axes["lat"], cdms2_axes["lon"] - ) - else: - cdms2_grid = cdms2.gengrid.AbstractGenericGrid( - cdms2_axes["lat"], cdms2_axes["lon"] - ) - for axis in cdms2_grid.getAxisList(): - cdms2_var.setAxis(cdms2_var.getAxisIds().index(axis.id), axis) - cdms2_var.setGrid(cdms2_grid) - - return cdms2_var - - def _pick_attrs(attrs, keys): """Return attrs with keys in keys list""" return {k: v for k, v in attrs.items() if k in keys} diff --git a/xarray/core/dataarray.py b/xarray/core/dataarray.py index c512e742fb8..ca208a1486b 100644 --- a/xarray/core/dataarray.py +++ b/xarray/core/dataarray.py @@ -56,7 +56,6 @@ ReprObject, _default, either_dict_or_kwargs, - emit_user_level_warning, ) from xarray.core.variable import ( IndexVariable, @@ -81,10 +80,6 @@ from dask.delayed import Delayed except ImportError: Delayed = None # type: ignore - try: - from cdms2 import Variable as cdms2_Variable - except ImportError: - cdms2_Variable = None try: from iris.cube import Cube as iris_Cube except ImportError: @@ -4402,47 +4397,6 @@ def from_series(cls, series: pd.Series, sparse: bool = False) -> DataArray: result.name = series.name return result - def to_cdms2(self) -> cdms2_Variable: - """Convert this array into a cdms2.Variable - - .. deprecated:: 2023.06.0 - The `cdms2`_ library has been deprecated. Please consider using the - `xcdat`_ library instead. - - .. _cdms2: https://github.com/CDAT/cdms - .. _xcdat: https://github.com/xCDAT/xcdat - """ - from xarray.convert import to_cdms2 - - emit_user_level_warning( - "The cdms2 library has been deprecated." - " Please consider using the xcdat library instead.", - DeprecationWarning, - ) - - return to_cdms2(self) - - @classmethod - def from_cdms2(cls, variable: cdms2_Variable) -> Self: - """Convert a cdms2.Variable into an xarray.DataArray - - .. deprecated:: 2023.06.0 - The `cdms2`_ library has been deprecated. Please consider using the - `xcdat`_ library instead. - - .. _cdms2: https://github.com/CDAT/cdms - .. _xcdat: https://github.com/xCDAT/xcdat - """ - from xarray.convert import from_cdms2 - - emit_user_level_warning( - "The cdms2 library has been deprecated." - " Please consider using the xcdat library instead.", - DeprecationWarning, - ) - - return from_cdms2(variable) - def to_iris(self) -> iris_Cube: """Convert this array into a iris.cube.Cube""" from xarray.convert import to_iris diff --git a/xarray/tests/test_dataarray.py b/xarray/tests/test_dataarray.py index 1fbb834b679..0612f0a6ac6 100644 --- a/xarray/tests/test_dataarray.py +++ b/xarray/tests/test_dataarray.py @@ -12,7 +12,6 @@ import numpy as np import pandas as pd import pytest -from packaging.version import Version # remove once numpy 2.0 is the oldest supported version try: @@ -31,7 +30,6 @@ set_options, ) from xarray.coding.times import CFDatetimeCoder -from xarray.convert import from_cdms2 from xarray.core import dtypes from xarray.core.common import full_like from xarray.core.coordinates import Coordinates @@ -3663,111 +3661,6 @@ def test_to_masked_array(self) -> None: ma = da.to_masked_array() assert len(ma.mask) == N - @pytest.mark.skipif( - Version(np.__version__) > Version("1.24") or sys.version_info[:2] > (3, 10), - reason="cdms2 is unmaintained and does not support newer `numpy` or python versions", - ) - def test_to_and_from_cdms2_classic(self) -> None: - """Classic with 1D axes""" - pytest.importorskip("cdms2") - - original = DataArray( - np.arange(6).reshape(2, 3), - [ - ("distance", [-2, 2], {"units": "meters"}), - ("time", pd.date_range("2000-01-01", periods=3)), - ], - name="foo", - attrs={"baz": 123}, - ) - expected_coords = [ - IndexVariable("distance", [-2, 2]), - IndexVariable("time", [0, 1, 2]), - ] - with pytest.deprecated_call(match=".*cdms2"): - actual = original.to_cdms2() - assert_array_equal(actual.asma(), original) - assert actual.id == original.name - assert tuple(actual.getAxisIds()) == original.dims - for axis, coord in zip(actual.getAxisList(), expected_coords): - assert axis.id == coord.name - assert_array_equal(axis, coord.values) - assert actual.baz == original.attrs["baz"] - - component_times = actual.getAxis(1).asComponentTime() - assert len(component_times) == 3 - assert str(component_times[0]) == "2000-1-1 0:0:0.0" - - with pytest.deprecated_call(match=".*cdms2"): - roundtripped = DataArray.from_cdms2(actual) - assert_identical(original, roundtripped) - - back = from_cdms2(actual) - assert original.dims == back.dims - assert original.coords.keys() == back.coords.keys() - for coord_name in original.coords.keys(): - assert_array_equal(original.coords[coord_name], back.coords[coord_name]) - - @pytest.mark.skipif( - Version(np.__version__) > Version("1.24") or sys.version_info[:2] > (3, 10), - reason="cdms2 is unmaintained and does not support newer `numpy` or python versions", - ) - def test_to_and_from_cdms2_sgrid(self) -> None: - """Curvilinear (structured) grid - - The rectangular grid case is covered by the classic case - """ - pytest.importorskip("cdms2") - - lonlat = np.mgrid[:3, :4] - lon = DataArray(lonlat[1], dims=["y", "x"], name="lon") - lat = DataArray(lonlat[0], dims=["y", "x"], name="lat") - x = DataArray(np.arange(lon.shape[1]), dims=["x"], name="x") - y = DataArray(np.arange(lon.shape[0]), dims=["y"], name="y") - original = DataArray( - lonlat.sum(axis=0), - dims=["y", "x"], - coords=dict(x=x, y=y, lon=lon, lat=lat), - name="sst", - ) - with pytest.deprecated_call(): - actual = original.to_cdms2() - assert tuple(actual.getAxisIds()) == original.dims - assert_array_equal(original.coords["lon"], actual.getLongitude().asma()) - assert_array_equal(original.coords["lat"], actual.getLatitude().asma()) - - back = from_cdms2(actual) - assert original.dims == back.dims - assert set(original.coords.keys()) == set(back.coords.keys()) - assert_array_equal(original.coords["lat"], back.coords["lat"]) - assert_array_equal(original.coords["lon"], back.coords["lon"]) - - @pytest.mark.skipif( - Version(np.__version__) > Version("1.24") or sys.version_info[:2] > (3, 10), - reason="cdms2 is unmaintained and does not support newer `numpy` or python versions", - ) - def test_to_and_from_cdms2_ugrid(self) -> None: - """Unstructured grid""" - pytest.importorskip("cdms2") - - lon = DataArray(np.random.uniform(size=5), dims=["cell"], name="lon") - lat = DataArray(np.random.uniform(size=5), dims=["cell"], name="lat") - cell = DataArray(np.arange(5), dims=["cell"], name="cell") - original = DataArray( - np.arange(5), dims=["cell"], coords={"lon": lon, "lat": lat, "cell": cell} - ) - with pytest.deprecated_call(match=".*cdms2"): - actual = original.to_cdms2() - assert tuple(actual.getAxisIds()) == original.dims - assert_array_equal(original.coords["lon"], actual.getLongitude().getValue()) - assert_array_equal(original.coords["lat"], actual.getLatitude().getValue()) - - back = from_cdms2(actual) - assert set(original.dims) == set(back.dims) - assert set(original.coords.keys()) == set(back.coords.keys()) - assert_array_equal(original.coords["lat"], back.coords["lat"]) - assert_array_equal(original.coords["lon"], back.coords["lon"]) - def test_to_dataset_whole(self) -> None: unnamed = DataArray([1, 2], dims="x") with pytest.raises(ValueError, match=r"unable to convert unnamed"): From 042d4d75962722d54a16dc5cef9e2a733233b157 Mon Sep 17 00:00:00 2001 From: Justus Magin Date: Fri, 10 Nov 2023 18:16:30 +0100 Subject: [PATCH 3/9] remove the `mypy` override for `cdms2` --- pyproject.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index b16063e0370..0af51d98e11 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -88,7 +88,6 @@ module = [ "affine.*", "bottleneck.*", "cartopy.*", - "cdms2.*", "cf_units.*", "cfgrib.*", "cftime.*", From 9ac8a2949af8a34a3b1030a82977acb834349050 Mon Sep 17 00:00:00 2001 From: Justus Magin Date: Fri, 10 Nov 2023 18:18:36 +0100 Subject: [PATCH 4/9] remove the special environment files for python 3.11 --- .github/workflows/ci.yaml | 8 +--- ci/requirements/environment-py311.yml | 48 ------------------- ci/requirements/environment-windows-py311.yml | 44 ----------------- 3 files changed, 1 insertion(+), 99 deletions(-) delete mode 100644 ci/requirements/environment-py311.yml delete mode 100644 ci/requirements/environment-windows-py311.yml diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 7ee197aeda3..028cb3ac817 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -67,13 +67,7 @@ jobs: run: | echo "TODAY=$(date +'%Y-%m-%d')" >> $GITHUB_ENV - if [[ "${{matrix.python-version}}" == "3.11" ]]; then - if [[ ${{matrix.os}} == windows* ]]; then - echo "CONDA_ENV_FILE=ci/requirements/environment-windows-py311.yml" >> $GITHUB_ENV - else - echo "CONDA_ENV_FILE=ci/requirements/environment-py311.yml" >> $GITHUB_ENV - fi - elif [[ ${{ matrix.os }} == windows* ]] ; + if [[ ${{ matrix.os }} == windows* ]] ; then echo "CONDA_ENV_FILE=ci/requirements/environment-windows.yml" >> $GITHUB_ENV elif [[ "${{ matrix.env }}" != "" ]] ; diff --git a/ci/requirements/environment-py311.yml b/ci/requirements/environment-py311.yml deleted file mode 100644 index 0b9817daef3..00000000000 --- a/ci/requirements/environment-py311.yml +++ /dev/null @@ -1,48 +0,0 @@ -name: xarray-tests -channels: - - conda-forge - - nodefaults -dependencies: - - aiobotocore - - boto3 - - bottleneck - - cartopy - # - cdms2 - - cftime - - dask-core - - distributed - - flox - - fsspec!=2021.7.0 - - h5netcdf - - h5py - - hdf5 - - hypothesis - - iris - - lxml # Optional dep of pydap - - matplotlib-base - - nc-time-axis - - netcdf4 - - numba - - numbagg - - numexpr - - numpy - - packaging - - pandas - - pint<0.21 - - pip - - pooch - - pre-commit - - pseudonetcdf - - pydap - - pytest - - pytest-cov - - pytest-env - - pytest-xdist - - pytest-timeout - - rasterio - - scipy - - seaborn - - sparse - - toolz - - typing_extensions - - zarr diff --git a/ci/requirements/environment-windows-py311.yml b/ci/requirements/environment-windows-py311.yml deleted file mode 100644 index 8c36c5a9fd4..00000000000 --- a/ci/requirements/environment-windows-py311.yml +++ /dev/null @@ -1,44 +0,0 @@ -name: xarray-tests -channels: - - conda-forge -dependencies: - - boto3 - - bottleneck - - cartopy - # - cdms2 # Not available on Windows - - cftime - - dask-core - - distributed - - flox - - fsspec!=2021.7.0 - - h5netcdf - - h5py - - hdf5 - - hypothesis - - iris - - lxml # Optional dep of pydap - - matplotlib-base - - nc-time-axis - - netcdf4 - # - numba - # - numbagg - - numpy - - packaging - - pandas - - pint<0.21 - - pip - - pre-commit - - pseudonetcdf - - pydap - - pytest - - pytest-cov - - pytest-env - - pytest-xdist - - pytest-timeout - - rasterio - - scipy - - seaborn - # - sparse - - toolz - - typing_extensions - - zarr From 94d47b41626aa326d29198eb1ce5b3533e94adfb Mon Sep 17 00:00:00 2001 From: Justus Magin Date: Fri, 10 Nov 2023 18:28:12 +0100 Subject: [PATCH 5/9] whats-new --- doc/whats-new.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 157282803cc..4096975c4da 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -29,6 +29,8 @@ New Features Breaking changes ~~~~~~~~~~~~~~~~ +- drop support for `cdms2 `_ (:pull:`8441`). + By `Justus Magin Date: Fri, 10 Nov 2023 18:30:01 +0100 Subject: [PATCH 6/9] more info on the replacement --- doc/whats-new.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 4096975c4da..5ab9cfcd30f 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -29,7 +29,8 @@ New Features Breaking changes ~~~~~~~~~~~~~~~~ -- drop support for `cdms2 `_ (:pull:`8441`). +- drop support for `cdms2 `_. Please use + `xcdat `_ instead (:pull:`8441`). By `Justus Magin Date: Fri, 10 Nov 2023 18:30:22 +0100 Subject: [PATCH 7/9] remove the `cdms2` methods from the api docs --- doc/api.rst | 2 -- 1 file changed, 2 deletions(-) diff --git a/doc/api.rst b/doc/api.rst index 51d79161578..e48e71208cf 100644 --- a/doc/api.rst +++ b/doc/api.rst @@ -628,11 +628,9 @@ DataArray methods load_dataarray open_dataarray DataArray.as_numpy - DataArray.from_cdms2 DataArray.from_dict DataArray.from_iris DataArray.from_series - DataArray.to_cdms2 DataArray.to_dask_dataframe DataArray.to_dataframe DataArray.to_dataset From 9a8c09c8409b8cb62abc1502b4c7e2c3e6f94645 Mon Sep 17 00:00:00 2001 From: Justus Magin Date: Mon, 13 Nov 2023 20:34:53 +0100 Subject: [PATCH 8/9] remove all mentions of UV-CDAT / cdms2 from the faq --- doc/getting-started-guide/faq.rst | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/doc/getting-started-guide/faq.rst b/doc/getting-started-guide/faq.rst index e0e44dc7781..7f99fa77e3a 100644 --- a/doc/getting-started-guide/faq.rst +++ b/doc/getting-started-guide/faq.rst @@ -168,18 +168,11 @@ integration with Cartopy_. .. _Iris: https://scitools-iris.readthedocs.io/en/stable/ .. _Cartopy: https://scitools.org.uk/cartopy/docs/latest/ -`UV-CDAT`__ is another Python library that implements in-memory netCDF-like -variables and `tools for working with climate data`__. - -__ https://uvcdat.llnl.gov/ -__ https://drclimate.wordpress.com/2014/01/02/a-beginners-guide-to-scripting-with-uv-cdat/ - We think the design decisions we have made for xarray (namely, basing it on pandas) make it a faster and more flexible data analysis tool. That said, Iris -and CDAT have some great domain specific functionality, and xarray includes -methods for converting back and forth between xarray and these libraries. See -:py:meth:`~xarray.DataArray.to_iris` and :py:meth:`~xarray.DataArray.to_cdms2` -for more details. +has some great domain specific functionality, and xarray includes +methods for converting back and forth between xarray and Iris. See +:py:meth:`~xarray.DataArray.to_iris` for more details. What other projects leverage xarray? ------------------------------------ From 52e193175929cb5588f157c16cd33be273818b87 Mon Sep 17 00:00:00 2001 From: Justus Magin Date: Tue, 14 Nov 2023 12:17:51 +0100 Subject: [PATCH 9/9] also change the binder environment --- .binder/environment.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.binder/environment.yml b/.binder/environment.yml index fa4e14c41c2..053b12dfc86 100644 --- a/.binder/environment.yml +++ b/.binder/environment.yml @@ -6,7 +6,6 @@ dependencies: - boto3 - bottleneck - cartopy - - cdms2 - cfgrib - cftime - coveralls @@ -38,5 +37,4 @@ dependencies: - toolz - xarray - zarr - - pip: - - numbagg + - numbagg