From 05c9443a1361f6a35f0ec178af1701cc327b298e Mon Sep 17 00:00:00 2001 From: dcherian Date: Tue, 21 Jan 2020 09:12:43 -0700 Subject: [PATCH 01/11] Bump numpy to 1.15, pandas to 0.25, scipy to 1.2 --- ci/requirements/py36-bare-minimum.yml | 4 ++-- ci/requirements/py36-min-all-deps.yml | 6 +++--- ci/requirements/py36-min-nep18.yml | 2 +- doc/installing.rst | 4 ++-- doc/whats-new.rst | 9 +++++++++ setup.py | 2 +- 6 files changed, 18 insertions(+), 9 deletions(-) diff --git a/ci/requirements/py36-bare-minimum.yml b/ci/requirements/py36-bare-minimum.yml index 05186bc8748..8b604ce02dd 100644 --- a/ci/requirements/py36-bare-minimum.yml +++ b/ci/requirements/py36-bare-minimum.yml @@ -7,5 +7,5 @@ dependencies: - pytest - pytest-cov - pytest-env - - numpy=1.14 - - pandas=0.24 + - numpy=1.15 + - pandas=0.25 diff --git a/ci/requirements/py36-min-all-deps.yml b/ci/requirements/py36-min-all-deps.yml index e7756172311..88534ce6f3a 100644 --- a/ci/requirements/py36-min-all-deps.yml +++ b/ci/requirements/py36-min-all-deps.yml @@ -29,8 +29,8 @@ dependencies: - nc-time-axis=1.2 - netcdf4=1.4 - numba=0.44 - - numpy=1.14 - - pandas=0.24 + - numpy=1.15 + - pandas=0.25 # - pint # See py36-min-nep18.yml - pip - pseudonetcdf=3.0 @@ -40,7 +40,7 @@ dependencies: - pytest-cov - pytest-env - rasterio=1.0 - - scipy=1.0 # Policy allows for 1.2, but scipy>=1.1 breaks numpy=1.14 + - scipy=1.2 # Policy allows for 1.3 - seaborn=0.9 # - sparse # See py36-min-nep18.yml - toolz=0.10 diff --git a/ci/requirements/py36-min-nep18.yml b/ci/requirements/py36-min-nep18.yml index fc9523ce249..8fe7644d626 100644 --- a/ci/requirements/py36-min-nep18.yml +++ b/ci/requirements/py36-min-nep18.yml @@ -9,7 +9,7 @@ dependencies: - dask=2.4 - distributed=2.4 - numpy=1.17 - - pandas=0.24 + - pandas=0.25 - pint=0.9 # Actually not enough as it doesn't implement __array_function__yet! - pytest - pytest-cov diff --git a/doc/installing.rst b/doc/installing.rst index 219cf109efe..5c39f9a3c49 100644 --- a/doc/installing.rst +++ b/doc/installing.rst @@ -7,8 +7,8 @@ Required dependencies --------------------- - Python (3.6 or later) -- `numpy `__ (1.14 or later) -- `pandas `__ (0.24 or later) +- `numpy `__ (1.15 or later) +- `pandas `__ (0.25 or later) Optional dependencies --------------------- diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 25133df8a29..6cddb9c0d98 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -21,6 +21,15 @@ v0.15.0 (unreleased) Breaking changes ~~~~~~~~~~~~~~~~ +- Bumped minimum tested versions for dependencies: + - numpy 1.15 + - pandas 0.25 + - scipy 1.3 + - bottleneck 1.3 + - h5py 2.10 + - rasterio 1.1 + - pseudonetcdf + - Remove ``compat`` and ``encoding`` kwargs from ``DataArray``, which have been deprecated since 0.12. (:pull:`3650`). Instead, specify the encoding when writing to disk or set diff --git a/setup.py b/setup.py index cba0c74aa3a..581e156ccf4 100755 --- a/setup.py +++ b/setup.py @@ -22,7 +22,7 @@ ] PYTHON_REQUIRES = ">=3.6" -INSTALL_REQUIRES = ["numpy >= 1.14", "pandas >= 0.24"] +INSTALL_REQUIRES = ["numpy >= 1.15", "pandas >= 0.25"] needs_pytest = {"pytest", "test", "ptr"}.intersection(sys.argv) SETUP_REQUIRES = ["pytest-runner >= 4.2"] if needs_pytest else [] TESTS_REQUIRE = ["pytest >= 2.7.1"] From 3540398a3e7316310536d78fbb7796d519d9b2d4 Mon Sep 17 00:00:00 2001 From: dcherian Date: Tue, 21 Jan 2020 09:38:10 -0700 Subject: [PATCH 02/11] Remove some backcompat code --- xarray/core/dataarray.py | 2 +- xarray/core/dataset.py | 8 ++------ xarray/core/groupby.py | 2 +- xarray/core/nputils.py | 7 +------ xarray/core/variable.py | 12 +++--------- xarray/plot/utils.py | 8 +------- xarray/tests/test_dataarray.py | 4 ---- 7 files changed, 9 insertions(+), 34 deletions(-) diff --git a/xarray/core/dataarray.py b/xarray/core/dataarray.py index 0e67a791834..4fc6773c459 100644 --- a/xarray/core/dataarray.py +++ b/xarray/core/dataarray.py @@ -2969,7 +2969,7 @@ def quantile( See Also -------- - numpy.nanpercentile, pandas.Series.quantile, Dataset.quantile + numpy.nanquantile, pandas.Series.quantile, Dataset.quantile Examples -------- diff --git a/xarray/core/dataset.py b/xarray/core/dataset.py index 82ddc8a535f..8e8d3e1332b 100644 --- a/xarray/core/dataset.py +++ b/xarray/core/dataset.py @@ -4466,11 +4466,7 @@ def _set_sparse_data_from_dataframe( idx = dataframe.index if isinstance(idx, pd.MultiIndex): - try: - codes = idx.codes - except AttributeError: - # deprecated since pandas 0.24 - codes = idx.labels + codes = idx.codes coords = np.stack([np.asarray(code) for code in codes], axis=0) is_sorted = idx.is_lexsorted else: @@ -5171,7 +5167,7 @@ def quantile( See Also -------- - numpy.nanpercentile, pandas.Series.quantile, DataArray.quantile + numpy.nanquantile, pandas.Series.quantile, DataArray.quantile Examples -------- diff --git a/xarray/core/groupby.py b/xarray/core/groupby.py index 5b52f48413d..f2a9ebac6eb 100644 --- a/xarray/core/groupby.py +++ b/xarray/core/groupby.py @@ -595,7 +595,7 @@ def quantile(self, q, dim=None, interpolation="linear", keep_attrs=None): See Also -------- - numpy.nanpercentile, pandas.Series.quantile, Dataset.quantile, + numpy.nanquantile, pandas.Series.quantile, Dataset.quantile, DataArray.quantile Examples diff --git a/xarray/core/nputils.py b/xarray/core/nputils.py index 3fe2c254b0f..c3357c99040 100644 --- a/xarray/core/nputils.py +++ b/xarray/core/nputils.py @@ -14,12 +14,7 @@ def _validate_axis(data, axis): - ndim = data.ndim - if not -ndim <= axis < ndim: - raise IndexError(f"axis {axis!r} out of bounds [-{ndim}, {ndim})") - if axis < 0: - axis += ndim - return axis + return np.core.multiarray.normalize_axis_index(axis, data.ndim) def _select_along_axis(values, idx, axis): diff --git a/xarray/core/variable.py b/xarray/core/variable.py index 0a9d0767b77..74d5d57e6f6 100644 --- a/xarray/core/variable.py +++ b/xarray/core/variable.py @@ -1722,7 +1722,7 @@ def quantile(self, q, dim=None, interpolation="linear", keep_attrs=None): See Also -------- - numpy.nanpercentile, pandas.Series.quantile, Dataset.quantile, + numpy.nanquantile, pandas.Series.quantile, Dataset.quantile, DataArray.quantile """ @@ -1734,10 +1734,6 @@ def quantile(self, q, dim=None, interpolation="linear", keep_attrs=None): scalar = utils.is_scalar(q) q = np.atleast_1d(np.asarray(q, dtype=np.float64)) - # TODO: remove once numpy >= 1.15.0 is the minimum requirement - if np.count_nonzero(q < 0.0) or np.count_nonzero(q > 1.0): - raise ValueError("Quantiles must be in the range [0, 1]") - if dim is None: dim = self.dims @@ -1746,9 +1742,7 @@ def quantile(self, q, dim=None, interpolation="linear", keep_attrs=None): def _wrapper(npa, **kwargs): # move quantile axis to end. required for apply_ufunc - - # TODO: use np.nanquantile once numpy >= 1.15.0 is the minimum requirement - return np.moveaxis(np.nanpercentile(npa, **kwargs), 0, -1) + return np.moveaxis(np.nanquantile(npa, **kwargs), 0, -1) axis = np.arange(-1, -1 * len(dim) - 1, -1) result = apply_ufunc( @@ -1760,7 +1754,7 @@ def _wrapper(npa, **kwargs): output_dtypes=[np.float64], output_sizes={"quantile": len(q)}, dask="parallelized", - kwargs={"q": q * 100, "axis": axis, "interpolation": interpolation}, + kwargs={"q": q, "axis": axis, "interpolation": interpolation}, ) # for backward compatibility diff --git a/xarray/plot/utils.py b/xarray/plot/utils.py index 3b739197fea..0986d1203db 100644 --- a/xarray/plot/utils.py +++ b/xarray/plot/utils.py @@ -547,10 +547,6 @@ def _add_colorbar(primitive, ax, cbar_ax, cbar_kwargs, cmap_params): def _rescale_imshow_rgb(darray, vmin, vmax, robust): assert robust or vmin is not None or vmax is not None - # TODO: remove when min numpy version is bumped to 1.13 - # There's a cyclic dependency via DataArray, so we can't import from - # xarray.ufuncs in global scope. - from xarray.ufuncs import maximum, minimum # Calculate vmin and vmax automatically for `robust=True` if robust: @@ -579,9 +575,7 @@ def _rescale_imshow_rgb(darray, vmin, vmax, robust): # After scaling, downcast to 32-bit float. This substantially reduces # memory usage after we hand `darray` off to matplotlib. darray = ((darray.astype("f8") - vmin) / (vmax - vmin)).astype("f4") - with warnings.catch_warnings(): - warnings.filterwarnings("ignore", "xarray.ufuncs", PendingDeprecationWarning) - return minimum(maximum(darray, 0), 1) + return np.minimum(np.maximum(darray, 0), 1) def _update_axes( diff --git a/xarray/tests/test_dataarray.py b/xarray/tests/test_dataarray.py index 962be7548bc..ea220a52b1b 100644 --- a/xarray/tests/test_dataarray.py +++ b/xarray/tests/test_dataarray.py @@ -79,10 +79,6 @@ def test_repr_multiindex(self): ) assert expected == repr(self.mda) - @pytest.mark.skipif( - LooseVersion(np.__version__) < "1.15", - reason="old versions of numpy have different printing behavior", - ) def test_repr_multiindex_long(self): mindex_long = pd.MultiIndex.from_product( [["a", "b", "c", "d"], [1, 2, 3, 4, 5, 6, 7, 8]], From d078cf113b552948c80b2437bc1cbf60f7ffc262 Mon Sep 17 00:00:00 2001 From: dcherian Date: Tue, 21 Jan 2020 09:39:34 -0700 Subject: [PATCH 03/11] bump pandas in doc.yml --- ci/requirements/doc.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/requirements/doc.yml b/ci/requirements/doc.yml index 1b479ad4ca2..a8b72dc0956 100644 --- a/ci/requirements/doc.yml +++ b/ci/requirements/doc.yml @@ -17,7 +17,7 @@ dependencies: - numba - numpy - numpydoc - - pandas<0.25 # Hack around https://github.com/pydata/xarray/issues/3369 + - pandas - rasterio - seaborn - sphinx From fe0ae222b8a6da1e3e2309f5fece4d975cd22a50 Mon Sep 17 00:00:00 2001 From: dcherian Date: Tue, 21 Jan 2020 10:01:32 -0700 Subject: [PATCH 04/11] Fix repr test --- xarray/tests/test_dataarray.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/xarray/tests/test_dataarray.py b/xarray/tests/test_dataarray.py index ea220a52b1b..b9b719e8af9 100644 --- a/xarray/tests/test_dataarray.py +++ b/xarray/tests/test_dataarray.py @@ -79,6 +79,10 @@ def test_repr_multiindex(self): ) assert expected == repr(self.mda) + @pytest.mark.skipif( + LooseVersion(np.__version__) < "1.16", + reason="old versions of numpy have different printing behavior", + ) def test_repr_multiindex_long(self): mindex_long = pd.MultiIndex.from_product( [["a", "b", "c", "d"], [1, 2, 3, 4, 5, 6, 7, 8]], From e5653be63fa3d1ae096806515c463df2d9ce3d33 Mon Sep 17 00:00:00 2001 From: dcherian Date: Tue, 21 Jan 2020 10:05:06 -0700 Subject: [PATCH 05/11] review cleanups. --- xarray/core/dataset.py | 3 +-- xarray/core/nputils.py | 10 ++++------ 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/xarray/core/dataset.py b/xarray/core/dataset.py index 8e8d3e1332b..5ac79999795 100644 --- a/xarray/core/dataset.py +++ b/xarray/core/dataset.py @@ -4466,8 +4466,7 @@ def _set_sparse_data_from_dataframe( idx = dataframe.index if isinstance(idx, pd.MultiIndex): - codes = idx.codes - coords = np.stack([np.asarray(code) for code in codes], axis=0) + coords = np.stack([np.asarray(code) for code in idx.codes], axis=0) is_sorted = idx.is_lexsorted else: coords = np.arange(idx.size).reshape(1, -1) diff --git a/xarray/core/nputils.py b/xarray/core/nputils.py index c3357c99040..83c0bd78409 100644 --- a/xarray/core/nputils.py +++ b/xarray/core/nputils.py @@ -3,6 +3,8 @@ import numpy as np import pandas as pd +from np.core.multiarray import normalize_axis_index + try: import bottleneck as bn @@ -13,10 +15,6 @@ _USE_BOTTLENECK = False -def _validate_axis(data, axis): - return np.core.multiarray.normalize_axis_index(axis, data.ndim) - - def _select_along_axis(values, idx, axis): other_ind = np.ix_(*[np.arange(s) for s in idx.shape]) sl = other_ind[:axis] + (idx,) + other_ind[axis:] @@ -24,13 +22,13 @@ def _select_along_axis(values, idx, axis): def nanfirst(values, axis): - axis = _validate_axis(values, axis) + axis = normalize_axis_index(axis, values.ndim) idx_first = np.argmax(~pd.isnull(values), axis=axis) return _select_along_axis(values, idx_first, axis) def nanlast(values, axis): - axis = _validate_axis(values, axis) + axis = normalize_axis_index(axis, values.ndim) rev = (slice(None),) * axis + (slice(None, None, -1),) idx_last = -1 - np.argmax(~pd.isnull(values)[rev], axis=axis) return _select_along_axis(values, idx_last, axis) From f5d4fbe3c0c5b39158092da29a9af2eb2cb8b343 Mon Sep 17 00:00:00 2001 From: dcherian Date: Tue, 21 Jan 2020 10:12:52 -0700 Subject: [PATCH 06/11] scipy to 1.3 --- ci/requirements/py36-min-all-deps.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/requirements/py36-min-all-deps.yml b/ci/requirements/py36-min-all-deps.yml index 88534ce6f3a..90858a11935 100644 --- a/ci/requirements/py36-min-all-deps.yml +++ b/ci/requirements/py36-min-all-deps.yml @@ -40,7 +40,7 @@ dependencies: - pytest-cov - pytest-env - rasterio=1.0 - - scipy=1.2 # Policy allows for 1.3 + - scipy=1.3 - seaborn=0.9 # - sparse # See py36-min-nep18.yml - toolz=0.10 From 41e87d99114468814ff80588228e7ce58643ad16 Mon Sep 17 00:00:00 2001 From: dcherian Date: Tue, 21 Jan 2020 10:13:27 -0700 Subject: [PATCH 07/11] Revert "bump pandas in doc.yml" This reverts commit d078cf113b552948c80b2437bc1cbf60f7ffc262. --- ci/requirements/doc.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/requirements/doc.yml b/ci/requirements/doc.yml index a8b72dc0956..1b479ad4ca2 100644 --- a/ci/requirements/doc.yml +++ b/ci/requirements/doc.yml @@ -17,7 +17,7 @@ dependencies: - numba - numpy - numpydoc - - pandas + - pandas<0.25 # Hack around https://github.com/pydata/xarray/issues/3369 - rasterio - seaborn - sphinx From fdc78308b62693fde85411d2739a93930e8a34c4 Mon Sep 17 00:00:00 2001 From: dcherian Date: Tue, 21 Jan 2020 10:14:18 -0700 Subject: [PATCH 08/11] bugfix. --- xarray/core/nputils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xarray/core/nputils.py b/xarray/core/nputils.py index 83c0bd78409..275c2990f80 100644 --- a/xarray/core/nputils.py +++ b/xarray/core/nputils.py @@ -179,7 +179,7 @@ def _rolling_window(a, window, axis=-1): This function is taken from https://github.com/numpy/numpy/pull/31 but slightly modified to accept axis option. """ - axis = _validate_axis(a, axis) + axis = normalize_axis_index(axis, a.ndim) a = np.swapaxes(a, axis, -1) if window < 1: From d0d9c14cb9c28a4f1b97e20997ebc49b37c43bd8 Mon Sep 17 00:00:00 2001 From: dcherian Date: Tue, 21 Jan 2020 10:24:53 -0700 Subject: [PATCH 09/11] fix tests. --- xarray/core/nputils.py | 2 +- xarray/tests/test_units.py | 25 ++++++++++++++++++++----- 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/xarray/core/nputils.py b/xarray/core/nputils.py index 275c2990f80..dba67174fc1 100644 --- a/xarray/core/nputils.py +++ b/xarray/core/nputils.py @@ -3,7 +3,7 @@ import numpy as np import pandas as pd -from np.core.multiarray import normalize_axis_index +from numpy.core.multiarray import normalize_axis_index try: import bottleneck as bn diff --git a/xarray/tests/test_units.py b/xarray/tests/test_units.py index 6f9128839a7..d98e5e23516 100644 --- a/xarray/tests/test_units.py +++ b/xarray/tests/test_units.py @@ -1838,7 +1838,10 @@ def test_squeeze(self, dtype): "func", ( method("coarsen", windows={"y": 2}, func=np.mean), - method("quantile", q=[0.25, 0.75]), + pytest.param( + method("quantile", q=[0.25, 0.75]), + marks=pytest.mark.xfail(reason="nanquantile not implemented"), + ), pytest.param( method("rank", dim="x"), marks=pytest.mark.xfail(reason="rank not implemented for non-ndarray"), @@ -3401,7 +3404,10 @@ def test_stacking_reordering(self, func, dtype): method("diff", dim="x"), method("differentiate", coord="x"), method("integrate", dim="x"), - method("quantile", q=[0.25, 0.75]), + pytest.param( + method("quantile", q=[0.25, 0.75]), + marks=pytest.mark.xfail(reason="nanquantile not implemented"), + ), method("reduce", func=np.sum, dim="x"), pytest.param( lambda x: x.dot(x), @@ -3491,7 +3497,10 @@ def test_resample(self, dtype): method("assign_coords", z=(["x"], np.arange(5) * unit_registry.s)), method("first"), method("last"), - method("quantile", q=np.array([0.25, 0.5, 0.75]), dim="x"), + pytest.param( + method("quantile", q=[0.25, 0.5, 0.75], dim="x"), + marks=pytest.mark.xfail(reason="nanquantile not implemented"), + ), ), ids=repr, ) @@ -4929,7 +4938,10 @@ def test_reindex_like(self, unit, error, dtype): method("diff", dim="x"), method("differentiate", coord="x"), method("integrate", coord="x"), - method("quantile", q=[0.25, 0.75]), + pytest.param( + method("quantile", q=[0.25, 0.75]), + marks=pytest.mark.xfail(reason="nanquantile not implemented"), + ), method("reduce", func=np.sum, dim="x"), method("map", np.fabs), ), @@ -5039,7 +5051,10 @@ def test_resample(self, dtype): method("assign_coords", v=("x", np.arange(10) * unit_registry.s)), method("first"), method("last"), - method("quantile", q=[0.25, 0.5, 0.75], dim="x"), + pytest.param( + method("quantile", q=[0.25, 0.5, 0.75], dim="x"), + marks=pytest.mark.xfail(reason="nanquantile not implemented"), + ), ), ids=repr, ) From c3e42a757eea3869db34e9d6de2ce02e96490a0a Mon Sep 17 00:00:00 2001 From: dcherian Date: Wed, 22 Jan 2020 08:42:33 -0700 Subject: [PATCH 10/11] update setup.cfg --- setup.cfg | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.cfg b/setup.cfg index 0e113c15306..9ad0c8226ef 100644 --- a/setup.cfg +++ b/setup.cfg @@ -79,8 +79,8 @@ zip_safe = True include_package_data = True python_requires = >=3.6 install_requires = - numpy >= 1.14 - pandas >= 0.24 + numpy >= 1.15 + pandas >= 0.25 setup_requires = setuptools_scm [options.package_data] From 6e4d6eb59e464f828340be488e7e4ce466867b81 Mon Sep 17 00:00:00 2001 From: dcherian Date: Wed, 22 Jan 2020 08:53:03 -0700 Subject: [PATCH 11/11] Update whats-new --- doc/whats-new.rst | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 9707937229c..1940dfcba56 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -24,13 +24,10 @@ Breaking changes - Bumped minimum tested versions for dependencies: - numpy 1.15 - pandas 0.25 + - dask 2.2 + - distributed 2.2 - scipy 1.3 - - bottleneck 1.3 - - h5py 2.10 - - rasterio 1.1 - - pseudonetcdf -- Bumped minimum ``dask`` version to 2.2. - Remove ``compat`` and ``encoding`` kwargs from ``DataArray``, which have been deprecated since 0.12. (:pull:`3650`). Instead, specify the encoding when writing to disk or set