From 62bb82558d4e9321c336f1178de2b96ca517b8a9 Mon Sep 17 00:00:00 2001 From: Keewis Date: Wed, 17 Jun 2020 22:50:51 +0200 Subject: [PATCH 01/50] globally promote UnitStrippedWarning to errors --- xarray/tests/test_units.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/xarray/tests/test_units.py b/xarray/tests/test_units.py index b477e8cccb2..b9d3f471b26 100644 --- a/xarray/tests/test_units.py +++ b/xarray/tests/test_units.py @@ -26,7 +26,7 @@ pytest.mark.skipif( not IS_NEP18_ACTIVE, reason="NUMPY_EXPERIMENTAL_ARRAY_FUNCTION is not enabled" ), - # pytest.mark.filterwarnings("ignore:::pint[.*]"), + pytest.mark.filterwarnings("error::pint.UnitStrippedWarning"), ] @@ -2166,7 +2166,6 @@ def test_pad_unit_constant_value(self, unit, error, dtype): class TestDataArray: - @pytest.mark.filterwarnings("error:::pint[.*]") @pytest.mark.parametrize( "variant", ( @@ -2201,7 +2200,6 @@ def test_init(self, variant, dtype): }.values() ) - @pytest.mark.filterwarnings("error:::pint[.*]") @pytest.mark.parametrize( "func", (pytest.param(str, id="str"), pytest.param(repr, id="repr")) ) @@ -3681,7 +3679,6 @@ def test_grouped_operations(self, func, dtype): xr.testing.assert_identical(expected, actual) -@pytest.mark.filterwarnings("error::pint.UnitStrippedWarning") class TestDataset: @pytest.mark.parametrize( "unit,error", From 6f7e7601be9397df7891dd5f1207a46bb041c30f Mon Sep 17 00:00:00 2001 From: Keewis Date: Wed, 17 Jun 2020 23:48:05 +0200 Subject: [PATCH 02/50] separately test apply_ufunc with units in dims, coords and data --- xarray/tests/test_units.py | 56 +++++++++++++++++++++++++++++++------- 1 file changed, 46 insertions(+), 10 deletions(-) diff --git a/xarray/tests/test_units.py b/xarray/tests/test_units.py index b9d3f471b26..391729d297a 100644 --- a/xarray/tests/test_units.py +++ b/xarray/tests/test_units.py @@ -354,14 +354,31 @@ def __repr__(self): return f"function_{self.name}" -def test_apply_ufunc_dataarray(dtype): +@pytest.mark.parametrize( + "variant", + ( + "data", + pytest.param( + "dims", marks=pytest.mark.xfail(reason="indexes don't support units") + ), + "coords", + ), +) +def test_apply_ufunc_dataarray(variant, dtype): + variants = { + "data": (unit_registry.m, 1, 1), + "dims": (1, unit_registry.m, 1), + "coords": (1, 1, unit_registry.m), + } + data_unit, dim_unit, coord_unit = variants.get(variant) func = functools.partial( xr.apply_ufunc, np.mean, input_core_dims=[["x"]], kwargs={"axis": -1} ) - array = np.linspace(0, 10, 20).astype(dtype) * unit_registry.m - x = np.arange(20) * unit_registry.s - data_array = xr.DataArray(data=array, dims="x", coords={"x": x}) + array = np.linspace(0, 10, 20).astype(dtype) * data_unit + x = np.arange(20) * dim_unit + u = np.linspace(-1, 1, 20) * coord_unit + data_array = xr.DataArray(data=array, dims="x", coords={"x": x, "u": ("x", u)}) expected = attach_units(func(strip_units(data_array)), extract_units(data_array)) actual = func(data_array) @@ -370,20 +387,39 @@ def test_apply_ufunc_dataarray(dtype): assert_identical(expected, actual) -def test_apply_ufunc_dataset(dtype): +@pytest.mark.parametrize( + "variant", + ( + "data", + pytest.param( + "dims", marks=pytest.mark.xfail(reason="indexes don't support units") + ), + "coords", + ), +) +def test_apply_ufunc_dataset(variant, dtype): + variants = { + "data": (unit_registry.m, 1, 1), + "dims": (1, unit_registry.m, 1), + "coords": (1, 1, unit_registry.s), + } + data_unit, dim_unit, coord_unit = variants.get(variant) + func = functools.partial( xr.apply_ufunc, np.mean, input_core_dims=[["x"]], kwargs={"axis": -1} ) - array1 = np.linspace(0, 10, 5 * 10).reshape(5, 10).astype(dtype) * unit_registry.m - array2 = np.linspace(0, 10, 5).astype(dtype) * unit_registry.m + array1 = np.linspace(0, 10, 5 * 10).reshape(5, 10).astype(dtype) * data_unit + array2 = np.linspace(0, 10, 5).astype(dtype) * data_unit + + x = np.arange(5) * dim_unit + y = np.arange(10) * dim_unit - x = np.arange(5) * unit_registry.s - y = np.arange(10) * unit_registry.m + u = np.linspace(-1, 1, 10) * coord_unit ds = xr.Dataset( data_vars={"a": (("x", "y"), array1), "b": ("x", array2)}, - coords={"x": x, "y": y}, + coords={"x": x, "y": y, "u": ("y", u)}, ) expected = attach_units(func(strip_units(ds)), extract_units(ds)) From b5a490291c3dd29d98cfe71a360650153964a567 Mon Sep 17 00:00:00 2001 From: Keewis Date: Fri, 19 Jun 2020 15:26:18 +0200 Subject: [PATCH 03/50] split the DataArray align test into data, dims and coords tests --- xarray/tests/test_units.py | 44 ++++++++++++++++++++++++-------------- 1 file changed, 28 insertions(+), 16 deletions(-) diff --git a/xarray/tests/test_units.py b/xarray/tests/test_units.py index 391729d297a..c396bd76999 100644 --- a/xarray/tests/test_units.py +++ b/xarray/tests/test_units.py @@ -450,29 +450,41 @@ def test_apply_ufunc_dataset(variant, dtype): "variant", ( "data", - pytest.param("dims", marks=pytest.mark.xfail(reason="indexes strip units")), + pytest.param( + "dims", marks=pytest.mark.xfail(reason="indexes don't support units") + ), "coords", ), ) @pytest.mark.parametrize("fill_value", (10, np.nan)) def test_align_dataarray(fill_value, variant, unit, error, dtype): + if variant == "coords" and ( + ~np.isnan(fill_value) or isinstance(unit, unit_registry.Unit) + ): + pytest.xfail(reason="fill_value is used for both data and coords") + original_unit = unit_registry.m variants = { - "data": (unit, original_unit, original_unit), - "dims": (original_unit, unit, original_unit), - "coords": (original_unit, original_unit, unit), + "data": ((original_unit, unit), (1, 1), (1, 1)), + "dims": ((1, 1), (original_unit, unit), (1, 1)), + "coords": ((1, 1), (1, 1), (original_unit, unit)), } - data_unit, dim_unit, coord_unit = variants.get(variant) + ( + (data_unit1, data_unit2), + (dim_unit1, dim_unit2), + (coord_unit1, coord_unit2), + ) = variants.get(variant) - array1 = np.linspace(0, 10, 2 * 5).reshape(2, 5).astype(dtype) * original_unit - array2 = np.linspace(0, 8, 2 * 5).reshape(2, 5).astype(dtype) * data_unit - x = np.arange(2) * original_unit + array1 = np.linspace(0, 10, 2 * 5).reshape(2, 5).astype(dtype) * data_unit1 + array2 = np.linspace(0, 8, 2 * 5).reshape(2, 5).astype(dtype) * data_unit2 + x = np.arange(2) * dim_unit1 - y1 = np.arange(5) * original_unit - y2 = np.arange(2, 7) * dim_unit - y_a1 = np.array([3, 5, 7, 8, 9]) * original_unit - y_a2 = np.array([7, 8, 9, 11, 13]) * coord_unit + y1 = np.arange(5) * dim_unit1 + y2 = np.arange(2, 7) * dim_unit2 + + y_a1 = np.array([3, 5, 7, 8, 9]) * coord_unit1 + y_a2 = np.array([7, 8, 9, 11, 13]) * coord_unit2 coords1 = {"x": x, "y": y1} coords2 = {"x": x, "y": y2} @@ -483,10 +495,10 @@ def test_align_dataarray(fill_value, variant, unit, error, dtype): data_array1 = xr.DataArray(data=array1, coords=coords1, dims=("x", "y")) data_array2 = xr.DataArray(data=array2, coords=coords2, dims=("x", "y")) - fill_value = fill_value * data_unit + fill_value = fill_value * data_unit2 func = function(xr.align, join="outer", fill_value=fill_value) - if error is not None and not ( - np.isnan(fill_value) and not isinstance(fill_value, Quantity) + if error is not None and ( + ~np.isnan(fill_value) or isinstance(fill_value, Quantity) ): with pytest.raises(error): func(data_array1, data_array2) @@ -495,7 +507,7 @@ def test_align_dataarray(fill_value, variant, unit, error, dtype): stripped_kwargs = { key: strip_units( - convert_units(value, {None: original_unit if data_unit != 1 else None}) + convert_units(value, {None: data_unit1 if data_unit2 != 1 else None}) ) for key, value in func.kwargs.items() } From 65473d41407fc981de387fa2322757f1d9efc128 Mon Sep 17 00:00:00 2001 From: Keewis Date: Fri, 19 Jun 2020 15:58:41 +0200 Subject: [PATCH 04/50] use dtypes instead of python types and use a dtype specific fill value --- xarray/tests/test_units.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/xarray/tests/test_units.py b/xarray/tests/test_units.py index c396bd76999..c54a0beb3b7 100644 --- a/xarray/tests/test_units.py +++ b/xarray/tests/test_units.py @@ -7,6 +7,7 @@ import pytest import xarray as xr +from xarray.core import dtypes from xarray.core.npcompat import IS_NEP18_ACTIVE from xarray.testing import assert_allclose, assert_equal, assert_identical @@ -264,7 +265,7 @@ def assert_units_equal(a, b): assert extract_units(a) == extract_units(b) -@pytest.fixture(params=[float, int]) +@pytest.fixture(params=[np.dtype(float), np.dtype(int)]) def dtype(request): return request.param @@ -456,13 +457,16 @@ def test_apply_ufunc_dataset(variant, dtype): "coords", ), ) -@pytest.mark.parametrize("fill_value", (10, np.nan)) +@pytest.mark.parametrize("fill_value", (10, dtypes.NA)) def test_align_dataarray(fill_value, variant, unit, error, dtype): if variant == "coords" and ( - ~np.isnan(fill_value) or isinstance(unit, unit_registry.Unit) + fill_value != dtypes.NA or isinstance(unit, unit_registry.Unit) ): pytest.xfail(reason="fill_value is used for both data and coords") + if fill_value == dtypes.NA: + fill_value = dtypes.get_fill_value(dtype) + original_unit = unit_registry.m variants = { From 3b0fdc365b619fb73c505768053afd9f6e4bf0c9 Mon Sep 17 00:00:00 2001 From: Keewis Date: Fri, 19 Jun 2020 16:02:09 +0200 Subject: [PATCH 05/50] rewrite the dataset align tests --- xarray/tests/test_units.py | 57 ++++++++++++++++++++++++++------------ 1 file changed, 39 insertions(+), 18 deletions(-) diff --git a/xarray/tests/test_units.py b/xarray/tests/test_units.py index c54a0beb3b7..ad2808d7a82 100644 --- a/xarray/tests/test_units.py +++ b/xarray/tests/test_units.py @@ -557,41 +557,55 @@ def test_align_dataarray(fill_value, variant, unit, error, dtype): "variant", ( "data", - pytest.param("dims", marks=pytest.mark.xfail(reason="indexes strip units")), + pytest.param( + "dims", marks=pytest.mark.xfail(reason="indexes don't support units") + ), "coords", ), ) -@pytest.mark.parametrize("fill_value", (np.float64(10), np.float64(np.nan))) +@pytest.mark.parametrize("fill_value", (10, dtypes.NA)) def test_align_dataset(fill_value, unit, variant, error, dtype): + if variant == "coords" and ( + fill_value != dtypes.NA or isinstance(unit, unit_registry.Unit) + ): + pytest.xfail(reason="fill_value is used for both data variables and coords") + + if fill_value == dtypes.NA: + fill_value = dtypes.get_fill_value(dtype) + original_unit = unit_registry.m variants = { - "data": (unit, original_unit, original_unit), - "dims": (original_unit, unit, original_unit), - "coords": (original_unit, original_unit, unit), + "data": ((original_unit, unit), (1, 1), (1, 1)), + "dims": ((1, 1), (original_unit, unit), (1, 1)), + "coords": ((1, 1), (1, 1), (original_unit, unit)), } - data_unit, dim_unit, coord_unit = variants.get(variant) + ( + (data_unit1, data_unit2), + (dim_unit1, dim_unit2), + (coord_unit1, coord_unit2), + ) = variants.get(variant) - array1 = np.linspace(0, 10, 2 * 5).reshape(2, 5).astype(dtype) * original_unit - array2 = np.linspace(0, 10, 2 * 5).reshape(2, 5).astype(dtype) * data_unit + array1 = np.linspace(0, 10, 2 * 5).reshape(2, 5).astype(dtype) * data_unit1 + array2 = np.linspace(0, 10, 2 * 5).reshape(2, 5).astype(dtype) * data_unit2 - x = np.arange(2) * original_unit + x = np.arange(2) * dim_unit1 + y1 = np.arange(5) * dim_unit1 + y2 = np.arange(2, 7) * dim_unit2 - y1 = np.arange(5) * original_unit - y2 = np.arange(2, 7) * dim_unit - y_a1 = np.array([3, 5, 7, 8, 9]) * original_unit - y_a2 = np.array([7, 8, 9, 11, 13]) * coord_unit + u1 = np.array([3, 5, 7, 8, 9]) * coord_unit1 + u2 = np.array([7, 8, 9, 11, 13]) * coord_unit2 coords1 = {"x": x, "y": y1} coords2 = {"x": x, "y": y2} if variant == "coords": - coords1["y_a"] = ("y", y_a1) - coords2["y_a"] = ("y", y_a2) + coords1["u"] = ("y", u1) + coords2["u"] = ("y", u2) ds1 = xr.Dataset(data_vars={"a": (("x", "y"), array1)}, coords=coords1) ds2 = xr.Dataset(data_vars={"a": (("x", "y"), array2)}, coords=coords2) - fill_value = fill_value * data_unit + fill_value = fill_value * data_unit2 func = function(xr.align, join="outer", fill_value=fill_value) if error is not None and not ( np.isnan(fill_value) and not isinstance(fill_value, Quantity) @@ -603,14 +617,21 @@ def test_align_dataset(fill_value, unit, variant, error, dtype): stripped_kwargs = { key: strip_units( - convert_units(value, {None: original_unit if data_unit != 1 else None}) + convert_units( + value, + { + None: data_unit1 + if isinstance(data_unit2, unit_registry.Unit) + else None + }, + ) ) for key, value in func.kwargs.items() } units_a = extract_units(ds1) units_b = extract_units(ds2) expected_a, expected_b = func( - strip_units(ds1), strip_units(convert_units(ds2, units_a)), **stripped_kwargs + strip_units(ds1), strip_units(convert_units(ds2, units_a)), **stripped_kwargs, ) expected_a = attach_units(expected_a, units_a) if isinstance(array2, Quantity): From a4ce5d826cd639d14502cc4cbd3c604ad50fe384 Mon Sep 17 00:00:00 2001 From: Keewis Date: Fri, 19 Jun 2020 16:42:15 +0200 Subject: [PATCH 06/50] compare with dtypes.NA instead of using np.isnan --- xarray/tests/test_units.py | 45 ++++++++++++++------------------------ 1 file changed, 16 insertions(+), 29 deletions(-) diff --git a/xarray/tests/test_units.py b/xarray/tests/test_units.py index ad2808d7a82..2f16f64d760 100644 --- a/xarray/tests/test_units.py +++ b/xarray/tests/test_units.py @@ -457,15 +457,14 @@ def test_apply_ufunc_dataset(variant, dtype): "coords", ), ) -@pytest.mark.parametrize("fill_value", (10, dtypes.NA)) -def test_align_dataarray(fill_value, variant, unit, error, dtype): +@pytest.mark.parametrize("value", (10, dtypes.NA)) +def test_align_dataarray(value, variant, unit, error, dtype): if variant == "coords" and ( - fill_value != dtypes.NA or isinstance(unit, unit_registry.Unit) + value != dtypes.NA or isinstance(unit, unit_registry.Unit) ): pytest.xfail(reason="fill_value is used for both data and coords") - if fill_value == dtypes.NA: - fill_value = dtypes.get_fill_value(dtype) + fill_value = dtypes.get_fill_value(dtype) if value == dtypes.NA else value original_unit = unit_registry.m @@ -482,28 +481,26 @@ def test_align_dataarray(fill_value, variant, unit, error, dtype): array1 = np.linspace(0, 10, 2 * 5).reshape(2, 5).astype(dtype) * data_unit1 array2 = np.linspace(0, 8, 2 * 5).reshape(2, 5).astype(dtype) * data_unit2 - x = np.arange(2) * dim_unit1 + x = np.arange(2) * dim_unit1 y1 = np.arange(5) * dim_unit1 y2 = np.arange(2, 7) * dim_unit2 - y_a1 = np.array([3, 5, 7, 8, 9]) * coord_unit1 - y_a2 = np.array([7, 8, 9, 11, 13]) * coord_unit2 + u1 = np.array([3, 5, 7, 8, 9]) * coord_unit1 + u2 = np.array([7, 8, 9, 11, 13]) * coord_unit2 coords1 = {"x": x, "y": y1} coords2 = {"x": x, "y": y2} if variant == "coords": - coords1["y_a"] = ("y", y_a1) - coords2["y_a"] = ("y", y_a2) + coords1["y_a"] = ("y", u1) + coords2["y_a"] = ("y", u2) data_array1 = xr.DataArray(data=array1, coords=coords1, dims=("x", "y")) data_array2 = xr.DataArray(data=array2, coords=coords2, dims=("x", "y")) fill_value = fill_value * data_unit2 func = function(xr.align, join="outer", fill_value=fill_value) - if error is not None and ( - ~np.isnan(fill_value) or isinstance(fill_value, Quantity) - ): + if error is not None and (value != dtypes.NA or isinstance(fill_value, Quantity)): with pytest.raises(error): func(data_array1, data_array2) @@ -563,15 +560,14 @@ def test_align_dataarray(fill_value, variant, unit, error, dtype): "coords", ), ) -@pytest.mark.parametrize("fill_value", (10, dtypes.NA)) -def test_align_dataset(fill_value, unit, variant, error, dtype): +@pytest.mark.parametrize("value", (10, dtypes.NA)) +def test_align_dataset(value, unit, variant, error, dtype): if variant == "coords" and ( - fill_value != dtypes.NA or isinstance(unit, unit_registry.Unit) + value != dtypes.NA or isinstance(unit, unit_registry.Unit) ): pytest.xfail(reason="fill_value is used for both data variables and coords") - if fill_value == dtypes.NA: - fill_value = dtypes.get_fill_value(dtype) + fill_value = dtypes.get_fill_value(dtype) if value == dtypes.NA else value original_unit = unit_registry.m @@ -607,9 +603,7 @@ def test_align_dataset(fill_value, unit, variant, error, dtype): fill_value = fill_value * data_unit2 func = function(xr.align, join="outer", fill_value=fill_value) - if error is not None and not ( - np.isnan(fill_value) and not isinstance(fill_value, Quantity) - ): + if error is not None and (value != dtypes.NA or isinstance(fill_value, Quantity)): with pytest.raises(error): func(ds1, ds2) @@ -617,14 +611,7 @@ def test_align_dataset(fill_value, unit, variant, error, dtype): stripped_kwargs = { key: strip_units( - convert_units( - value, - { - None: data_unit1 - if isinstance(data_unit2, unit_registry.Unit) - else None - }, - ) + convert_units(value, {None: data_unit1 if data_unit2 != 1 else None},) ) for key, value in func.kwargs.items() } From 15399b2bdc83ee2a71ca5f34ccc09958bcfa78d6 Mon Sep 17 00:00:00 2001 From: Keewis Date: Fri, 19 Jun 2020 17:10:57 +0200 Subject: [PATCH 07/50] mention the issue in the xfail reason --- xarray/tests/test_units.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/xarray/tests/test_units.py b/xarray/tests/test_units.py index 2f16f64d760..3633f6d9a2b 100644 --- a/xarray/tests/test_units.py +++ b/xarray/tests/test_units.py @@ -462,7 +462,12 @@ def test_align_dataarray(value, variant, unit, error, dtype): if variant == "coords" and ( value != dtypes.NA or isinstance(unit, unit_registry.Unit) ): - pytest.xfail(reason="fill_value is used for both data and coords") + pytest.xfail( + reason=( + "fill_value is used for both data variables and coords. " + "See https://github.com/pydata/xarray/issues/4165" + ) + ) fill_value = dtypes.get_fill_value(dtype) if value == dtypes.NA else value @@ -565,7 +570,12 @@ def test_align_dataset(value, unit, variant, error, dtype): if variant == "coords" and ( value != dtypes.NA or isinstance(unit, unit_registry.Unit) ): - pytest.xfail(reason="fill_value is used for both data variables and coords") + pytest.xfail( + reason=( + "fill_value is used for both data variables and coords. " + "See https://github.com/pydata/xarray/issues/4165" + ) + ) fill_value = dtypes.get_fill_value(dtype) if value == dtypes.NA else value @@ -611,7 +621,7 @@ def test_align_dataset(value, unit, variant, error, dtype): stripped_kwargs = { key: strip_units( - convert_units(value, {None: data_unit1 if data_unit2 != 1 else None},) + convert_units(value, {None: data_unit1 if data_unit2 != 1 else None}) ) for key, value in func.kwargs.items() } From 86bc323c0c7244298819ceaf4fdc5ab87cc4fdc2 Mon Sep 17 00:00:00 2001 From: Keewis Date: Fri, 19 Jun 2020 17:17:27 +0200 Subject: [PATCH 08/50] make sure the combine_* variants are properly separated from each other --- xarray/tests/test_units.py | 88 +++++++++++++++++++++----------------- 1 file changed, 48 insertions(+), 40 deletions(-) diff --git a/xarray/tests/test_units.py b/xarray/tests/test_units.py index 3633f6d9a2b..02d3e7c4b38 100644 --- a/xarray/tests/test_units.py +++ b/xarray/tests/test_units.py @@ -725,31 +725,35 @@ def test_combine_by_coords(variant, unit, error, dtype): original_unit = unit_registry.m variants = { - "data": (unit, original_unit, original_unit), - "dims": (original_unit, unit, original_unit), - "coords": (original_unit, original_unit, unit), + "data": ((original_unit, unit), (1, 1), (1, 1)), + "dims": ((1, 1), (original_unit, unit), (1, 1)), + "coords": ((1, 1), (1, 1), (original_unit, unit)), } - data_unit, dim_unit, coord_unit = variants.get(variant) + ( + (data_unit1, data_unit2), + (dim_unit1, dim_unit2), + (coord_unit1, coord_unit2), + ) = variants.get(variant) - array1 = np.zeros(shape=(2, 3), dtype=dtype) * original_unit - array2 = np.zeros(shape=(2, 3), dtype=dtype) * original_unit - x = np.arange(1, 4) * 10 * original_unit - y = np.arange(2) * original_unit - z = np.arange(3) * original_unit + array1 = np.zeros(shape=(2, 3), dtype=dtype) * data_unit1 + array2 = np.zeros(shape=(2, 3), dtype=dtype) * data_unit1 + x = np.arange(1, 4) * 10 * dim_unit1 + y = np.arange(2) * dim_unit1 + u = np.arange(3) * coord_unit1 - other_array1 = np.ones_like(array1) * data_unit - other_array2 = np.ones_like(array2) * data_unit - other_x = np.arange(1, 4) * 10 * dim_unit - other_y = np.arange(2, 4) * dim_unit - other_z = np.arange(3, 6) * coord_unit + other_array1 = np.ones_like(array1) * data_unit2 + other_array2 = np.ones_like(array2) * data_unit2 + other_x = np.arange(1, 4) * 10 * dim_unit2 + other_y = np.arange(2, 4) * dim_unit2 + other_u = np.arange(3, 6) * coord_unit2 ds = xr.Dataset( data_vars={"a": (("y", "x"), array1), "b": (("y", "x"), array2)}, - coords={"x": x, "y": y, "z": ("x", z)}, + coords={"x": x, "y": y, "u": ("x", u)}, ) other = xr.Dataset( data_vars={"a": (("y", "x"), other_array1), "b": (("y", "x"), other_array2)}, - coords={"x": other_x, "y": other_y, "z": ("x", other_z)}, + coords={"x": other_x, "y": other_y, "u": ("x", other_u)}, ) if error is not None: @@ -796,18 +800,22 @@ def test_combine_nested(variant, unit, error, dtype): original_unit = unit_registry.m variants = { - "data": (unit, original_unit, original_unit), - "dims": (original_unit, unit, original_unit), - "coords": (original_unit, original_unit, unit), + "data": ((original_unit, unit), (1, 1), (1, 1)), + "dims": ((1, 1), (original_unit, unit), (1, 1)), + "coords": ((1, 1), (1, 1), (original_unit, unit)), } - data_unit, dim_unit, coord_unit = variants.get(variant) + ( + (data_unit1, data_unit2), + (dim_unit1, dim_unit2), + (coord_unit1, coord_unit2), + ) = variants.get(variant) - array1 = np.zeros(shape=(2, 3), dtype=dtype) * original_unit - array2 = np.zeros(shape=(2, 3), dtype=dtype) * original_unit + array1 = np.zeros(shape=(2, 3), dtype=dtype) * data_unit1 + array2 = np.zeros(shape=(2, 3), dtype=dtype) * data_unit1 - x = np.arange(1, 4) * 10 * original_unit - y = np.arange(2) * original_unit - z = np.arange(3) * original_unit + x = np.arange(1, 4) * 10 * dim_unit1 + y = np.arange(2) * dim_unit1 + z = np.arange(3) * coord_unit1 ds1 = xr.Dataset( data_vars={"a": (("y", "x"), array1), "b": (("y", "x"), array2)}, @@ -815,35 +823,35 @@ def test_combine_nested(variant, unit, error, dtype): ) ds2 = xr.Dataset( data_vars={ - "a": (("y", "x"), np.ones_like(array1) * data_unit), - "b": (("y", "x"), np.ones_like(array2) * data_unit), + "a": (("y", "x"), np.ones_like(array1) * data_unit2), + "b": (("y", "x"), np.ones_like(array2) * data_unit2), }, coords={ - "x": np.arange(3) * dim_unit, - "y": np.arange(2, 4) * dim_unit, - "z": ("x", np.arange(-3, 0) * coord_unit), + "x": np.arange(3) * dim_unit2, + "y": np.arange(2, 4) * dim_unit2, + "z": ("x", np.arange(-3, 0) * coord_unit2), }, ) ds3 = xr.Dataset( data_vars={ - "a": (("y", "x"), np.zeros_like(array1) * np.nan * data_unit), - "b": (("y", "x"), np.zeros_like(array2) * np.nan * data_unit), + "a": (("y", "x"), np.full_like(array1, fill_value=np.nan) * data_unit2), + "b": (("y", "x"), np.full_like(array2, fill_value=np.nan) * data_unit2), }, coords={ - "x": np.arange(3, 6) * dim_unit, - "y": np.arange(4, 6) * dim_unit, - "z": ("x", np.arange(3, 6) * coord_unit), + "x": np.arange(3, 6) * dim_unit2, + "y": np.arange(4, 6) * dim_unit2, + "z": ("x", np.arange(3, 6) * coord_unit2), }, ) ds4 = xr.Dataset( data_vars={ - "a": (("y", "x"), -1 * np.ones_like(array1) * data_unit), - "b": (("y", "x"), -1 * np.ones_like(array2) * data_unit), + "a": (("y", "x"), -1 * np.ones_like(array1) * data_unit2), + "b": (("y", "x"), -1 * np.ones_like(array2) * data_unit2), }, coords={ - "x": np.arange(6, 9) * dim_unit, - "y": np.arange(6, 8) * dim_unit, - "z": ("x", np.arange(6, 9) * coord_unit), + "x": np.arange(6, 9) * dim_unit2, + "y": np.arange(6, 8) * dim_unit2, + "z": ("x", np.arange(6, 9) * coord_unit2), }, ) From 5cc076c80c7aa6dd0cbec1610637c2f1ffaea7fe Mon Sep 17 00:00:00 2001 From: Keewis Date: Fri, 19 Jun 2020 17:52:26 +0200 Subject: [PATCH 09/50] improve the test case names --- xarray/tests/test_units.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xarray/tests/test_units.py b/xarray/tests/test_units.py index 02d3e7c4b38..385af017591 100644 --- a/xarray/tests/test_units.py +++ b/xarray/tests/test_units.py @@ -265,7 +265,7 @@ def assert_units_equal(a, b): assert extract_units(a) == extract_units(b) -@pytest.fixture(params=[np.dtype(float), np.dtype(int)]) +@pytest.fixture(params=[np.dtype(float), np.dtype(int)], ids=str) def dtype(request): return request.param From 4b71aa606d78b83dccdf2e1d87bee302147ab4da Mon Sep 17 00:00:00 2001 From: Keewis Date: Fri, 19 Jun 2020 23:52:22 +0200 Subject: [PATCH 10/50] note that broadcast uses align --- xarray/tests/test_units.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/xarray/tests/test_units.py b/xarray/tests/test_units.py index 385af017591..99eb041c5aa 100644 --- a/xarray/tests/test_units.py +++ b/xarray/tests/test_units.py @@ -645,6 +645,7 @@ def test_align_dataset(value, unit, variant, error, dtype): def test_broadcast_dataarray(dtype): + # uses align internally so more thorough tests are not needed array1 = np.linspace(0, 10, 2) * unit_registry.Pa array2 = np.linspace(0, 10, 3) * unit_registry.Pa @@ -666,6 +667,7 @@ def test_broadcast_dataarray(dtype): def test_broadcast_dataset(dtype): + # uses align internally so more thorough tests are not needed array1 = np.linspace(0, 10, 2) * unit_registry.Pa array2 = np.linspace(0, 10, 3) * unit_registry.Pa From 218bb27e9e369d38fcc4ab955529165d8fe50e8e Mon Sep 17 00:00:00 2001 From: Keewis Date: Fri, 19 Jun 2020 23:57:14 +0200 Subject: [PATCH 11/50] properly separate the test cases for concat --- xarray/tests/test_units.py | 58 +++++++++++++++++++++++++++----------- 1 file changed, 42 insertions(+), 16 deletions(-) diff --git a/xarray/tests/test_units.py b/xarray/tests/test_units.py index 99eb041c5aa..b4190cb77d7 100644 --- a/xarray/tests/test_units.py +++ b/xarray/tests/test_units.py @@ -899,21 +899,34 @@ def test_combine_nested(variant, unit, error, dtype): ( "data", pytest.param("dims", marks=pytest.mark.xfail(reason="indexes strip units")), + "coords", ), ) def test_concat_dataarray(variant, unit, error, dtype): original_unit = unit_registry.m - variants = {"data": (unit, original_unit), "dims": (original_unit, unit)} - data_unit, dims_unit = variants.get(variant) + variants = { + "data": ((original_unit, unit), (1, 1), (1, 1)), + "dims": ((1, 1), (original_unit, unit), (1, 1)), + "coords": ((1, 1), (1, 1), (original_unit, unit)), + } + ( + (data_unit1, data_unit2), + (dim_unit1, dim_unit2), + (coord_unit1, coord_unit2), + ) = variants.get(variant) + + array1 = np.linspace(0, 5, 10).astype(dtype) * data_unit1 + array2 = np.linspace(-5, 0, 5).astype(dtype) * data_unit2 - array1 = np.linspace(0, 5, 10).astype(dtype) * unit_registry.m - array2 = np.linspace(-5, 0, 5).astype(dtype) * data_unit - x1 = np.arange(5, 15) * original_unit - x2 = np.arange(5) * dims_unit + x1 = np.arange(5, 15) * dim_unit1 + x2 = np.arange(5) * dim_unit2 + + u1 = np.linspace(1, 2, 10).astype(dtype) * coord_unit1 + u2 = np.linspace(0, 1, 5).astype(dtype) * coord_unit2 - arr1 = xr.DataArray(data=array1, coords={"x": x1}, dims="x") - arr2 = xr.DataArray(data=array2, coords={"x": x2}, dims="x") + arr1 = xr.DataArray(data=array1, coords={"x": x1, "u": ("x", u1)}, dims="x") + arr2 = xr.DataArray(data=array2, coords={"x": x2, "u": ("x", u2)}, dims="x") if error is not None: with pytest.raises(error): @@ -952,21 +965,34 @@ def test_concat_dataarray(variant, unit, error, dtype): ( "data", pytest.param("dims", marks=pytest.mark.xfail(reason="indexes strip units")), + "coords", ), ) def test_concat_dataset(variant, unit, error, dtype): original_unit = unit_registry.m - variants = {"data": (unit, original_unit), "dims": (original_unit, unit)} - data_unit, dims_unit = variants.get(variant) + variants = { + "data": ((original_unit, unit), (1, 1), (1, 1)), + "dims": ((1, 1), (original_unit, unit), (1, 1)), + "coords": ((1, 1), (1, 1), (original_unit, unit)), + } + ( + (data_unit1, data_unit2), + (dim_unit1, dim_unit2), + (coord_unit1, coord_unit2), + ) = variants.get(variant) + + array1 = np.linspace(0, 5, 10).astype(dtype) * data_unit1 + array2 = np.linspace(-5, 0, 5).astype(dtype) * data_unit2 - array1 = np.linspace(0, 5, 10).astype(dtype) * unit_registry.m - array2 = np.linspace(-5, 0, 5).astype(dtype) * data_unit - x1 = np.arange(5, 15) * original_unit - x2 = np.arange(5) * dims_unit + x1 = np.arange(5, 15) * dim_unit1 + x2 = np.arange(5) * dim_unit2 + + u1 = np.linspace(1, 2, 10).astype(dtype) * coord_unit1 + u2 = np.linspace(0, 1, 5).astype(dtype) * coord_unit2 - ds1 = xr.Dataset(data_vars={"a": ("x", array1)}, coords={"x": x1}) - ds2 = xr.Dataset(data_vars={"a": ("x", array2)}, coords={"x": x2}) + ds1 = xr.Dataset(data_vars={"a": ("x", array1)}, coords={"x": x1, "u": ("x", u1)}) + ds2 = xr.Dataset(data_vars={"a": ("x", array2)}, coords={"x": x2, "u": ("x", u2)}) if error is not None: with pytest.raises(error): From 766638d8228e24a78620c5a3f72b1227d2ea4f2a Mon Sep 17 00:00:00 2001 From: Keewis Date: Sat, 20 Jun 2020 00:00:39 +0200 Subject: [PATCH 12/50] always use the same reason when xfailing units in indexes tests --- xarray/tests/test_units.py | 33 ++++++++++++++++++++++----------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/xarray/tests/test_units.py b/xarray/tests/test_units.py index b4190cb77d7..c1f56117f5b 100644 --- a/xarray/tests/test_units.py +++ b/xarray/tests/test_units.py @@ -719,7 +719,9 @@ def test_broadcast_dataset(dtype): "variant", ( "data", - pytest.param("dims", marks=pytest.mark.xfail(reason="indexes strip units")), + pytest.param( + "dims", marks=pytest.mark.xfail(reason="indexes don't support units") + ), "coords", ), ) @@ -794,7 +796,9 @@ def test_combine_by_coords(variant, unit, error, dtype): "variant", ( "data", - pytest.param("dims", marks=pytest.mark.xfail(reason="indexes strip units")), + pytest.param( + "dims", marks=pytest.mark.xfail(reason="indexes don't support units") + ), "coords", ), ) @@ -898,7 +902,9 @@ def test_combine_nested(variant, unit, error, dtype): "variant", ( "data", - pytest.param("dims", marks=pytest.mark.xfail(reason="indexes strip units")), + pytest.param( + "dims", marks=pytest.mark.xfail(reason="indexes don't support units") + ), "coords", ), ) @@ -964,7 +970,9 @@ def test_concat_dataarray(variant, unit, error, dtype): "variant", ( "data", - pytest.param("dims", marks=pytest.mark.xfail(reason="indexes strip units")), + pytest.param( + "dims", marks=pytest.mark.xfail(reason="indexes don't support units") + ), "coords", ), ) @@ -1032,7 +1040,9 @@ def test_concat_dataset(variant, unit, error, dtype): "variant", ( "data", - pytest.param("dims", marks=pytest.mark.xfail(reason="indexes strip units")), + pytest.param( + "dims", marks=pytest.mark.xfail(reason="indexes don't support units") + ), "coords", ), ) @@ -1143,7 +1153,9 @@ def test_merge_dataarray(variant, unit, error, dtype): "variant", ( "data", - pytest.param("dims", marks=pytest.mark.xfail(reason="indexes strip units")), + pytest.param( + "dims", marks=pytest.mark.xfail(reason="indexes don't support units") + ), "coords", ), ) @@ -2277,7 +2289,7 @@ class TestDataArray: ( pytest.param( "with_dims", - marks=pytest.mark.xfail(reason="units in indexes are not supported"), + marks=pytest.mark.xfail(reason="indexes don't support units"), ), "with_coords", "without_coords", @@ -2314,7 +2326,7 @@ def test_init(self, variant, dtype): ( pytest.param( "with_dims", - marks=pytest.mark.xfail(reason="units in indexes are not supported"), + marks=pytest.mark.xfail(reason="indexes don't support units"), ), pytest.param("with_coords"), pytest.param("without_coords"), @@ -3869,8 +3881,7 @@ def test_init(self, shared, unit, error, dtype): ( "data", pytest.param( - "dims", - marks=pytest.mark.xfail(reason="units in indexes are not supported"), + "dims", marks=pytest.mark.xfail(reason="indexes don't support units"), ), "coords", ), @@ -4359,7 +4370,7 @@ def test_combine_first(self, variant, unit, error, dtype): ( "data", pytest.param( - "dims", marks=pytest.mark.xfail(reason="units in indexes not supported") + "dims", marks=pytest.mark.xfail(reason="indexes don't support units") ), "coords", ), From e5569a4f7c8e4424f4beb66a54289075e0611355 Mon Sep 17 00:00:00 2001 From: Keewis Date: Sat, 20 Jun 2020 00:20:34 +0200 Subject: [PATCH 13/50] also check that the replication functions work with dims and units --- xarray/tests/test_units.py | 74 ++++++++++++++++++++++++++++++-------- 1 file changed, 59 insertions(+), 15 deletions(-) diff --git a/xarray/tests/test_units.py b/xarray/tests/test_units.py index c1f56117f5b..446471eebc5 100644 --- a/xarray/tests/test_units.py +++ b/xarray/tests/test_units.py @@ -1228,35 +1228,79 @@ def test_merge_dataset(variant, unit, error, dtype): assert_allclose(expected, actual) +@pytest.mark.parametrize( + "variant", + ( + "data", + pytest.param( + "dims", marks=pytest.mark.xfail(reason="indexes don't support units") + ), + "coords", + ), +) @pytest.mark.parametrize("func", (xr.zeros_like, xr.ones_like)) -def test_replication_dataarray(func, dtype): - array = np.linspace(0, 10, 20).astype(dtype) * unit_registry.s - data_array = xr.DataArray(data=array, dims="x") +def test_replication_dataarray(func, variant, dtype): + unit = unit_registry.m - numpy_func = getattr(np, func.__name__) - units = extract_units(numpy_func(data_array)) - expected = attach_units(func(data_array), units) + variants = { + "data": (unit, 1, 1), + "dims": (1, unit, 1), + "coords": (1, 1, unit), + } + data_unit, dim_unit, coord_unit = variants.get(variant) + + array = np.linspace(0, 10, 20).astype(dtype) * data_unit + x = np.arange(20) * dim_unit + u = np.linspace(0, 1, 20) * coord_unit + + data_array = xr.DataArray(data=array, dims="x", coords={"x": x, "u": ("x", u)}) + units = extract_units(data_array) + units.pop(data_array.name) + + expected = attach_units(func(strip_units(data_array)), units) actual = func(data_array) assert_units_equal(expected, actual) assert_identical(expected, actual) +@pytest.mark.parametrize( + "variant", + ( + "data", + pytest.param( + "dims", marks=pytest.mark.xfail(reason="indexes don't support units") + ), + "coords", + ), +) @pytest.mark.parametrize("func", (xr.zeros_like, xr.ones_like)) -def test_replication_dataset(func, dtype): - array1 = np.linspace(0, 10, 20).astype(dtype) * unit_registry.s - array2 = np.linspace(5, 10, 10).astype(dtype) * unit_registry.Pa - x = np.arange(20).astype(dtype) * unit_registry.m - y = np.arange(10).astype(dtype) * unit_registry.m - z = y.to(unit_registry.mm) +def test_replication_dataset(func, variant, dtype): + unit = unit_registry.m + + variants = { + "data": ((unit_registry.m, unit_registry.Pa), 1, 1), + "dims": ((1, 1), unit, 1), + "coords": ((1, 1), 1, unit), + } + (data_unit1, data_unit2), dim_unit, coord_unit = variants.get(variant) + + array1 = np.linspace(0, 10, 20).astype(dtype) * data_unit1 + array2 = np.linspace(5, 10, 10).astype(dtype) * data_unit2 + x = np.arange(20).astype(dtype) * dim_unit + y = np.arange(10).astype(dtype) * dim_unit + u = np.linspace(0, 1, 10) * coord_unit ds = xr.Dataset( data_vars={"a": ("x", array1), "b": ("y", array2)}, - coords={"x": x, "y": y, "z": ("y", z)}, + coords={"x": x, "y": y, "u": ("y", u)}, ) + units = { + name: unit + for name, unit in extract_units(ds).items() + if name not in ds.data_vars + } - numpy_func = getattr(np, func.__name__) - units = extract_units(ds.map(numpy_func)) expected = attach_units(func(strip_units(ds)), units) actual = func(ds) From 4737d41076c25422562a49e43d911933060d063c Mon Sep 17 00:00:00 2001 From: Keewis Date: Sat, 20 Jun 2020 00:21:03 +0200 Subject: [PATCH 14/50] apply full_like to the data instead of the variable --- xarray/core/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xarray/core/common.py b/xarray/core/common.py index e343f342040..070847d4579 100644 --- a/xarray/core/common.py +++ b/xarray/core/common.py @@ -1434,7 +1434,7 @@ def _full_like_variable(other, fill_value, dtype: DTypeLike = None): other.shape, fill_value, dtype=dtype, chunks=other.data.chunks ) else: - data = np.full_like(other, fill_value, dtype=dtype) + data = np.full_like(other.data, fill_value, dtype=dtype) return Variable(dims=other.dims, data=data, attrs=other.attrs) From b080cb75409a54ea3f6deec7d0cbac8c33eaeb6a Mon Sep 17 00:00:00 2001 From: Keewis Date: Sat, 20 Jun 2020 01:06:26 +0200 Subject: [PATCH 15/50] check full_like with units in dims, data and coords separately --- xarray/tests/test_units.py | 100 +++++++++++++++++++------------------ 1 file changed, 51 insertions(+), 49 deletions(-) diff --git a/xarray/tests/test_units.py b/xarray/tests/test_units.py index 446471eebc5..d2f8f680220 100644 --- a/xarray/tests/test_units.py +++ b/xarray/tests/test_units.py @@ -1309,37 +1309,40 @@ def test_replication_dataset(func, variant, dtype): assert_identical(expected, actual) -@pytest.mark.xfail( - reason=( - "pint is undecided on how `full_like` should work, so incorrect errors " - "may be expected: hgrecco/pint#882" - ) -) @pytest.mark.parametrize( - "unit,error", + "variant", ( - pytest.param(1, DimensionalityError, id="no_unit"), + "data", pytest.param( - unit_registry.dimensionless, DimensionalityError, id="dimensionless" + "dims", marks=pytest.mark.xfail(reason="indexes don't support units") + ), + pytest.param( + "coords", + marks=pytest.mark.xfail(reason="can't copy quantity into non-quantity"), ), - pytest.param(unit_registry.m, DimensionalityError, id="incompatible_unit"), - pytest.param(unit_registry.ms, None, id="compatible_unit"), - pytest.param(unit_registry.s, None, id="identical_unit"), ), - ids=repr, ) -def test_replication_full_like_dataarray(unit, error, dtype): - array = np.linspace(0, 5, 10) * unit_registry.s - data_array = xr.DataArray(data=array, dims="x") +def test_replication_full_like_dataarray(variant, dtype): + # since full_like will strip units and then use the units of the + # fill value, we don't need to try multiple units + unit = unit_registry.m - fill_value = -1 * unit - if error is not None: - with pytest.raises(error): - xr.full_like(data_array, fill_value=fill_value) + variants = { + "data": (unit, 1, 1), + "dims": (1, unit, 1), + "coords": (1, 1, unit), + } + data_unit, dim_unit, coord_unit = variants.get(variant) - return + array = np.linspace(0, 5, 10) * data_unit + x = np.arange(10) * dim_unit + u = np.linspace(0, 1, 10) * coord_unit + data_array = xr.DataArray(data=array, dims="x", coords={"x": x, "u": ("x", u)}) - units = {**extract_units(data_array), **{None: unit if unit != 1 else None}} + fill_value = -1 * unit_registry.degK + + units = extract_units(data_array) + units[data_array.name] = fill_value.units expected = attach_units( xr.full_like(strip_units(data_array), fill_value=strip_units(fill_value)), units ) @@ -1349,47 +1352,46 @@ def test_replication_full_like_dataarray(unit, error, dtype): assert_identical(expected, actual) -@pytest.mark.xfail( - reason=( - "pint is undecided on how `full_like` should work, so incorrect errors " - "may be expected: hgrecco/pint#882" - ) -) @pytest.mark.parametrize( - "unit,error", + "variant", ( - pytest.param(1, DimensionalityError, id="no_unit"), + "data", pytest.param( - unit_registry.dimensionless, DimensionalityError, id="dimensionless" + "dims", marks=pytest.mark.xfail(reason="indexes don't support units") + ), + pytest.param( + "coords", + marks=pytest.mark.xfail(reason="can't copy quantity into non-quantity"), ), - pytest.param(unit_registry.m, DimensionalityError, id="incompatible_unit"), - pytest.param(unit_registry.ms, None, id="compatible_unit"), - pytest.param(unit_registry.s, None, id="identical_unit"), ), - ids=repr, ) -def test_replication_full_like_dataset(unit, error, dtype): - array1 = np.linspace(0, 10, 20).astype(dtype) * unit_registry.s - array2 = np.linspace(5, 10, 10).astype(dtype) * unit_registry.Pa - x = np.arange(20).astype(dtype) * unit_registry.m - y = np.arange(10).astype(dtype) * unit_registry.m - z = y.to(unit_registry.mm) +def test_replication_full_like_dataset(variant, dtype): + unit = unit_registry.m + + variants = { + "data": ((unit_registry.s, unit_registry.Pa), 1, 1), + "dims": ((1, 1), unit, 1), + "coords": ((1, 1), 1, unit), + } + (data_unit1, data_unit2), dim_unit, coord_unit = variants.get(variant) + + array1 = np.linspace(0, 10, 20).astype(dtype) * data_unit1 + array2 = np.linspace(5, 10, 10).astype(dtype) * data_unit2 + x = np.arange(20).astype(dtype) * dim_unit + y = np.arange(10).astype(dtype) * dim_unit + + u = np.linspace(0, 1, 10) * coord_unit ds = xr.Dataset( data_vars={"a": ("x", array1), "b": ("y", array2)}, - coords={"x": x, "y": y, "z": ("y", z)}, + coords={"x": x, "y": y, "u": ("y", u)}, ) - fill_value = -1 * unit - if error is not None: - with pytest.raises(error): - xr.full_like(ds, fill_value=fill_value) - - return + fill_value = -1 * unit_registry.degK units = { **extract_units(ds), - **{name: unit if unit != 1 else None for name in ds.data_vars}, + **{name: unit_registry.degK for name in ds.data_vars}, } expected = attach_units( xr.full_like(strip_units(ds), fill_value=strip_units(fill_value)), units From b35bb6c824193add3166d06a77d3a7eb4177da19 Mon Sep 17 00:00:00 2001 From: Keewis Date: Sat, 20 Jun 2020 01:21:57 +0200 Subject: [PATCH 16/50] clearly separate the test variants of the merge tests --- xarray/tests/test_units.py | 133 ++++++++++++++++++------------------- 1 file changed, 63 insertions(+), 70 deletions(-) diff --git a/xarray/tests/test_units.py b/xarray/tests/test_units.py index d2f8f680220..aceb098772b 100644 --- a/xarray/tests/test_units.py +++ b/xarray/tests/test_units.py @@ -1050,29 +1050,33 @@ def test_merge_dataarray(variant, unit, error, dtype): original_unit = unit_registry.m variants = { - "data": (unit, original_unit, original_unit), - "dims": (original_unit, unit, original_unit), - "coords": (original_unit, original_unit, unit), + "data": ((original_unit, unit), (1, 1), (1, 1)), + "dims": ((1, 1), (original_unit, unit), (1, 1)), + "coords": ((1, 1), (1, 1), (original_unit, unit)), } - data_unit, dim_unit, coord_unit = variants.get(variant) + ( + (data_unit1, data_unit2), + (dim_unit1, dim_unit2), + (coord_unit1, coord_unit2), + ) = variants.get(variant) - array1 = np.linspace(0, 1, 2 * 3).reshape(2, 3).astype(dtype) * original_unit - x1 = np.arange(2) * original_unit - y1 = np.arange(3) * original_unit - u1 = np.linspace(10, 20, 2) * original_unit - v1 = np.linspace(10, 20, 3) * original_unit + array1 = np.linspace(0, 1, 2 * 3).reshape(2, 3).astype(dtype) * data_unit1 + x1 = np.arange(2) * dim_unit1 + y1 = np.arange(3) * dim_unit1 + u1 = np.linspace(10, 20, 2) * coord_unit1 + v1 = np.linspace(10, 20, 3) * coord_unit1 - array2 = np.linspace(1, 2, 2 * 4).reshape(2, 4).astype(dtype) * data_unit - x2 = np.arange(2, 4) * dim_unit - z2 = np.arange(4) * original_unit - u2 = np.linspace(20, 30, 2) * coord_unit - w2 = np.linspace(10, 20, 4) * original_unit + array2 = np.linspace(1, 2, 2 * 4).reshape(2, 4).astype(dtype) * data_unit2 + x2 = np.arange(2, 4) * dim_unit2 + z2 = np.arange(4) * dim_unit1 + u2 = np.linspace(20, 30, 2) * coord_unit2 + w2 = np.linspace(10, 20, 4) * coord_unit1 - array3 = np.linspace(0, 2, 3 * 4).reshape(3, 4).astype(dtype) * data_unit - y3 = np.arange(3, 6) * dim_unit - z3 = np.arange(4, 8) * dim_unit - v3 = np.linspace(10, 20, 3) * coord_unit - w3 = np.linspace(10, 20, 4) * coord_unit + array3 = np.linspace(0, 2, 3 * 4).reshape(3, 4).astype(dtype) * data_unit2 + y3 = np.arange(3, 6) * dim_unit2 + z3 = np.arange(4, 8) * dim_unit2 + v3 = np.linspace(10, 20, 3) * coord_unit2 + w3 = np.linspace(10, 20, 4) * coord_unit2 arr1 = xr.DataArray( name="a", @@ -1099,31 +1103,22 @@ def test_merge_dataarray(variant, unit, error, dtype): return - units = {name: original_unit for name in list("axyzuvw")} - - convert_and_strip = lambda arr: strip_units(convert_units(arr, units)) - expected_units = { - "a": original_unit, - "u": original_unit, - "v": original_unit, - "w": original_unit, - "x": original_unit, - "y": original_unit, - "z": original_unit, + units = { + "a": data_unit1, + "u": coord_unit1, + "v": coord_unit1, + "w": coord_unit1, + "x": dim_unit1, + "y": dim_unit1, + "z": dim_unit1, } + convert_and_strip = lambda arr: strip_units(convert_units(arr, units)) - expected = convert_units( - attach_units( - xr.merge( - [ - convert_and_strip(arr1), - convert_and_strip(arr2), - convert_and_strip(arr3), - ] - ), - units, + expected = attach_units( + xr.merge( + [convert_and_strip(arr1), convert_and_strip(arr2), convert_and_strip(arr3)] ), - expected_units, + units, ) actual = xr.merge([arr1, arr2, arr3]) @@ -1163,43 +1158,47 @@ def test_merge_dataset(variant, unit, error, dtype): original_unit = unit_registry.m variants = { - "data": (unit, original_unit, original_unit), - "dims": (original_unit, unit, original_unit), - "coords": (original_unit, original_unit, unit), + "data": ((original_unit, unit), (1, 1), (1, 1)), + "dims": ((1, 1), (original_unit, unit), (1, 1)), + "coords": ((1, 1), (1, 1), (original_unit, unit)), } - data_unit, dim_unit, coord_unit = variants.get(variant) + ( + (data_unit1, data_unit2), + (dim_unit1, dim_unit2), + (coord_unit1, coord_unit2), + ) = variants.get(variant) - array1 = np.zeros(shape=(2, 3), dtype=dtype) * original_unit - array2 = np.zeros(shape=(2, 3), dtype=dtype) * original_unit + array1 = np.zeros(shape=(2, 3), dtype=dtype) * data_unit1 + array2 = np.zeros(shape=(2, 3), dtype=dtype) * data_unit1 - x = np.arange(11, 14) * original_unit - y = np.arange(2) * original_unit - z = np.arange(3) * original_unit + x = np.arange(11, 14) * dim_unit1 + y = np.arange(2) * dim_unit1 + u = np.arange(3) * coord_unit1 ds1 = xr.Dataset( data_vars={"a": (("y", "x"), array1), "b": (("y", "x"), array2)}, - coords={"x": x, "y": y, "u": ("x", z)}, + coords={"x": x, "y": y, "u": ("x", u)}, ) ds2 = xr.Dataset( data_vars={ - "a": (("y", "x"), np.ones_like(array1) * data_unit), - "b": (("y", "x"), np.ones_like(array2) * data_unit), + "a": (("y", "x"), np.ones_like(array1) * data_unit2), + "b": (("y", "x"), np.ones_like(array2) * data_unit2), }, coords={ - "x": np.arange(3) * dim_unit, - "y": np.arange(2, 4) * dim_unit, - "u": ("x", np.arange(-3, 0) * coord_unit), + "x": np.arange(3) * dim_unit2, + "y": np.arange(2, 4) * dim_unit2, + "u": ("x", np.arange(-3, 0) * coord_unit2), }, ) ds3 = xr.Dataset( data_vars={ - "a": (("y", "x"), np.full_like(array1, np.nan) * data_unit), - "b": (("y", "x"), np.full_like(array2, np.nan) * data_unit), + "a": (("y", "x"), np.full_like(array1, np.nan) * data_unit2), + "b": (("y", "x"), np.full_like(array2, np.nan) * data_unit2), }, coords={ - "x": np.arange(3, 6) * dim_unit, - "y": np.arange(4, 6) * dim_unit, - "u": ("x", np.arange(3, 6) * coord_unit), + "x": np.arange(3, 6) * dim_unit2, + "y": np.arange(4, 6) * dim_unit2, + "u": ("x", np.arange(3, 6) * coord_unit2), }, ) @@ -1212,15 +1211,9 @@ def test_merge_dataset(variant, unit, error, dtype): units = extract_units(ds1) convert_and_strip = lambda ds: strip_units(convert_units(ds, units)) - expected_units = {name: original_unit for name in list("abxyzu")} - expected = convert_units( - attach_units( - func( - [convert_and_strip(ds1), convert_and_strip(ds2), convert_and_strip(ds3)] - ), - units, - ), - expected_units, + expected = attach_units( + func([convert_and_strip(ds1), convert_and_strip(ds2), convert_and_strip(ds3)]), + units, ) actual = func([ds1, ds2, ds3]) From a7540564a1ae90825665dd451d50046790ea654f Mon Sep 17 00:00:00 2001 From: Keewis Date: Sat, 20 Jun 2020 01:24:22 +0200 Subject: [PATCH 17/50] don't use indexes for the dataset where tests --- xarray/tests/test_units.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/xarray/tests/test_units.py b/xarray/tests/test_units.py index aceb098772b..f9e091d4f1d 100644 --- a/xarray/tests/test_units.py +++ b/xarray/tests/test_units.py @@ -1455,10 +1455,9 @@ def test_where_dataarray(fill_value, unit, error, dtype): def test_where_dataset(fill_value, unit, error, dtype): array1 = np.linspace(0, 5, 10).astype(dtype) * unit_registry.m array2 = np.linspace(-5, 0, 10).astype(dtype) * unit_registry.m - x = np.arange(10) * unit_registry.s - ds = xr.Dataset(data_vars={"a": ("x", array1), "b": ("x", array2)}, coords={"x": x}) - cond = x < 5 * unit_registry.s + ds = xr.Dataset(data_vars={"a": ("x", array1), "b": ("x", array2)}) + cond = array1 < 2 * unit_registry.m fill_value = fill_value * unit if error is not None and not ( From 08cf7be15a8a8495e96920f9a4bcebb9e5d1b3a5 Mon Sep 17 00:00:00 2001 From: Keewis Date: Sat, 20 Jun 2020 23:58:39 +0200 Subject: [PATCH 18/50] replace numpy.testing.assert_allclose with assert numpy.allclose --- xarray/tests/test_units.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/xarray/tests/test_units.py b/xarray/tests/test_units.py index f9e091d4f1d..c5ea1463ffe 100644 --- a/xarray/tests/test_units.py +++ b/xarray/tests/test_units.py @@ -1728,7 +1728,8 @@ def test_raw_numpy_methods(self, func, unit, error, dtype): actual = func(variable, *args, **kwargs) assert_units_equal(expected, actual) - np.testing.assert_allclose(expected, actual) + # can't use `np.testing.assert_allclose` since that casts to numpy.array + assert np.allclose(expected, actual) @pytest.mark.parametrize( "func", (method("isnull"), method("notnull"), method("count")), ids=repr From daceb130e17d7d13253a2f904937ed8bceed958c Mon Sep 17 00:00:00 2001 From: Keewis Date: Sun, 21 Jun 2020 13:01:17 +0200 Subject: [PATCH 19/50] remove a conditional xfail that depends on a very old pint version --- xarray/tests/test_units.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/xarray/tests/test_units.py b/xarray/tests/test_units.py index c5ea1463ffe..6fdf949a05f 100644 --- a/xarray/tests/test_units.py +++ b/xarray/tests/test_units.py @@ -2278,15 +2278,7 @@ def test_pad_constant_values(self, dtype, xr_arg, np_arg): @pytest.mark.parametrize( "unit,error", ( - pytest.param( - 1, - DimensionalityError, - id="no_unit", - marks=pytest.mark.xfail( - LooseVersion(pint.__version__) < LooseVersion("0.10.2"), - reason="bug in pint's implementation of np.pad", - ), - ), + pytest.param(1, DimensionalityError, id="no_unit"), pytest.param( unit_registry.dimensionless, DimensionalityError, id="dimensionless" ), From 8fdee006b3a653e852f65fcf070aa9337ab2db4f Mon Sep 17 00:00:00 2001 From: Keewis Date: Mon, 22 Jun 2020 11:59:35 +0200 Subject: [PATCH 20/50] use assert_identical from the local namespace --- xarray/tests/test_units.py | 104 ++++++++++++++++++------------------- 1 file changed, 52 insertions(+), 52 deletions(-) diff --git a/xarray/tests/test_units.py b/xarray/tests/test_units.py index 6fdf949a05f..ca2d423462b 100644 --- a/xarray/tests/test_units.py +++ b/xarray/tests/test_units.py @@ -9,8 +9,8 @@ import xarray as xr from xarray.core import dtypes from xarray.core.npcompat import IS_NEP18_ACTIVE -from xarray.testing import assert_allclose, assert_equal, assert_identical +from . import assert_allclose, assert_equal, assert_identical from .test_variable import _PAD_XR_NP_ARGS, VariableSubclassobjects pint = pytest.importorskip("pint") @@ -1594,7 +1594,7 @@ def test_aggregation(self, func, dtype): actual = func(variable) assert_units_equal(expected, actual) - xr.testing.assert_identical(expected, actual) + assert_identical(expected, actual) # TODO: remove once pint==0.12 has been released @pytest.mark.xfail( @@ -1752,7 +1752,7 @@ def test_missing_value_detection(self, func): actual = func(variable) assert_units_equal(expected, actual) - xr.testing.assert_identical(expected, actual) + assert_identical(expected, actual) @pytest.mark.parametrize( "unit,error", @@ -1798,7 +1798,7 @@ def test_missing_value_fillna(self, unit, error): actual = variable.fillna(value=fill_value) assert_units_equal(expected, actual) - xr.testing.assert_identical(expected, actual) + assert_identical(expected, actual) @pytest.mark.parametrize( "unit", @@ -1909,7 +1909,7 @@ def test_isel(self, indices, dtype): actual = variable.isel(x=indices) assert_units_equal(expected, actual) - xr.testing.assert_identical(expected, actual) + assert_identical(expected, actual) # TODO: remove once pint==0.12 has been released @pytest.mark.xfail( @@ -2020,7 +2020,7 @@ def test_masking(self, func, unit, error, dtype): actual = func(variable, cond, other) assert_units_equal(expected, actual) - xr.testing.assert_identical(expected, actual) + assert_identical(expected, actual) def test_squeeze(self, dtype): shape = (2, 1, 3, 1, 1, 2) @@ -2034,7 +2034,7 @@ def test_squeeze(self, dtype): actual = variable.squeeze() assert_units_equal(expected, actual) - xr.testing.assert_identical(expected, actual) + assert_identical(expected, actual) names = tuple(name for name, size in zip(names, shape) if shape == 1) for name in names: @@ -2044,7 +2044,7 @@ def test_squeeze(self, dtype): actual = variable.squeeze(dim=name) assert_units_equal(expected, actual) - xr.testing.assert_identical(expected, actual) + assert_identical(expected, actual) @pytest.mark.parametrize( "func", @@ -2083,7 +2083,7 @@ def test_computation(self, func, dtype): actual = func(variable) assert_units_equal(expected, actual) - xr.testing.assert_identical(expected, actual) + assert_identical(expected, actual) @pytest.mark.parametrize( "unit,error", @@ -2129,7 +2129,7 @@ def test_stack(self, dtype): actual = variable.stack(z=("x", "y")) assert_units_equal(expected, actual) - xr.testing.assert_identical(expected, actual) + assert_identical(expected, actual) def test_unstack(self, dtype): array = np.linspace(0, 5, 3 * 10).astype(dtype) * unit_registry.m @@ -2141,7 +2141,7 @@ def test_unstack(self, dtype): actual = variable.unstack(z={"x": 3, "y": 10}) assert_units_equal(expected, actual) - xr.testing.assert_identical(expected, actual) + assert_identical(expected, actual) @pytest.mark.parametrize( "unit,error", @@ -2181,7 +2181,7 @@ def test_concat(self, unit, error, dtype): actual = xr.Variable.concat([variable, other], dim="y") assert_units_equal(expected, actual) - xr.testing.assert_identical(expected, actual) + assert_identical(expected, actual) def test_set_dims(self, dtype): array = np.linspace(0, 5, 3 * 10).reshape(3, 10).astype(dtype) * unit_registry.m @@ -2194,7 +2194,7 @@ def test_set_dims(self, dtype): actual = variable.set_dims(dims) assert_units_equal(expected, actual) - xr.testing.assert_identical(expected, actual) + assert_identical(expected, actual) def test_copy(self, dtype): array = np.linspace(0, 5, 10).astype(dtype) * unit_registry.m @@ -2207,7 +2207,7 @@ def test_copy(self, dtype): actual = variable.copy(data=other) assert_units_equal(expected, actual) - xr.testing.assert_identical(expected, actual) + assert_identical(expected, actual) @pytest.mark.parametrize( "unit", @@ -2260,7 +2260,7 @@ def test_pad_constant_values(self, dtype, xr_arg, np_arg): v.data.astype(float), np_arg, mode="constant", constant_values=np.nan, ), ) - xr.testing.assert_identical(expected, actual) + assert_identical(expected, actual) assert_units_equal(expected, actual) assert isinstance(actual._data, type(v._data)) @@ -2272,7 +2272,7 @@ def test_pad_constant_values(self, dtype, xr_arg, np_arg): v.dims, np.pad(v.data, np_arg, mode="constant", constant_values=v.data.flat[0]), ) - xr.testing.assert_identical(actual, expected) + assert_identical(actual, expected) assert_units_equal(expected, actual) @pytest.mark.parametrize( @@ -2311,7 +2311,7 @@ def test_pad_unit_constant_value(self, unit, error, dtype): actual = func(variable, constant_values=fill_value) assert_units_equal(expected, actual) - xr.testing.assert_identical(expected, actual) + assert_identical(expected, actual) class TestDataArray: @@ -2462,7 +2462,7 @@ def test_unary_operations(self, func, dtype): actual = func(data_array) assert_units_equal(expected, actual) - xr.testing.assert_identical(expected, actual) + assert_identical(expected, actual) @pytest.mark.parametrize( "func", @@ -2482,7 +2482,7 @@ def test_binary_operations(self, func, dtype): actual = func(data_array) assert_units_equal(expected, actual) - xr.testing.assert_identical(expected, actual) + assert_identical(expected, actual) @pytest.mark.parametrize( "comparison", @@ -2533,7 +2533,7 @@ def test_comparison_operations(self, comparison, unit, error, dtype): ) assert_units_equal(expected, actual) - xr.testing.assert_identical(expected, actual) + assert_identical(expected, actual) @pytest.mark.parametrize( "units,error", @@ -2562,7 +2562,7 @@ def test_univariate_ufunc(self, units, error, dtype): actual = func(data_array) assert_units_equal(expected, actual) - xr.testing.assert_identical(expected, actual) + assert_identical(expected, actual) @pytest.mark.xfail(reason="needs the type register system for __array_ufunc__") @pytest.mark.parametrize( @@ -2604,11 +2604,11 @@ def test_bivariate_ufunc(self, unit, error, dtype): actual = np.maximum(data_array, 1 * unit) assert_units_equal(expected, actual) - xr.testing.assert_identical(expected, actual) + assert_identical(expected, actual) actual = np.maximum(1 * unit, data_array) assert_units_equal(expected, actual) - xr.testing.assert_identical(expected, actual) + assert_identical(expected, actual) @pytest.mark.parametrize("property", ("T", "imag", "real")) def test_numpy_properties(self, property, dtype): @@ -2625,7 +2625,7 @@ def test_numpy_properties(self, property, dtype): actual = getattr(data_array, property) assert_units_equal(expected, actual) - xr.testing.assert_identical(expected, actual) + assert_identical(expected, actual) @pytest.mark.parametrize( "func", @@ -2641,7 +2641,7 @@ def test_numpy_methods(self, func, dtype): actual = func(data_array) assert_units_equal(expected, actual) - xr.testing.assert_identical(expected, actual) + assert_identical(expected, actual) def test_item(self, dtype): array = np.arange(10).astype(dtype) * unit_registry.m @@ -2767,7 +2767,7 @@ def test_numpy_methods_with_args(self, func, unit, error, dtype): actual = func(data_array, *args, **kwargs) assert_units_equal(expected, actual) - xr.testing.assert_identical(expected, actual) + assert_identical(expected, actual) @pytest.mark.parametrize( "func", (method("isnull"), method("notnull"), method("count")), ids=repr @@ -2790,7 +2790,7 @@ def test_missing_value_detection(self, func, dtype): actual = func(data_array) assert_units_equal(expected, actual) - xr.testing.assert_identical(expected, actual) + assert_identical(expected, actual) @pytest.mark.xfail(reason="ffill and bfill lose units in data") @pytest.mark.parametrize("func", (method("ffill"), method("bfill")), ids=repr) @@ -2808,7 +2808,7 @@ def test_missing_value_filling(self, func, dtype): actual = func(data_array, dim="x") assert_units_equal(expected, actual) - xr.testing.assert_identical(expected, actual) + assert_identical(expected, actual) @pytest.mark.parametrize( "unit,error", @@ -2857,7 +2857,7 @@ def test_fillna(self, fill_value, unit, error, dtype): actual = func(data_array, value=value) assert_units_equal(expected, actual) - xr.testing.assert_identical(expected, actual) + assert_identical(expected, actual) def test_dropna(self, dtype): array = ( @@ -2872,7 +2872,7 @@ def test_dropna(self, dtype): actual = data_array.dropna(dim="x") assert_units_equal(expected, actual) - xr.testing.assert_identical(expected, actual) + assert_identical(expected, actual) @pytest.mark.parametrize( "unit", @@ -2901,7 +2901,7 @@ def test_isin(self, unit, dtype): actual = data_array.isin(values) assert_units_equal(expected, actual) - xr.testing.assert_identical(expected, actual) + assert_identical(expected, actual) @pytest.mark.parametrize( "variant", ("masking", "replacing_scalar", "replacing_array", "dropping") @@ -2955,7 +2955,7 @@ def test_where(self, variant, unit, error, dtype): actual = data_array.where(**kwargs) assert_units_equal(expected, actual) - xr.testing.assert_identical(expected, actual) + assert_identical(expected, actual) @pytest.mark.xfail(reason="uses numpy.vectorize") def test_interpolate_na(self): @@ -2971,7 +2971,7 @@ def test_interpolate_na(self): actual = data_array.interpolate_na(dim="x") assert_units_equal(expected, actual) - xr.testing.assert_identical(expected, actual) + assert_identical(expected, actual) @pytest.mark.parametrize( "unit,error", @@ -3012,7 +3012,7 @@ def test_combine_first(self, unit, error, dtype): actual = data_array.combine_first(other) assert_units_equal(expected, actual) - xr.testing.assert_identical(expected, actual) + assert_identical(expected, actual) @pytest.mark.parametrize( "unit", @@ -3119,7 +3119,7 @@ def test_broadcast_like(self, unit, dtype): actual = arr1.broadcast_like(arr2) assert_units_equal(expected, actual) - xr.testing.assert_identical(expected, actual) + assert_identical(expected, actual) @pytest.mark.parametrize( "unit", @@ -3194,7 +3194,7 @@ def test_content_manipulation(self, func, dtype): actual = func(data_array) assert_units_equal(expected, actual) - xr.testing.assert_identical(expected, actual) + assert_identical(expected, actual) @pytest.mark.parametrize( "func", (pytest.param(method("copy", data=np.arange(20))),), ids=repr @@ -3222,7 +3222,7 @@ def test_content_manipulation_with_units(self, func, unit, dtype): actual = func(data_array, **kwargs) assert_units_equal(expected, actual) - xr.testing.assert_identical(expected, actual) + assert_identical(expected, actual) @pytest.mark.parametrize( "indices", @@ -3243,7 +3243,7 @@ def test_isel(self, indices, dtype): actual = data_array.isel(x=indices) assert_units_equal(expected, actual) - xr.testing.assert_identical(expected, actual) + assert_identical(expected, actual) @pytest.mark.xfail(reason="indexes don't support units") @pytest.mark.parametrize( @@ -3288,7 +3288,7 @@ def test_sel(self, raw_values, unit, error, dtype): actual = data_array.sel(x=values) assert_units_equal(expected, actual) - xr.testing.assert_identical(expected, actual) + assert_identical(expected, actual) @pytest.mark.xfail(reason="indexes don't support units") @pytest.mark.parametrize( @@ -3333,7 +3333,7 @@ def test_loc(self, raw_values, unit, error, dtype): actual = data_array.loc[{"x": values}] assert_units_equal(expected, actual) - xr.testing.assert_identical(expected, actual) + assert_identical(expected, actual) @pytest.mark.xfail(reason="indexes don't support units") @pytest.mark.parametrize( @@ -3378,7 +3378,7 @@ def test_drop_sel(self, raw_values, unit, error, dtype): actual = data_array.drop_sel(x=values) assert_units_equal(expected, actual) - xr.testing.assert_identical(expected, actual) + assert_identical(expected, actual) @pytest.mark.parametrize( "shape", @@ -3408,7 +3408,7 @@ def test_squeeze(self, shape, dtype): actual = data_array.squeeze() assert_units_equal(expected, actual) - xr.testing.assert_identical(expected, actual) + assert_identical(expected, actual) # try squeezing the dimensions separately names = tuple(dim for dim, coord in coords.items() if len(coord) == 1) @@ -3419,7 +3419,7 @@ def test_squeeze(self, shape, dtype): actual = data_array.squeeze(dim=name) assert_units_equal(expected, actual) - xr.testing.assert_identical(expected, actual) + assert_identical(expected, actual) @pytest.mark.parametrize( "func", @@ -3442,7 +3442,7 @@ def test_head_tail_thin(self, func, dtype): actual = func(data_array) assert_units_equal(expected, actual) - xr.testing.assert_identical(expected, actual) + assert_identical(expected, actual) # TODO: remove once pint==0.12 has been released @pytest.mark.xfail( @@ -3519,7 +3519,7 @@ def test_interp_reindex_indexing(self, func, unit, error, dtype): actual = func(data_array, x=new_x) assert_units_equal(expected, actual) - xr.testing.assert_identical(expected, actual) + assert_identical(expected, actual) # TODO: remove once pint==0.12 has been released @pytest.mark.xfail( @@ -3599,7 +3599,7 @@ def test_interp_reindex_like_indexing(self, func, unit, error, dtype): actual = func(data_array, other) assert_units_equal(expected, actual) - xr.testing.assert_identical(expected, actual) + assert_identical(expected, actual) @pytest.mark.parametrize( "func", @@ -3622,7 +3622,7 @@ def test_stacking_stacked(self, func, dtype): actual = func(stacked) assert_units_equal(expected, actual) - xr.testing.assert_identical(expected, actual) + assert_identical(expected, actual) @pytest.mark.xfail(reason="indexes don't support units") def test_to_unstacked_dataset(self, dtype): @@ -3646,7 +3646,7 @@ def test_to_unstacked_dataset(self, dtype): actual = func(data_array) assert_units_equal(expected, actual) - xr.testing.assert_identical(expected, actual) + assert_identical(expected, actual) @pytest.mark.parametrize( "func", @@ -3681,7 +3681,7 @@ def test_stacking_reordering(self, func, dtype): actual = func(data_array) assert_units_equal(expected, actual) - xr.testing.assert_identical(expected, actual) + assert_identical(expected, actual) @pytest.mark.parametrize( "func", @@ -3725,7 +3725,7 @@ def test_computation(self, func, dtype): actual = func(data_array) assert_units_equal(expected, actual) - xr.testing.assert_identical(expected, actual) + assert_identical(expected, actual) # TODO: remove once pint==0.12 has been released @pytest.mark.xfail( @@ -3782,7 +3782,7 @@ def test_resample(self, dtype): actual = func(data_array).mean() assert_units_equal(expected, actual) - xr.testing.assert_identical(expected, actual) + assert_identical(expected, actual) @pytest.mark.parametrize( "func", @@ -3825,7 +3825,7 @@ def test_grouped_operations(self, func, dtype): actual = func(data_array.groupby("y")) assert_units_equal(expected, actual) - xr.testing.assert_identical(expected, actual) + assert_identical(expected, actual) class TestDataset: From 72ad5b04e816f4234025e8c210a6908c3da089be Mon Sep 17 00:00:00 2001 From: Keewis Date: Mon, 22 Jun 2020 12:00:26 +0200 Subject: [PATCH 21/50] properly separate between the broadcast_like test variants --- xarray/tests/test_units.py | 48 +++++++++++++++++++++++++++++++------- 1 file changed, 39 insertions(+), 9 deletions(-) diff --git a/xarray/tests/test_units.py b/xarray/tests/test_units.py index ca2d423462b..5a5fc83fce9 100644 --- a/xarray/tests/test_units.py +++ b/xarray/tests/test_units.py @@ -3101,17 +3101,47 @@ def is_compatible(a, b): pytest.param(unit_registry.m, id="identical_unit"), ), ) - def test_broadcast_like(self, unit, dtype): - array1 = np.linspace(1, 2, 2 * 1).reshape(2, 1).astype(dtype) * unit_registry.Pa - array2 = np.linspace(0, 1, 2 * 3).reshape(2, 3).astype(dtype) * unit_registry.Pa + @pytest.mark.parametrize( + "variant", + ( + "data", + pytest.param( + "dims", marks=pytest.mark.xfail(reason="indexes don't support units") + ), + "coords", + ), + ) + def test_broadcast_like(self, variant, unit, dtype): + original_unit = unit_registry.m + + variants = { + "data": ((original_unit, unit), (1, 1), (1, 1)), + "dims": ((1, 1), (original_unit, unit), (1, 1)), + "coords": ((1, 1), (1, 1), (original_unit, unit)), + } + ( + (data_unit1, data_unit2), + (dim_unit1, dim_unit2), + (coord_unit1, coord_unit2), + ) = variants.get(variant) - x1 = np.arange(2) * unit_registry.m - x2 = np.arange(2) * unit - y1 = np.array([0]) * unit_registry.m - y2 = np.arange(3) * unit + array1 = np.linspace(1, 2, 2 * 1).reshape(2, 1).astype(dtype) * data_unit1 + array2 = np.linspace(0, 1, 2 * 3).reshape(2, 3).astype(dtype) * data_unit2 - arr1 = xr.DataArray(data=array1, coords={"x": x1, "y": y1}, dims=("x", "y")) - arr2 = xr.DataArray(data=array2, coords={"x": x2, "y": y2}, dims=("x", "y")) + x1 = np.arange(2) * dim_unit1 + x2 = np.arange(2) * dim_unit2 + y1 = np.array([0]) * dim_unit1 + y2 = np.arange(3) * dim_unit2 + + u1 = np.linspace(0, 1, 2) * coord_unit1 + u2 = np.linspace(0, 1, 2) * coord_unit2 + + arr1 = xr.DataArray( + data=array1, coords={"x": x1, "y": y1, "u": ("x", u1)}, dims=("x", "y") + ) + arr2 = xr.DataArray( + data=array2, coords={"x": x2, "y": y2, "u": ("x", u2)}, dims=("x", "y") + ) expected = attach_units( strip_units(arr1).broadcast_like(strip_units(arr2)), extract_units(arr1) From c9d1c188068750a9fa069680e65c4682d4511c5f Mon Sep 17 00:00:00 2001 From: Keewis Date: Mon, 22 Jun 2020 12:18:09 +0200 Subject: [PATCH 22/50] don't accept "data" as an alias of the DataArray's data --- xarray/tests/test_units.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/xarray/tests/test_units.py b/xarray/tests/test_units.py index 5a5fc83fce9..d2a606a0bf3 100644 --- a/xarray/tests/test_units.py +++ b/xarray/tests/test_units.py @@ -181,12 +181,7 @@ def attach_units(obj, units): new_obj = xr.Dataset(data_vars=data_vars, coords=coords, attrs=obj.attrs) elif isinstance(obj, xr.DataArray): # try the array name, "data" and None, then fall back to dimensionless - data_units = ( - units.get(obj.name, None) - or units.get("data", None) - or units.get(None, None) - or 1 - ) + data_units = units.get(obj.name, None) or units.get(None, None) or 1 data = array_attach_units(obj.data, data_units) From e59bd5966469a234977b64daf507d2eac1882cdc Mon Sep 17 00:00:00 2001 From: Keewis Date: Mon, 22 Jun 2020 12:19:17 +0200 Subject: [PATCH 23/50] properly separate between the variants of the content manipulation tests --- xarray/tests/test_units.py | 62 ++++++++++++++++++++++++++------------ 1 file changed, 43 insertions(+), 19 deletions(-) diff --git a/xarray/tests/test_units.py b/xarray/tests/test_units.py index d2a606a0bf3..5423726efeb 100644 --- a/xarray/tests/test_units.py +++ b/xarray/tests/test_units.py @@ -3174,46 +3174,70 @@ def test_broadcast_equals(self, unit, dtype): assert expected == actual + @pytest.mark.parametrize( + "variant", + ( + "data", + pytest.param( + "dims", marks=pytest.mark.xfail(reason="indexes don't support units") + ), + "coords", + ), + ) @pytest.mark.parametrize( "func", ( method("pipe", lambda da: da * 10), - method("assign_coords", y2=("y", np.arange(10) * unit_registry.mm)), + method("assign_coords", w=("y", np.arange(10) * unit_registry.mm)), method("assign_attrs", attr1="value"), - method("rename", x2="x_mm"), - method("swap_dims", {"x": "x2"}), - method( - "expand_dims", - dim={"z": np.linspace(10, 20, 12) * unit_registry.s}, - axis=1, + method("rename", u="v"), + pytest.param( + method("swap_dims", {"x": "u"}), + marks=pytest.mark.xfail(reason="indexes don't support units"), + ), + pytest.param( + method( + "expand_dims", + dim={"z": np.linspace(10, 20, 12) * unit_registry.s}, + axis=1, + ), + marks=pytest.mark.xfail(reason="indexes don't support units"), ), method("drop_vars", "x"), - method("reset_coords", names="x2"), + method("reset_coords", names="u"), method("copy"), method("astype", np.float32), ), ids=repr, ) - def test_content_manipulation(self, func, dtype): - quantity = ( - np.linspace(0, 10, 5 * 10).reshape(5, 10).astype(dtype) - * unit_registry.pascal - ) - x = np.arange(quantity.shape[0]) * unit_registry.m - y = np.arange(quantity.shape[1]) * unit_registry.m - x2 = x.to(unit_registry.mm) + def test_content_manipulation(self, func, variant, dtype): + unit = unit_registry.m + + variants = { + "data": (unit, 1, 1), + "dims": (1, unit, 1), + "coords": (1, 1, unit), + } + data_unit, dim_unit, coord_unit = variants.get(variant) + + quantity = np.linspace(0, 10, 5 * 10).reshape(5, 10).astype(dtype) * data_unit + x = np.arange(quantity.shape[0]) * dim_unit + y = np.arange(quantity.shape[1]) * dim_unit + u = np.linspace(0, 1, quantity.shape[0]) * coord_unit data_array = xr.DataArray( - name="data", + name="a", data=quantity, - coords={"x": x, "x2": ("x", x2), "y": y}, + coords={"x": x, "u": ("x", u), "y": y}, dims=("x", "y"), ) stripped_kwargs = { key: array_strip_units(value) for key, value in func.kwargs.items() } - units = {**{"x_mm": x2.units, "x2": x2.units}, **extract_units(data_array)} + units = extract_units(data_array) + units["u"] = getattr(u, "units", None) + units["v"] = getattr(u, "units", None) expected = attach_units(func(strip_units(data_array), **stripped_kwargs), units) actual = func(data_array) From d91922d480544cb5409bd447ac5035ae727c91ca Mon Sep 17 00:00:00 2001 From: Keewis Date: Mon, 22 Jun 2020 12:21:05 +0200 Subject: [PATCH 24/50] use assert np.allclose(...) instead of np.testing.assert_allclose(...) --- xarray/tests/test_units.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/xarray/tests/test_units.py b/xarray/tests/test_units.py index 5423726efeb..adcd53895f1 100644 --- a/xarray/tests/test_units.py +++ b/xarray/tests/test_units.py @@ -2647,7 +2647,9 @@ def test_item(self, dtype): expected = func(strip_units(data_array)) * unit_registry.m actual = func(data_array) - np.testing.assert_allclose(expected, actual) + # TODO: use something like np.testing.assert_allclose, but + # with duckarray support + assert np.allclose(expected, actual) @pytest.mark.parametrize( "unit,error", From b4f7bb2a04b3b0b3a3c3cc57398dbb929de3810d Mon Sep 17 00:00:00 2001 From: Keewis Date: Mon, 22 Jun 2020 12:22:59 +0200 Subject: [PATCH 25/50] don't test units in indexes in the isel tests --- xarray/tests/test_units.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/xarray/tests/test_units.py b/xarray/tests/test_units.py index adcd53895f1..0836e07f93a 100644 --- a/xarray/tests/test_units.py +++ b/xarray/tests/test_units.py @@ -3283,10 +3283,10 @@ def test_content_manipulation_with_units(self, func, unit, dtype): ), ) def test_isel(self, indices, dtype): + # TODO: maybe test for units in indexes? array = np.arange(10).astype(dtype) * unit_registry.s - x = np.arange(len(array)) * unit_registry.m - data_array = xr.DataArray(data=array, coords={"x": x}, dims="x") + data_array = xr.DataArray(data=array, dims="x") expected = attach_units( strip_units(data_array).isel(x=indices), extract_units(data_array) From db44dd561a8ef95d0beb82050e5124e4b7ac8608 Mon Sep 17 00:00:00 2001 From: Keewis Date: Mon, 22 Jun 2020 12:24:04 +0200 Subject: [PATCH 26/50] don't use units in indexes for the head / tail / thin tests --- xarray/tests/test_units.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/xarray/tests/test_units.py b/xarray/tests/test_units.py index 0836e07f93a..21ea749cc8b 100644 --- a/xarray/tests/test_units.py +++ b/xarray/tests/test_units.py @@ -3478,14 +3478,10 @@ def test_squeeze(self, shape, dtype): ids=repr, ) def test_head_tail_thin(self, func, dtype): + # TODO: works like isel. Maybe also test units in indexes? array = np.linspace(1, 2, 10 * 5).reshape(10, 5) * unit_registry.degK - coords = { - "x": np.arange(10) * unit_registry.m, - "y": np.arange(5) * unit_registry.m, - } - - data_array = xr.DataArray(data=array, coords=coords, dims=("x", "y")) + data_array = xr.DataArray(data=array, dims=("x", "y")) expected = attach_units( func(strip_units(data_array)), extract_units(data_array) From cef0dda0b048b169bf9c7127b9babb7bcb50b084 Mon Sep 17 00:00:00 2001 From: Keewis Date: Mon, 22 Jun 2020 12:38:42 +0200 Subject: [PATCH 27/50] properly separate the variants of more tests --- xarray/tests/test_units.py | 117 +++++++++++++++++++++++++++---------- 1 file changed, 87 insertions(+), 30 deletions(-) diff --git a/xarray/tests/test_units.py b/xarray/tests/test_units.py index 21ea749cc8b..43bcea53de2 100644 --- a/xarray/tests/test_units.py +++ b/xarray/tests/test_units.py @@ -3730,6 +3730,16 @@ def test_stacking_reordering(self, func, dtype): assert_units_equal(expected, actual) assert_identical(expected, actual) + @pytest.mark.parametrize( + "variant", + ( + "data", + pytest.param( + "dims", marks=pytest.mark.xfail(reason="indexes don't support units") + ), + "coords", + ), + ) @pytest.mark.parametrize( "func", ( @@ -3748,25 +3758,31 @@ def test_stacking_reordering(self, func, dtype): ), ids=repr, ) - def test_computation(self, func, dtype): - array = ( - np.linspace(0, 10, 5 * 10).reshape(5, 10).astype(dtype) * unit_registry.m - ) + def test_computation(self, func, variant, dtype): + unit = unit_registry.m - x = np.arange(array.shape[0]) * unit_registry.m - y = np.arange(array.shape[1]) * unit_registry.s + variants = { + "data": (unit, 1, 1), + "dims": (1, unit, 1), + "coords": (1, 1, unit), + } + data_unit, dim_unit, coord_unit = variants.get(variant) + + array = np.linspace(0, 10, 5 * 10).reshape(5, 10).astype(dtype) * data_unit - data_array = xr.DataArray(data=array, coords={"x": x, "y": y}, dims=("x", "y")) + x = np.arange(array.shape[0]) * dim_unit + y = np.arange(array.shape[1]) * dim_unit + + u = np.linspace(0, 1, array.shape[0]) * coord_unit + + data_array = xr.DataArray( + data=array, coords={"x": x, "y": y, "u": ("x", u)}, dims=("x", "y") + ) # we want to make sure the output unit is correct - units = { - **extract_units(data_array), - **( - {} - if isinstance(func, (function, method)) - else extract_units(func(array.reshape(-1))) - ), - } + units = extract_units(data_array) + if not isinstance(func, (function, method)): + units.update(extract_units(func(array.reshape(-1)))) expected = attach_units(func(strip_units(data_array)), units) actual = func(data_array) @@ -3778,6 +3794,16 @@ def test_computation(self, func, dtype): @pytest.mark.xfail( LooseVersion(pint.__version__) <= "0.12", reason="pint bug in isclose" ) + @pytest.mark.parametrize( + "variant", + ( + "data", + pytest.param( + "dims", marks=pytest.mark.xfail(reason="indexes don't support units") + ), + "coords", + ), + ) @pytest.mark.parametrize( "func", ( @@ -3799,15 +3825,26 @@ def test_computation(self, func, dtype): ), ids=repr, ) - def test_computation_objects(self, func, dtype): - array = ( - np.linspace(0, 10, 5 * 10).reshape(5, 10).astype(dtype) * unit_registry.m - ) + def test_computation_objects(self, func, variant, dtype): + unit = unit_registry.m - x = np.array([0, 0, 1, 2, 2]) * unit_registry.m - y = np.arange(array.shape[1]) * 3 * unit_registry.s + variants = { + "data": (unit, 1, 1), + "dims": (1, unit, 1), + "coords": (1, 1, unit), + } + data_unit, dim_unit, coord_unit = variants.get(variant) - data_array = xr.DataArray(data=array, coords={"x": x, "y": y}, dims=("x", "y")) + array = np.linspace(0, 10, 5 * 10).reshape(5, 10).astype(dtype) * data_unit + + x = np.array([0, 0, 1, 2, 2]) * dim_unit + y = np.arange(array.shape[1]) * 3 * dim_unit + + u = np.linspace(0, 1, 5) * coord_unit + + data_array = xr.DataArray( + data=array, coords={"x": x, "y": y, "u": ("x", u)}, dims=("x", "y") + ) units = extract_units(data_array) expected = attach_units(func(strip_units(data_array)).mean(), units) @@ -3831,10 +3868,20 @@ def test_resample(self, dtype): assert_units_equal(expected, actual) assert_identical(expected, actual) + @pytest.mark.parametrize( + "variant", + ( + "data", + pytest.param( + "dims", marks=pytest.mark.xfail(reason="indexes don't support units") + ), + "coords", + ), + ) @pytest.mark.parametrize( "func", ( - method("assign_coords", z=(["x"], np.arange(5) * unit_registry.s)), + method("assign_coords", z=("x", np.arange(5) * unit_registry.s)), method("first"), method("last"), pytest.param( @@ -3847,15 +3894,25 @@ def test_resample(self, dtype): ), ids=repr, ) - def test_grouped_operations(self, func, dtype): - array = ( - np.linspace(0, 10, 5 * 10).reshape(5, 10).astype(dtype) * unit_registry.m - ) + def test_grouped_operations(self, func, variant, dtype): + unit = unit_registry.m - x = np.arange(array.shape[0]) * unit_registry.m - y = np.arange(array.shape[1]) * 3 * unit_registry.s + variants = { + "data": (unit, 1, 1), + "dims": (1, unit, 1), + "coords": (1, 1, unit), + } + data_unit, dim_unit, coord_unit = variants.get(variant) + array = np.linspace(0, 10, 5 * 10).reshape(5, 10).astype(dtype) * data_unit + + x = np.arange(array.shape[0]) * dim_unit + y = np.arange(array.shape[1]) * 3 * dim_unit + + u = np.linspace(0, 1, array.shape[0]) * coord_unit - data_array = xr.DataArray(data=array, coords={"x": x, "y": y}, dims=("x", "y")) + data_array = xr.DataArray( + data=array, coords={"x": x, "y": y, "u": ("x", u)}, dims=("x", "y") + ) units = {**extract_units(data_array), **{"z": unit_registry.s, "q": None}} stripped_kwargs = { From bdf7145ca4cf6d0195f099a3944a9373f783656c Mon Sep 17 00:00:00 2001 From: Keewis Date: Mon, 22 Jun 2020 12:45:26 +0200 Subject: [PATCH 28/50] rewrite the squeeze tests --- xarray/tests/test_units.py | 50 ++++++++++++-------------------------- 1 file changed, 15 insertions(+), 35 deletions(-) diff --git a/xarray/tests/test_units.py b/xarray/tests/test_units.py index 43bcea53de2..d30a9e080d7 100644 --- a/xarray/tests/test_units.py +++ b/xarray/tests/test_units.py @@ -2017,30 +2017,23 @@ def test_masking(self, func, unit, error, dtype): assert_units_equal(expected, actual) assert_identical(expected, actual) - def test_squeeze(self, dtype): + @pytest.mark.parametrize("dim", ("x", "y", "z", "t", "all")) + def test_squeeze(self, dim, dtype): shape = (2, 1, 3, 1, 1, 2) names = list("abcdef") + dim_lengths = dict(zip(names, shape)) array = np.ones(shape=shape) * unit_registry.m variable = xr.Variable(names, array) + kwargs = {"dim": dim} if dim != "all" and dim_lengths.get(dim, 0) == 1 else {} expected = attach_units( - strip_units(variable).squeeze(), extract_units(variable) + strip_units(variable).squeeze(**kwargs), extract_units(variable) ) - actual = variable.squeeze() + actual = variable.squeeze(**kwargs) assert_units_equal(expected, actual) assert_identical(expected, actual) - names = tuple(name for name, size in zip(names, shape) if shape == 1) - for name in names: - expected = attach_units( - strip_units(variable).squeeze(dim=name), extract_units(variable) - ) - actual = variable.squeeze(dim=name) - - assert_units_equal(expected, actual) - assert_identical(expected, actual) - @pytest.mark.parametrize( "func", ( @@ -3431,6 +3424,7 @@ def test_drop_sel(self, raw_values, unit, error, dtype): assert_units_equal(expected, actual) assert_identical(expected, actual) + @pytest.mark.parametrize("dim", ("x", "y", "z", "t", "all")) @pytest.mark.parametrize( "shape", ( @@ -3441,37 +3435,23 @@ def test_drop_sel(self, raw_values, unit, error, dtype): pytest.param((1, 10, 1, 20), id="first_and_last_dimension_squeezable"), ), ) - def test_squeeze(self, shape, dtype): + def test_squeeze(self, shape, dim, dtype): + names = "xyzt" + dim_lengths = dict(zip(names, shape)) names = "xyzt" - coords = { - name: np.arange(length).astype(dtype) - * (unit_registry.m if name != "t" else unit_registry.s) - for name, length in zip(names, shape) - } array = np.arange(10 * 20).astype(dtype).reshape(shape) * unit_registry.J - data_array = xr.DataArray( - data=array, coords=coords, dims=tuple(names[: len(shape)]) - ) + data_array = xr.DataArray(data=array, dims=tuple(names[: len(shape)])) + + kwargs = {"dim": dim} if dim != "all" and dim_lengths.get(dim, 0) == 1 else {} expected = attach_units( - strip_units(data_array).squeeze(), extract_units(data_array) + strip_units(data_array).squeeze(**kwargs), extract_units(data_array) ) - actual = data_array.squeeze() + actual = data_array.squeeze(**kwargs) assert_units_equal(expected, actual) assert_identical(expected, actual) - # try squeezing the dimensions separately - names = tuple(dim for dim, coord in coords.items() if len(coord) == 1) - for index, name in enumerate(names): - expected = attach_units( - strip_units(data_array).squeeze(dim=name), extract_units(data_array) - ) - actual = data_array.squeeze(dim=name) - - assert_units_equal(expected, actual) - assert_identical(expected, actual) - @pytest.mark.parametrize( "func", (method("head", x=7, y=3), method("tail", x=7, y=3), method("thin", x=7, y=3)), From dbd5512b849e223a8afd1d261d04c78df6e2a538 Mon Sep 17 00:00:00 2001 From: Keewis Date: Mon, 22 Jun 2020 12:52:49 +0200 Subject: [PATCH 29/50] use assert_allclose from the module's namespace --- xarray/tests/test_units.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/xarray/tests/test_units.py b/xarray/tests/test_units.py index d30a9e080d7..5fe15902367 100644 --- a/xarray/tests/test_units.py +++ b/xarray/tests/test_units.py @@ -1601,7 +1601,7 @@ def test_aggregate_complex(self): actual = variable.mean() assert_units_equal(expected, actual) - xr.testing.assert_allclose(expected, actual) + assert_allclose(expected, actual) # TODO: remove once pint==0.12 has been released @pytest.mark.xfail( @@ -1663,7 +1663,7 @@ def test_numpy_methods(self, func, unit, error, dtype): actual = func(variable, *args, **kwargs) assert_units_equal(expected, actual) - xr.testing.assert_allclose(expected, actual) + assert_allclose(expected, actual) @pytest.mark.parametrize( "func", (method("item", 5), method("searchsorted", 5)), ids=repr @@ -1966,7 +1966,7 @@ def test_1d_math(self, func, unit, error, dtype): actual = func(variable, y) assert_units_equal(expected, actual) - xr.testing.assert_allclose(expected, actual) + assert_allclose(expected, actual) @pytest.mark.parametrize( "unit,error", @@ -2431,7 +2431,7 @@ def test_aggregation(self, func, dtype): actual = func(data_array) assert_units_equal(expected, actual) - xr.testing.assert_allclose(expected, actual) + assert_allclose(expected, actual) @pytest.mark.parametrize( "func", @@ -3505,7 +3505,7 @@ def test_interp_reindex(self, variant, func, dtype): actual = func(data_array, x=new_x) assert_units_equal(expected, actual) - xr.testing.assert_allclose(expected, actual) + assert_allclose(expected, actual) @pytest.mark.xfail(reason="indexes don't support units") @pytest.mark.parametrize( @@ -3583,7 +3583,7 @@ def test_interp_reindex_like(self, variant, func, dtype): actual = func(data_array, other) assert_units_equal(expected, actual) - xr.testing.assert_allclose(expected, actual) + assert_allclose(expected, actual) @pytest.mark.xfail(reason="indexes don't support units") @pytest.mark.parametrize( @@ -3831,7 +3831,7 @@ def test_computation_objects(self, func, variant, dtype): actual = func(data_array).mean() assert_units_equal(expected, actual) - xr.testing.assert_allclose(expected, actual) + assert_allclose(expected, actual) def test_resample(self, dtype): array = np.linspace(0, 5, 10).astype(dtype) * unit_registry.m From a1539fffdfd654e37398993f30a12474d402ba1d Mon Sep 17 00:00:00 2001 From: Keewis Date: Mon, 22 Jun 2020 12:53:12 +0200 Subject: [PATCH 30/50] rewrite the copy tests --- xarray/tests/test_units.py | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/xarray/tests/test_units.py b/xarray/tests/test_units.py index 5fe15902367..3b13e79a552 100644 --- a/xarray/tests/test_units.py +++ b/xarray/tests/test_units.py @@ -3240,9 +3240,6 @@ def test_content_manipulation(self, func, variant, dtype): assert_units_equal(expected, actual) assert_identical(expected, actual) - @pytest.mark.parametrize( - "func", (pytest.param(method("copy", data=np.arange(20))),), ids=repr - ) @pytest.mark.parametrize( "unit", ( @@ -3251,19 +3248,17 @@ def test_content_manipulation(self, func, variant, dtype): pytest.param(unit_registry.degK, id="with_unit"), ), ) - def test_content_manipulation_with_units(self, func, unit, dtype): + def test_copy(self, unit, dtype): quantity = np.linspace(0, 10, 20, dtype=dtype) * unit_registry.pascal - x = np.arange(len(quantity)) * unit_registry.m - - data_array = xr.DataArray(data=quantity, coords={"x": x}, dims="x") + new_data = np.arange(20) - kwargs = {key: value * unit for key, value in func.kwargs.items()} + data_array = xr.DataArray(data=quantity, dims="x") expected = attach_units( - func(strip_units(data_array)), {None: unit, "x": x.units} + strip_units(data_array).copy(data=new_data), {None: unit} ) - actual = func(data_array, **kwargs) + actual = data_array.copy(data=new_data * unit) assert_units_equal(expected, actual) assert_identical(expected, actual) From a1f3f726004c03390ddeab0ae88e62874ca6dad3 Mon Sep 17 00:00:00 2001 From: Keewis Date: Mon, 22 Jun 2020 13:19:59 +0200 Subject: [PATCH 31/50] xfail the equal comparison for a pint version lower than 0.14 --- xarray/tests/test_units.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/xarray/tests/test_units.py b/xarray/tests/test_units.py index 3b13e79a552..eeb3c179de4 100644 --- a/xarray/tests/test_units.py +++ b/xarray/tests/test_units.py @@ -2477,7 +2477,14 @@ def test_binary_operations(self, func, dtype): ( pytest.param(operator.lt, id="less_than"), pytest.param(operator.ge, id="greater_equal"), - pytest.param(operator.eq, id="equal"), + pytest.param( + operator.eq, + id="equal", + marks=pytest.mark.xfail( + LooseVersion(pint.__version__) < "0.14", + reason="inconsistencies in the return values of pint's eq", + ), + ), ), ) @pytest.mark.parametrize( From 80f49d60f7419f0c00a972f5d81e066464721b06 Mon Sep 17 00:00:00 2001 From: Keewis Date: Tue, 23 Jun 2020 01:59:55 +0200 Subject: [PATCH 32/50] try to implement a duckarray friendly assert_array_equal --- xarray/testing.py | 51 ++++++++++++++++++++++++++++++++++++ xarray/tests/test_testing.py | 30 +++++++++++++++++++++ 2 files changed, 81 insertions(+) diff --git a/xarray/testing.py b/xarray/testing.py index 9681503414e..0314c9190d0 100644 --- a/xarray/testing.py +++ b/xarray/testing.py @@ -148,6 +148,57 @@ def compat_variable(a, b): raise TypeError("{} not supported by assertion comparison".format(type(a))) +def assert_duckarray_equal(x, y, err_msg="", verbose=True): + """ Like `np.testing.assert_array_equal`, but for duckarrays """ + __tracebackhide__ = True + + def array_like(x): + return hasattr(x, "ndim") and hasattr(x, "shape") and hasattr(x, "dtype") + + def scalar(x): + return isinstance(x, (bool, int, float, complex, str)) or ( + array_like(x) and x.ndim == 0 + ) + + def format_message(x, y, err_msg, verbose): + diff = x - y + abs_diff = float(max(abs(diff))) + rel_diff = "not implemented" + + n_diff = int(np.count_nonzero(diff)) + n_total = diff.size + + fraction = f"{n_diff} / {n_total}" + percentage = float(n_diff / n_total * 100) + + parts = [ + "Arrays are not equal", + err_msg, + f"Mismatched elements: {fraction} ({percentage:.0f}%)", + f"Max absolute difference: {abs_diff}", + f"Max relative difference: {rel_diff}", + ] + if verbose: + parts += [ + f" x: {x!r}", + f" y: {y!r}", + ] + + return "\n".join(parts) + + if not scalar(x) and not array_like(x): + x = np.asarray(x) + + if not scalar(y) and not array_like(y): + y = np.asarray(y) + + if (array_like(x) and scalar(y)) or (scalar(x) and array_like(y)): + equiv = (x == y).all() + else: + equiv = duck_array_ops.array_equiv(x, y) + assert equiv, format_message(x, y, err_msg=err_msg, verbose=verbose) + + def assert_chunks_equal(a, b): """ Assert that chunksizes along chunked dimensions are equal. diff --git a/xarray/tests/test_testing.py b/xarray/tests/test_testing.py index f4961af58e9..99c98988b54 100644 --- a/xarray/tests/test_testing.py +++ b/xarray/tests/test_testing.py @@ -1,7 +1,16 @@ +import numpy as np import pytest import xarray as xr +from . import requires_dask + +try: + import dask.array as da +except ImportError: + da = object() + da.Array = lambda x: x + def test_allclose_regression(): x = xr.DataArray(1.01) @@ -30,3 +39,24 @@ def test_allclose_regression(): def test_assert_allclose(obj1, obj2): with pytest.raises(AssertionError): xr.testing.assert_allclose(obj1, obj2) + + +@requires_dask +@pytest.mark.parametrize( + "duckarray", + (pytest.param(np.array, id="numpy"), pytest.param(da.from_array, id="dask")), +) +@pytest.mark.parametrize( + ["obj1", "obj2"], + ( + pytest.param([1e-10, 2], [0.0, 2.0], id="both arrays"), + pytest.param([1e-17, 2], 0.0, id="second scalar"), + pytest.param(0.0, [1e-17, 2], id="first scalar"), + ), +) +def test_assert_duckarray_equal(duckarray, obj1, obj2): + # TODO: actually check the repr + a = duckarray(obj1) + b = duckarray(obj2) + with pytest.raises(AssertionError): + xr.testing.assert_duckarray_equal(a, b) From 32da9b9672b490b855033f50247c64123ac6c642 Mon Sep 17 00:00:00 2001 From: Keewis Date: Tue, 23 Jun 2020 15:45:28 +0200 Subject: [PATCH 33/50] add tests for not raising an assertion error --- xarray/tests/test_testing.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/xarray/tests/test_testing.py b/xarray/tests/test_testing.py index 99c98988b54..7b870420e84 100644 --- a/xarray/tests/test_testing.py +++ b/xarray/tests/test_testing.py @@ -54,9 +54,28 @@ def test_assert_allclose(obj1, obj2): pytest.param(0.0, [1e-17, 2], id="first scalar"), ), ) -def test_assert_duckarray_equal(duckarray, obj1, obj2): +def test_assert_duckarray_equal_failing(duckarray, obj1, obj2): # TODO: actually check the repr a = duckarray(obj1) b = duckarray(obj2) with pytest.raises(AssertionError): xr.testing.assert_duckarray_equal(a, b) + + +@pytest.mark.parametrize( + "duckarray", + (pytest.param(np.array, id="numpy"), pytest.param(da.from_array, id="dask")), +) +@pytest.mark.parametrize( + ["obj1", "obj2"], + ( + pytest.param([0, 2], [0.0, 2.0], id="both arrays"), + pytest.param([0, 0], 0.0, id="second scalar"), + pytest.param(0.0, [0, 0], id="first scalar"), + ), +) +def test_assert_duckarray_equal(duckarray, obj1, obj2): + a = duckarray(obj1) + b = duckarray(obj2) + + xr.testing.assert_duckarray_equal(a, b) From ea14ef385a3bb8988bccc548fd09aab63ca5cb43 Mon Sep 17 00:00:00 2001 From: Keewis Date: Tue, 23 Jun 2020 15:48:29 +0200 Subject: [PATCH 34/50] skip only the dask test if it isn't installed --- xarray/tests/test_testing.py | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/xarray/tests/test_testing.py b/xarray/tests/test_testing.py index 7b870420e84..e82d338c6f1 100644 --- a/xarray/tests/test_testing.py +++ b/xarray/tests/test_testing.py @@ -3,7 +3,7 @@ import xarray as xr -from . import requires_dask +from . import has_dask try: import dask.array as da @@ -41,10 +41,16 @@ def test_assert_allclose(obj1, obj2): xr.testing.assert_allclose(obj1, obj2) -@requires_dask @pytest.mark.parametrize( "duckarray", - (pytest.param(np.array, id="numpy"), pytest.param(da.from_array, id="dask")), + ( + pytest.param(np.array, id="numpy"), + pytest.param( + da.from_array, + id="dask", + marks=pytest.mark.skipif(not has_dask, reason="requires dask"), + ), + ), ) @pytest.mark.parametrize( ["obj1", "obj2"], @@ -64,7 +70,14 @@ def test_assert_duckarray_equal_failing(duckarray, obj1, obj2): @pytest.mark.parametrize( "duckarray", - (pytest.param(np.array, id="numpy"), pytest.param(da.from_array, id="dask")), + ( + pytest.param(np.array, id="numpy"), + pytest.param( + da.from_array, + id="dask", + marks=pytest.mark.skipif(not has_dask, reason="requires dask"), + ), + ), ) @pytest.mark.parametrize( ["obj1", "obj2"], From 21d2e8e2b63be3396a49bac6f4a84ce81898dcd4 Mon Sep 17 00:00:00 2001 From: Keewis Date: Tue, 23 Jun 2020 16:02:38 +0200 Subject: [PATCH 35/50] also check using pint if available --- xarray/tests/test_testing.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/xarray/tests/test_testing.py b/xarray/tests/test_testing.py index e82d338c6f1..f4f47174e00 100644 --- a/xarray/tests/test_testing.py +++ b/xarray/tests/test_testing.py @@ -11,6 +11,22 @@ da = object() da.Array = lambda x: x +try: + import pint + + unit_registry = pint.UnitRegistry(force_ndarray_like=True) + + def quantity(x): + return unit_registry.Quantity(x, "m") + + has_pint = True +except ImportError: + + def quantity(x): + return x + + has_pint = False + def test_allclose_regression(): x = xr.DataArray(1.01) @@ -41,6 +57,7 @@ def test_assert_allclose(obj1, obj2): xr.testing.assert_allclose(obj1, obj2) +@pytest.mark.filterwarnings("error") @pytest.mark.parametrize( "duckarray", ( @@ -50,6 +67,11 @@ def test_assert_allclose(obj1, obj2): id="dask", marks=pytest.mark.skipif(not has_dask, reason="requires dask"), ), + pytest.param( + quantity, + id="pint", + marks=pytest.mark.skipif(not has_pint, reason="requires pint"), + ), ), ) @pytest.mark.parametrize( @@ -68,6 +90,7 @@ def test_assert_duckarray_equal_failing(duckarray, obj1, obj2): xr.testing.assert_duckarray_equal(a, b) +@pytest.mark.filterwarnings("error") @pytest.mark.parametrize( "duckarray", ( @@ -77,6 +100,11 @@ def test_assert_duckarray_equal_failing(duckarray, obj1, obj2): id="dask", marks=pytest.mark.skipif(not has_dask, reason="requires dask"), ), + pytest.param( + quantity, + id="pint", + marks=pytest.mark.skipif(not has_pint, reason="requires pint"), + ), ), ) @pytest.mark.parametrize( From 7a597b36355efd010662c4e63fbc4259132b8630 Mon Sep 17 00:00:00 2001 From: Keewis Date: Tue, 23 Jun 2020 16:03:19 +0200 Subject: [PATCH 36/50] add a duckarray version of np.testing.assert_allclose --- xarray/testing.py | 69 +++++++++++++++++++++++++++-------------------- 1 file changed, 40 insertions(+), 29 deletions(-) diff --git a/xarray/testing.py b/xarray/testing.py index 0314c9190d0..1a2dd4af923 100644 --- a/xarray/testing.py +++ b/xarray/testing.py @@ -148,6 +148,43 @@ def compat_variable(a, b): raise TypeError("{} not supported by assertion comparison".format(type(a))) +def _format_message(x, y, err_msg, verbose): + diff = x - y + abs_diff = max(abs(diff)) + rel_diff = "not implemented" + + n_diff = int(np.count_nonzero(diff)) + n_total = diff.size + + fraction = f"{n_diff} / {n_total}" + percentage = float(n_diff / n_total * 100) + + parts = [ + "Arrays are not equal", + err_msg, + f"Mismatched elements: {fraction} ({percentage:.0f}%)", + f"Max absolute difference: {abs_diff}", + f"Max relative difference: {rel_diff}", + ] + if verbose: + parts += [ + f" x: {x!r}", + f" y: {y!r}", + ] + + return "\n".join(parts) + + +def assert_duckarray_allclose( + actual, desired, rtol=1e-07, atol=0, err_msg="", verbose=True +): + """ Like `np.testing.assert_allclose`, but for duckarrays. """ + __tracebackhide__ = True + + allclose = duck_array_ops.allclose_or_equiv(actual, desired, rtol=rtol, atol=atol) + assert allclose, _format_message(actual, desired, err_msg=err_msg, verbose=verbose) + + def assert_duckarray_equal(x, y, err_msg="", verbose=True): """ Like `np.testing.assert_array_equal`, but for duckarrays """ __tracebackhide__ = True @@ -160,43 +197,17 @@ def scalar(x): array_like(x) and x.ndim == 0 ) - def format_message(x, y, err_msg, verbose): - diff = x - y - abs_diff = float(max(abs(diff))) - rel_diff = "not implemented" - - n_diff = int(np.count_nonzero(diff)) - n_total = diff.size - - fraction = f"{n_diff} / {n_total}" - percentage = float(n_diff / n_total * 100) - - parts = [ - "Arrays are not equal", - err_msg, - f"Mismatched elements: {fraction} ({percentage:.0f}%)", - f"Max absolute difference: {abs_diff}", - f"Max relative difference: {rel_diff}", - ] - if verbose: - parts += [ - f" x: {x!r}", - f" y: {y!r}", - ] - - return "\n".join(parts) - - if not scalar(x) and not array_like(x): + if not array_like(x) and not scalar(x): x = np.asarray(x) - if not scalar(y) and not array_like(y): + if not array_like(y) and not scalar(y): y = np.asarray(y) if (array_like(x) and scalar(y)) or (scalar(x) and array_like(y)): equiv = (x == y).all() else: equiv = duck_array_ops.array_equiv(x, y) - assert equiv, format_message(x, y, err_msg=err_msg, verbose=verbose) + assert equiv, _format_message(x, y, err_msg=err_msg, verbose=verbose) def assert_chunks_equal(a, b): From cc4fff02fa78422c6df893bc7bb6c120b3435474 Mon Sep 17 00:00:00 2001 From: Keewis Date: Tue, 23 Jun 2020 16:03:45 +0200 Subject: [PATCH 37/50] add both to __all__ --- xarray/testing.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/xarray/testing.py b/xarray/testing.py index 1a2dd4af923..472f8d60e31 100644 --- a/xarray/testing.py +++ b/xarray/testing.py @@ -11,7 +11,14 @@ from xarray.core.indexes import default_indexes from xarray.core.variable import IndexVariable, Variable -__all__ = ("assert_allclose", "assert_chunks_equal", "assert_equal", "assert_identical") +__all__ = ( + "assert_allclose", + "assert_chunks_equal", + "assert_duckarray_equal", + "assert_duckarray_allclose", + "assert_equal", + "assert_identical", +) def _decode_string_data(data): From 7320d43e413989b258be49307bc6f37d2761dff2 Mon Sep 17 00:00:00 2001 From: Keewis Date: Tue, 23 Jun 2020 16:07:13 +0200 Subject: [PATCH 38/50] make both available in xarray.tests --- xarray/tests/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/xarray/tests/__init__.py b/xarray/tests/__init__.py index 40c5cfa267c..9021c4e7dbc 100644 --- a/xarray/tests/__init__.py +++ b/xarray/tests/__init__.py @@ -16,6 +16,10 @@ from xarray.core.duck_array_ops import allclose_or_equiv # noqa: F401 from xarray.core.indexing import ExplicitlyIndexed from xarray.core.options import set_options +from xarray.testing import ( # noqa: F401 + assert_duckarray_allclose, + assert_duckarray_equal, +) # import mpl and change the backend before other mpl imports try: From c9c0c8716ba9b387383f4c9b29848ba5eaf81f79 Mon Sep 17 00:00:00 2001 From: Keewis Date: Thu, 25 Jun 2020 16:58:50 +0200 Subject: [PATCH 39/50] don't inherit from VariableSubtests since that was not written to test duck arrays. --- xarray/tests/test_units.py | 91 +++++++++++++++----------------------- 1 file changed, 35 insertions(+), 56 deletions(-) diff --git a/xarray/tests/test_units.py b/xarray/tests/test_units.py index eeb3c179de4..895823939a1 100644 --- a/xarray/tests/test_units.py +++ b/xarray/tests/test_units.py @@ -11,7 +11,7 @@ from xarray.core.npcompat import IS_NEP18_ACTIVE from . import assert_allclose, assert_equal, assert_identical -from .test_variable import _PAD_XR_NP_ARGS, VariableSubclassobjects +from .test_variable import _PAD_XR_NP_ARGS pint = pytest.importorskip("pint") DimensionalityError = pint.errors.DimensionalityError @@ -1499,61 +1499,7 @@ def test_dot_dataarray(dtype): assert_identical(expected, actual) -def delete_attrs(*to_delete): - def wrapper(cls): - for item in to_delete: - setattr(cls, item, None) - - return cls - - return wrapper - - -@delete_attrs( - "test_getitem_with_mask", - "test_getitem_with_mask_nd_indexer", - "test_index_0d_string", - "test_index_0d_datetime", - "test_index_0d_timedelta64", - "test_0d_time_data", - "test_index_0d_not_a_time", - "test_datetime64_conversion", - "test_timedelta64_conversion", - "test_pandas_period_index", - "test_1d_reduce", - "test_array_interface", - "test___array__", - "test_copy_index", - "test_concat_number_strings", - "test_concat_fixed_len_str", - "test_concat_mixed_dtypes", - "test_pandas_datetime64_with_tz", - "test_pandas_data", - "test_multiindex", -) -class TestVariable(VariableSubclassobjects): - @staticmethod - def cls(dims, data, *args, **kwargs): - return xr.Variable( - dims, unit_registry.Quantity(data, unit_registry.m), *args, **kwargs - ) - - def example_1d_objects(self): - for data in [ - range(3), - 0.5 * np.arange(3), - 0.5 * np.arange(3, dtype=np.float32), - np.array(["a", "b", "c"], dtype=object), - ]: - yield (self.cls("x", data), data) - - # TODO: remove once pint==0.12 has been released - @pytest.mark.xfail( - LooseVersion(pint.__version__) <= "0.12", reason="pint bug in isclose" - ) - def test_real_and_imag(self): - super().test_real_and_imag() - +class TestVariable: @pytest.mark.parametrize( "func", ( @@ -2236,6 +2182,39 @@ def test_no_conflicts(self, unit, dtype): assert expected == actual + @pytest.mark.parametrize( + "mode", + [ + "mean", + "median", + "reflect", + "edge", + pytest.param( + "linear_ramp", + marks=pytest.mark.xfail( + reason="pint bug: https://github.com/hgrecco/pint/issues/1026" + ), + ), + "maximum", + "minimum", + "symmetric", + "wrap", + ], + ) + @pytest.mark.parametrize("xr_arg, np_arg", _PAD_XR_NP_ARGS) + def test_pad(self, mode, xr_arg, np_arg): + data = np.arange(4 * 3 * 2).reshape(4, 3, 2) * unit_registry.m + v = xr.Variable(["x", "y", "z"], data) + + expected = attach_units( + strip_units(v).pad(mode=mode, **xr_arg), extract_units(v), + ) + actual = v.pad(mode=mode, **xr_arg) + + assert_units_equal(expected, actual) + assert_equal(actual, expected) + assert isinstance(actual._data, type(v._data)) + @pytest.mark.parametrize("xr_arg, np_arg", _PAD_XR_NP_ARGS) def test_pad_constant_values(self, dtype, xr_arg, np_arg): data = np.arange(4 * 3 * 2).reshape(4, 3, 2).astype(dtype) * unit_registry.m From 3bcb050abfa82059a8980acef4eba7f97d1efa2b Mon Sep 17 00:00:00 2001 From: Keewis Date: Thu, 25 Jun 2020 17:04:51 +0200 Subject: [PATCH 40/50] test the constant pad mode along with all other modes --- xarray/tests/test_units.py | 29 +---------------------------- 1 file changed, 1 insertion(+), 28 deletions(-) diff --git a/xarray/tests/test_units.py b/xarray/tests/test_units.py index 895823939a1..91a6f367afb 100644 --- a/xarray/tests/test_units.py +++ b/xarray/tests/test_units.py @@ -2185,6 +2185,7 @@ def test_no_conflicts(self, unit, dtype): @pytest.mark.parametrize( "mode", [ + "constant", "mean", "median", "reflect", @@ -2213,34 +2214,6 @@ def test_pad(self, mode, xr_arg, np_arg): assert_units_equal(expected, actual) assert_equal(actual, expected) - assert isinstance(actual._data, type(v._data)) - - @pytest.mark.parametrize("xr_arg, np_arg", _PAD_XR_NP_ARGS) - def test_pad_constant_values(self, dtype, xr_arg, np_arg): - data = np.arange(4 * 3 * 2).reshape(4, 3, 2).astype(dtype) * unit_registry.m - v = xr.Variable(["x", "y", "z"], data) - - actual = v.pad(**xr_arg, mode="constant") - expected = xr.Variable( - v.dims, - np.pad( - v.data.astype(float), np_arg, mode="constant", constant_values=np.nan, - ), - ) - assert_identical(expected, actual) - assert_units_equal(expected, actual) - assert isinstance(actual._data, type(v._data)) - - # for the boolean array, we pad False - data = np.full_like(data, False, dtype=bool).reshape(4, 3, 2) - v = xr.Variable(["x", "y", "z"], data) - actual = v.pad(**xr_arg, mode="constant", constant_values=data.flat[0]) - expected = xr.Variable( - v.dims, - np.pad(v.data, np_arg, mode="constant", constant_values=v.data.flat[0]), - ) - assert_identical(actual, expected) - assert_units_equal(expected, actual) @pytest.mark.parametrize( "unit,error", From c3414018f84a75f73df1f7aa3270b5849237d5a4 Mon Sep 17 00:00:00 2001 From: Keewis Date: Fri, 26 Jun 2020 00:20:33 +0200 Subject: [PATCH 41/50] remove most pint version checks, now that pint 0.13 has been released --- xarray/tests/test_units.py | 98 +++----------------------------------- 1 file changed, 7 insertions(+), 91 deletions(-) diff --git a/xarray/tests/test_units.py b/xarray/tests/test_units.py index 91a6f367afb..946942d3227 100644 --- a/xarray/tests/test_units.py +++ b/xarray/tests/test_units.py @@ -425,10 +425,6 @@ def test_apply_ufunc_dataset(variant, dtype): assert_identical(expected, actual) -# TODO: remove once pint==0.12 has been released -@pytest.mark.xfail( - LooseVersion(pint.__version__) <= "0.12", reason="pint bug in isclose" -) @pytest.mark.parametrize( "unit,error", ( @@ -533,10 +529,6 @@ def test_align_dataarray(value, variant, unit, error, dtype): assert_allclose(expected_b, actual_b) -# TODO: remove once pint==0.12 has been released -@pytest.mark.xfail( - LooseVersion(pint.__version__) <= "0.12", reason="pint bug in isclose" -) @pytest.mark.parametrize( "unit,error", ( @@ -1014,10 +1006,6 @@ def test_concat_dataset(variant, unit, error, dtype): assert_identical(expected, actual) -# TODO: remove once pint==0.12 has been released -@pytest.mark.xfail( - LooseVersion(pint.__version__) <= "0.12", reason="pint bug in isclose" -) @pytest.mark.parametrize( "unit,error", ( @@ -1122,10 +1110,6 @@ def test_merge_dataarray(variant, unit, error, dtype): assert_allclose(expected, actual) -# TODO: remove once pint==0.12 has been released -@pytest.mark.xfail( - LooseVersion(pint.__version__) <= "0.12", reason="pint bug in isclose" -) @pytest.mark.parametrize( "unit,error", ( @@ -1537,10 +1521,6 @@ def test_aggregation(self, func, dtype): assert_units_equal(expected, actual) assert_identical(expected, actual) - # TODO: remove once pint==0.12 has been released - @pytest.mark.xfail( - LooseVersion(pint.__version__) <= "0.12", reason="pint bug in isclose" - ) def test_aggregate_complex(self): variable = xr.Variable("x", [1, 2j, np.nan] * unit_registry.m) expected = xr.Variable((), (0.5 + 1j) * unit_registry.m) @@ -1549,10 +1529,6 @@ def test_aggregate_complex(self): assert_units_equal(expected, actual) assert_allclose(expected, actual) - # TODO: remove once pint==0.12 has been released - @pytest.mark.xfail( - LooseVersion(pint.__version__) <= "0.12", reason="pint bug in isclose" - ) @pytest.mark.parametrize( "func", ( @@ -1852,10 +1828,6 @@ def test_isel(self, indices, dtype): assert_units_equal(expected, actual) assert_identical(expected, actual) - # TODO: remove once pint==0.12 has been released - @pytest.mark.xfail( - LooseVersion(pint.__version__) <= "0.12", reason="pint bug in isclose" - ) @pytest.mark.parametrize( "unit,error", ( @@ -1984,13 +1956,7 @@ def test_squeeze(self, dim, dtype): "func", ( method("coarsen", windows={"y": 2}, func=np.mean), - pytest.param( - method("quantile", q=[0.25, 0.75]), - marks=pytest.mark.xfail( - LooseVersion(pint.__version__) <= "0.12", - reason="quantile / nanquantile not implemented yet", - ), - ), + method("quantile", q=[0.25, 0.75]), pytest.param( method("rank", dim="x"), marks=pytest.mark.xfail(reason="rank not implemented for non-ndarray"), @@ -2321,10 +2287,6 @@ def test_repr(self, func, variant, dtype): # warnings or errors, but does not check the result func(data_array) - # TODO: remove once pint==0.12 has been released - @pytest.mark.xfail( - LooseVersion(pint.__version__) <= "0.12", reason="pint bug in isclose", - ) @pytest.mark.parametrize( "func", ( @@ -3425,10 +3387,6 @@ def test_head_tail_thin(self, func, dtype): assert_units_equal(expected, actual) assert_identical(expected, actual) - # TODO: remove once pint==0.12 has been released - @pytest.mark.xfail( - LooseVersion(pint.__version__) <= "0.12", reason="pint bug in isclose" - ) @pytest.mark.parametrize("variant", ("data", "coords")) @pytest.mark.parametrize( "func", @@ -3502,10 +3460,6 @@ def test_interp_reindex_indexing(self, func, unit, error, dtype): assert_units_equal(expected, actual) assert_identical(expected, actual) - # TODO: remove once pint==0.12 has been released - @pytest.mark.xfail( - LooseVersion(pint.__version__) <= "0.12", reason="pint bug in isclose" - ) @pytest.mark.parametrize("variant", ("data", "coords")) @pytest.mark.parametrize( "func", @@ -3680,13 +3634,7 @@ def test_stacking_reordering(self, func, dtype): method("diff", dim="x"), method("differentiate", coord="x"), method("integrate", dim="x"), - pytest.param( - method("quantile", q=[0.25, 0.75]), - marks=pytest.mark.xfail( - LooseVersion(pint.__version__) <= "0.12", - reason="quantile / nanquantile not implemented yet", - ), - ), + method("quantile", q=[0.25, 0.75]), method("reduce", func=np.sum, dim="x"), pytest.param(lambda x: x.dot(x), id="method_dot"), ), @@ -3724,10 +3672,6 @@ def test_computation(self, func, variant, dtype): assert_units_equal(expected, actual) assert_identical(expected, actual) - # TODO: remove once pint==0.12 has been released - @pytest.mark.xfail( - LooseVersion(pint.__version__) <= "0.12", reason="pint bug in isclose" - ) @pytest.mark.parametrize( "variant", ( @@ -3818,13 +3762,7 @@ def test_resample(self, dtype): method("assign_coords", z=("x", np.arange(5) * unit_registry.s)), method("first"), method("last"), - pytest.param( - method("quantile", q=[0.25, 0.5, 0.75], dim="x"), - marks=pytest.mark.xfail( - LooseVersion(pint.__version__) <= "0.12", - reason="quantile / nanquantile not implemented yet", - ), - ), + method("quantile", q=[0.25, 0.5, 0.75], dim="x"), ), ids=repr, ) @@ -5137,13 +5075,7 @@ def test_interp_reindex_like_indexing(self, func, unit, error, dtype): method("diff", dim="x"), method("differentiate", coord="x"), method("integrate", coord="x"), - pytest.param( - method("quantile", q=[0.25, 0.75]), - marks=pytest.mark.xfail( - LooseVersion(pint.__version__) <= "0.12", - reason="nanquantile not implemented yet", - ), - ), + method("quantile", q=[0.25, 0.75]), method("reduce", func=np.sum, dim="x"), method("map", np.fabs), ), @@ -5193,13 +5125,7 @@ def test_computation(self, func, variant, dtype): "func", ( method("groupby", "x"), - pytest.param( - method("groupby_bins", "x", bins=2), - marks=pytest.mark.xfail( - LooseVersion(pint.__version__) <= "0.12", - reason="needs assert_allclose but that does not work with pint", - ), - ), + method("groupby_bins", "x", bins=2), method("coarsen", x=2), pytest.param( method("rolling", x=3), marks=pytest.mark.xfail(reason="strips units") @@ -5248,11 +5174,7 @@ def test_computation_objects(self, func, variant, dtype): actual = func(ds).mean(*args) assert_units_equal(expected, actual) - # TODO: remove once pint 0.12 has been released - if LooseVersion(pint.__version__) <= "0.12": - assert_equal(expected, actual) - else: - assert_allclose(expected, actual) + assert_allclose(expected, actual) @pytest.mark.parametrize( "variant", @@ -5303,13 +5225,7 @@ def test_resample(self, variant, dtype): method("assign_coords", v=("x", np.arange(5) * unit_registry.s)), method("first"), method("last"), - pytest.param( - method("quantile", q=[0.25, 0.5, 0.75], dim="x"), - marks=pytest.mark.xfail( - LooseVersion(pint.__version__) <= "0.12", - reason="nanquantile not implemented", - ), - ), + method("quantile", q=[0.25, 0.5, 0.75], dim="x"), ), ids=repr, ) From a9cc7d4274174bfad87203b86c9f74f286caeb18 Mon Sep 17 00:00:00 2001 From: Keewis Date: Fri, 26 Jun 2020 00:28:12 +0200 Subject: [PATCH 42/50] use conda to install pint --- ci/requirements/py36-min-nep18.yml | 3 +-- ci/requirements/py36.yml | 2 +- ci/requirements/py37-windows.yml | 2 +- ci/requirements/py37.yml | 2 +- ci/requirements/py38-all-but-dask.yml | 2 +- ci/requirements/py38.yml | 2 +- 6 files changed, 6 insertions(+), 7 deletions(-) diff --git a/ci/requirements/py36-min-nep18.yml b/ci/requirements/py36-min-nep18.yml index cd2b1a18c77..dd543ce4ddf 100644 --- a/ci/requirements/py36-min-nep18.yml +++ b/ci/requirements/py36-min-nep18.yml @@ -11,6 +11,7 @@ dependencies: - msgpack-python=0.6 # remove once distributed is bumped. distributed GH3491 - numpy=1.17 - pandas=0.25 + - pint=0.13 - pip - pytest - pytest-cov @@ -18,5 +19,3 @@ dependencies: - scipy=1.2 - setuptools=41.2 - sparse=0.8 - - pip: - - pint==0.13 diff --git a/ci/requirements/py36.yml b/ci/requirements/py36.yml index aa2baf9dcce..a500173f277 100644 --- a/ci/requirements/py36.yml +++ b/ci/requirements/py36.yml @@ -28,6 +28,7 @@ dependencies: - numba - numpy - pandas + - pint - pip - pseudonetcdf - pydap @@ -44,4 +45,3 @@ dependencies: - zarr - pip: - numbagg - - pint diff --git a/ci/requirements/py37-windows.yml b/ci/requirements/py37-windows.yml index 8b12704d644..e9e5c7a900a 100644 --- a/ci/requirements/py37-windows.yml +++ b/ci/requirements/py37-windows.yml @@ -28,6 +28,7 @@ dependencies: - numba - numpy - pandas + - pint - pip - pseudonetcdf - pydap @@ -44,4 +45,3 @@ dependencies: - zarr - pip: - numbagg - - pint diff --git a/ci/requirements/py37.yml b/ci/requirements/py37.yml index 70c453e8776..dba3926596e 100644 --- a/ci/requirements/py37.yml +++ b/ci/requirements/py37.yml @@ -28,6 +28,7 @@ dependencies: - numba - numpy - pandas + - pint - pip - pseudonetcdf - pydap @@ -44,4 +45,3 @@ dependencies: - zarr - pip: - numbagg - - pint diff --git a/ci/requirements/py38-all-but-dask.yml b/ci/requirements/py38-all-but-dask.yml index 6d76eecbd6a..a375d9e1e5a 100644 --- a/ci/requirements/py38-all-but-dask.yml +++ b/ci/requirements/py38-all-but-dask.yml @@ -25,6 +25,7 @@ dependencies: - numba - numpy - pandas + - pint - pip - pseudonetcdf - pydap @@ -41,4 +42,3 @@ dependencies: - zarr - pip: - numbagg - - pint diff --git a/ci/requirements/py38.yml b/ci/requirements/py38.yml index 6f35138978c..7dff3a1bd97 100644 --- a/ci/requirements/py38.yml +++ b/ci/requirements/py38.yml @@ -28,6 +28,7 @@ dependencies: - numba - numpy - pandas + - pint - pip - pseudonetcdf - pydap @@ -44,4 +45,3 @@ dependencies: - zarr - pip: - numbagg - - pint From 673cabc11b083cd25a860776fa25a22d6655f362 Mon Sep 17 00:00:00 2001 From: Keewis Date: Fri, 26 Jun 2020 01:13:27 +0200 Subject: [PATCH 43/50] xfail the DataArray comparison test until pint's dev version fixed it --- xarray/tests/test_units.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/xarray/tests/test_units.py b/xarray/tests/test_units.py index c70e0c8129a..63d61211874 100644 --- a/xarray/tests/test_units.py +++ b/xarray/tests/test_units.py @@ -1,6 +1,5 @@ import functools import operator -from distutils.version import LooseVersion import numpy as np import pandas as pd @@ -2395,7 +2394,7 @@ def test_binary_operations(self, func, dtype): operator.eq, id="equal", marks=pytest.mark.xfail( - LooseVersion(pint.__version__) < "0.14", + # LooseVersion(pint.__version__) < "0.14", reason="inconsistencies in the return values of pint's eq", ), ), From 158c83e0beb3104d4b0491c5eef3fb716e1ef944 Mon Sep 17 00:00:00 2001 From: Keewis Date: Sat, 27 Jun 2020 00:04:22 +0200 Subject: [PATCH 44/50] add tests for the pad method of DataArray and Dataset --- xarray/tests/test_units.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/xarray/tests/test_units.py b/xarray/tests/test_units.py index 63d61211874..9413087f3d0 100644 --- a/xarray/tests/test_units.py +++ b/xarray/tests/test_units.py @@ -3089,6 +3089,18 @@ def test_broadcast_equals(self, unit, dtype): assert expected == actual + def test_pad(self, dtype): + array = np.linspace(0, 5, 10).astype(dtype) * unit_registry.m + + data_array = xr.DataArray(data=array, dims="x") + units = extract_units(data_array) + + expected = attach_units(strip_units(data_array).pad(x=(2, 3)), units) + actual = data_array.pad(x=(2, 3)) + + assert_units_equal(expected, actual) + assert_equal(expected, actual) + @pytest.mark.parametrize( "variant", ( @@ -4536,6 +4548,19 @@ def test_broadcast_equals(self, unit, dtype): assert expected == actual + def test_pad(self, dtype): + a = np.linspace(0, 5, 10).astype(dtype) * unit_registry.Pa + b = np.linspace(-5, 0, 10).astype(dtype) * unit_registry.degK + + ds = xr.Dataset({"a": ("x", a), "b": ("x", b)}) + units = extract_units(ds) + + expected = attach_units(strip_units(ds).pad(x=(2, 3)), units) + actual = ds.pad(x=(2, 3)) + + assert_units_equal(expected, actual) + assert_equal(expected, actual) + @pytest.mark.parametrize( "func", (method("unstack"), method("reset_index", "v"), method("reorder_levels")), From c8a64662d7794d1c544287f9a5aa247e3c72ffe9 Mon Sep 17 00:00:00 2001 From: Keewis Date: Sat, 27 Jun 2020 00:10:17 +0200 Subject: [PATCH 45/50] add tests for weighted --- xarray/tests/test_units.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/xarray/tests/test_units.py b/xarray/tests/test_units.py index 9413087f3d0..264cac01337 100644 --- a/xarray/tests/test_units.py +++ b/xarray/tests/test_units.py @@ -3711,6 +3711,7 @@ def test_computation(self, func, variant, dtype): reason="numbagg functions are not supported by pint" ), ), + method("weighted", xr.DataArray(data=np.linspace(0, 1, 10), dims="y")), ), ids=repr, ) @@ -5160,6 +5161,7 @@ def test_computation(self, func, variant, dtype): reason="numbagg functions are not supported by pint" ), ), + method("weighted", xr.DataArray(data=np.linspace(0, 1, 5), dims="y")), ), ids=repr, ) From d8783e4c2e5529cff6b170e68453cc418d5059b2 Mon Sep 17 00:00:00 2001 From: Keewis Date: Sat, 27 Jun 2020 00:23:18 +0200 Subject: [PATCH 46/50] update whats-new.rst --- doc/whats-new.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index d82be79270e..49e5bdc48e4 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -82,7 +82,7 @@ New Features - Support dask handling for :py:meth:`DataArray.idxmax`, :py:meth:`DataArray.idxmin`, :py:meth:`Dataset.idxmax`, :py:meth:`Dataset.idxmin`. (:pull:`3922`, :pull:`4135`) By `Kai Mühlbauer `_ and `Pascal Bourgault `_. -- More support for unit aware arrays with pint (:pull:`3643`, :pull:`3975`) +- More support for unit aware arrays with pint (:pull:`3643`, :pull:`3975`, :pull:`4163`) By `Justus Magin `_. - Support overriding existing variables in ``to_zarr()`` with ``mode='a'`` even without ``append_dim``, as long as dimension sizes do not change. From 1c827dad7c87a694faa8f9dace88e323ca3bf672 Mon Sep 17 00:00:00 2001 From: Keewis Date: Sun, 28 Jun 2020 18:47:09 +0200 Subject: [PATCH 47/50] replace assert np.allclose(...) with assert_duckarray_allclose(...) --- xarray/tests/test_units.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/xarray/tests/test_units.py b/xarray/tests/test_units.py index 264cac01337..70fe76b9107 100644 --- a/xarray/tests/test_units.py +++ b/xarray/tests/test_units.py @@ -9,7 +9,7 @@ from xarray.core import dtypes from xarray.core.npcompat import IS_NEP18_ACTIVE -from . import assert_allclose, assert_equal, assert_identical +from . import assert_allclose, assert_duckarray_allclose, assert_equal, assert_identical from .test_variable import _PAD_XR_NP_ARGS pint = pytest.importorskip("pint") @@ -1644,8 +1644,7 @@ def test_raw_numpy_methods(self, func, unit, error, dtype): actual = func(variable, *args, **kwargs) assert_units_equal(expected, actual) - # can't use `np.testing.assert_allclose` since that casts to numpy.array - assert np.allclose(expected, actual) + assert_duckarray_allclose(expected, actual) @pytest.mark.parametrize( "func", (method("isnull"), method("notnull"), method("count")), ids=repr @@ -2560,9 +2559,7 @@ def test_item(self, dtype): expected = func(strip_units(data_array)) * unit_registry.m actual = func(data_array) - # TODO: use something like np.testing.assert_allclose, but - # with duckarray support - assert np.allclose(expected, actual) + assert_duckarray_allclose(expected, actual) @pytest.mark.parametrize( "unit,error", From 1ce16eaf15cccbb99c26ce75136093a9b96fa5cd Mon Sep 17 00:00:00 2001 From: Keewis Date: Sun, 28 Jun 2020 18:56:18 +0200 Subject: [PATCH 48/50] fix the dask fallback --- xarray/tests/test_testing.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/xarray/tests/test_testing.py b/xarray/tests/test_testing.py index f4f47174e00..fe2153acab8 100644 --- a/xarray/tests/test_testing.py +++ b/xarray/tests/test_testing.py @@ -6,10 +6,9 @@ from . import has_dask try: - import dask.array as da + from dask.array import from_array as dask_from_array except ImportError: - da = object() - da.Array = lambda x: x + dask_from_array = lambda x: x try: import pint @@ -63,7 +62,7 @@ def test_assert_allclose(obj1, obj2): ( pytest.param(np.array, id="numpy"), pytest.param( - da.from_array, + dask_from_array, id="dask", marks=pytest.mark.skipif(not has_dask, reason="requires dask"), ), @@ -96,7 +95,7 @@ def test_assert_duckarray_equal_failing(duckarray, obj1, obj2): ( pytest.param(np.array, id="numpy"), pytest.param( - da.from_array, + dask_from_array, id="dask", marks=pytest.mark.skipif(not has_dask, reason="requires dask"), ), From a68048b53373df50d43d3cb6d94a46274d3ebc50 Mon Sep 17 00:00:00 2001 From: Keewis Date: Sun, 28 Jun 2020 19:23:01 +0200 Subject: [PATCH 49/50] xfail the pint tests for now since there's a bug in pint --- xarray/tests/test_testing.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/xarray/tests/test_testing.py b/xarray/tests/test_testing.py index fe2153acab8..39ad250246b 100644 --- a/xarray/tests/test_testing.py +++ b/xarray/tests/test_testing.py @@ -69,7 +69,12 @@ def test_assert_allclose(obj1, obj2): pytest.param( quantity, id="pint", - marks=pytest.mark.skipif(not has_pint, reason="requires pint"), + marks=[ + pytest.mark.skipif(not has_pint, reason="requires pint"), + pytest.mark.xfail( + reason="inconsistencies in the return value of pint's implementation of eq" + ), + ], ), ), ) @@ -102,7 +107,12 @@ def test_assert_duckarray_equal_failing(duckarray, obj1, obj2): pytest.param( quantity, id="pint", - marks=pytest.mark.skipif(not has_pint, reason="requires pint"), + marks=[ + pytest.mark.skipif(not has_pint, reason="requires pint"), + pytest.mark.xfail( + reason="inconsistencies in the return value of pint's implementation of eq" + ), + ], ), ), ) From 34931eb837c3b948b46e1983df4d98109b500a59 Mon Sep 17 00:00:00 2001 From: Keewis Date: Mon, 29 Jun 2020 22:05:42 +0200 Subject: [PATCH 50/50] use utils.is_array_like and utils.is_scalar --- xarray/core/utils.py | 6 ++++++ xarray/testing.py | 16 +++++----------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/xarray/core/utils.py b/xarray/core/utils.py index 0542f850b02..668405ba574 100644 --- a/xarray/core/utils.py +++ b/xarray/core/utils.py @@ -247,6 +247,12 @@ def is_list_like(value: Any) -> bool: return isinstance(value, list) or isinstance(value, tuple) +def is_array_like(value: Any) -> bool: + return ( + hasattr(value, "ndim") and hasattr(value, "shape") and hasattr(value, "dtype") + ) + + def either_dict_or_kwargs( pos_kwargs: Optional[Mapping[Hashable, T]], kw_kwargs: Mapping[str, T], diff --git a/xarray/testing.py b/xarray/testing.py index 472f8d60e31..ec479ef09d4 100644 --- a/xarray/testing.py +++ b/xarray/testing.py @@ -196,21 +196,15 @@ def assert_duckarray_equal(x, y, err_msg="", verbose=True): """ Like `np.testing.assert_array_equal`, but for duckarrays """ __tracebackhide__ = True - def array_like(x): - return hasattr(x, "ndim") and hasattr(x, "shape") and hasattr(x, "dtype") - - def scalar(x): - return isinstance(x, (bool, int, float, complex, str)) or ( - array_like(x) and x.ndim == 0 - ) - - if not array_like(x) and not scalar(x): + if not utils.is_array_like(x) and not utils.is_scalar(x): x = np.asarray(x) - if not array_like(y) and not scalar(y): + if not utils.is_array_like(y) and not utils.is_scalar(y): y = np.asarray(y) - if (array_like(x) and scalar(y)) or (scalar(x) and array_like(y)): + if (utils.is_array_like(x) and utils.is_scalar(y)) or ( + utils.is_scalar(x) and utils.is_array_like(y) + ): equiv = (x == y).all() else: equiv = duck_array_ops.array_equiv(x, y)