From 9309de5c2efeff20482997a5cd406378b1f06f07 Mon Sep 17 00:00:00 2001 From: garciam Date: Fri, 2 Sep 2022 10:50:56 +0200 Subject: [PATCH 1/6] Added support for the new compression argument in netCDF4 > 1.6.0 --- xarray/backends/netCDF4_.py | 6 +++++- xarray/tests/test_backends.py | 13 +++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/xarray/backends/netCDF4_.py b/xarray/backends/netCDF4_.py index 19047d17c6c..59478befbd1 100644 --- a/xarray/backends/netCDF4_.py +++ b/xarray/backends/netCDF4_.py @@ -6,6 +6,7 @@ from contextlib import suppress import numpy as np +from packaging.version import Version from .. import coding from ..coding.variables import pop_to @@ -242,13 +243,16 @@ def _extract_nc4_variable_encoding( "shuffle", "_FillValue", "dtype", + "compression" } if lsd_okay: valid_encodings.add("least_significant_digit") if h5py_okay: - valid_encodings.add("compression") valid_encodings.add("compression_opts") + if backend == "netCDF4" and Version(netCDF4.__version__) < Version("1.6.0"): + valid_encodings.remove("compression") + if not raise_on_invalid and encoding.get("chunksizes") is not None: # It's possible to get encoded chunksizes larger than a dimension size # if the original file had an unlimited dimension. This is problematic diff --git a/xarray/tests/test_backends.py b/xarray/tests/test_backends.py index 4f5434ec05e..5682ff5668e 100644 --- a/xarray/tests/test_backends.py +++ b/xarray/tests/test_backends.py @@ -4888,6 +4888,19 @@ def test_extract_nc4_variable_encoding(self): encoding = _extract_nc4_variable_encoding(var, unlimited_dims=("x",)) assert {} == encoding + @requires_netCDF4 + def test_extract_nc4_variable_encoding_netcdf4(self, monkeypatch): + # New netCDF4 1.6.0 compression argument. + var = xr.Variable(("x",), [1, 2, 3], {}, {"compression": "szlib"}) + _extract_nc4_variable_encoding(var, backend="netCDF4", raise_on_invalid=True) + # Check that fails for older versions + import netCDF4 + with monkeypatch.context() as m: + m.setattr(netCDF4, "__version__", "1.5.0") + with pytest.raises(ValueError): + _extract_nc4_variable_encoding(var, backend="netCDF4", + raise_on_invalid=True) + def test_extract_h5nc_encoding(self): # not supported with h5netcdf (yet) var = xr.Variable(("x",), [1, 2, 3], {}, {"least_sigificant_digit": 2}) From 591d5b56176f4e38151c98c2c67c4d45b1a9c77f Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 2 Sep 2022 09:08:11 +0000 Subject: [PATCH 2/6] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- xarray/backends/netCDF4_.py | 2 +- xarray/tests/test_backends.py | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/xarray/backends/netCDF4_.py b/xarray/backends/netCDF4_.py index 59478befbd1..958c228ab06 100644 --- a/xarray/backends/netCDF4_.py +++ b/xarray/backends/netCDF4_.py @@ -243,7 +243,7 @@ def _extract_nc4_variable_encoding( "shuffle", "_FillValue", "dtype", - "compression" + "compression", } if lsd_okay: valid_encodings.add("least_significant_digit") diff --git a/xarray/tests/test_backends.py b/xarray/tests/test_backends.py index 5682ff5668e..8b01ebf91ce 100644 --- a/xarray/tests/test_backends.py +++ b/xarray/tests/test_backends.py @@ -4895,11 +4895,13 @@ def test_extract_nc4_variable_encoding_netcdf4(self, monkeypatch): _extract_nc4_variable_encoding(var, backend="netCDF4", raise_on_invalid=True) # Check that fails for older versions import netCDF4 + with monkeypatch.context() as m: m.setattr(netCDF4, "__version__", "1.5.0") with pytest.raises(ValueError): - _extract_nc4_variable_encoding(var, backend="netCDF4", - raise_on_invalid=True) + _extract_nc4_variable_encoding( + var, backend="netCDF4", raise_on_invalid=True + ) def test_extract_h5nc_encoding(self): # not supported with h5netcdf (yet) From 402ac49d3343bc90a07c8c54948357c4ca9897cf Mon Sep 17 00:00:00 2001 From: garciam Date: Thu, 8 Sep 2022 10:21:26 +0200 Subject: [PATCH 3/6] Check both netCDF4 and libnetcdf versions to see if they support the new compression arguments. --- xarray/backends/netCDF4_.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/xarray/backends/netCDF4_.py b/xarray/backends/netCDF4_.py index 958c228ab06..4f945e4de1e 100644 --- a/xarray/backends/netCDF4_.py +++ b/xarray/backends/netCDF4_.py @@ -250,8 +250,14 @@ def _extract_nc4_variable_encoding( if h5py_okay: valid_encodings.add("compression_opts") - if backend == "netCDF4" and Version(netCDF4.__version__) < Version("1.6.0"): - valid_encodings.remove("compression") + if backend == "netCDF4" and has_netcdf4: + # if using netCDF4 check the versions are updated enough for supporting the new + # compression + if ( + Version(netCDF4.__version__) < Version("1.6.0") or + Version(netCDF4.getlibversion().split(" ")[0]) < Version("4.9.0") + ): + valid_encodings.remove("compression") if not raise_on_invalid and encoding.get("chunksizes") is not None: # It's possible to get encoded chunksizes larger than a dimension size From 8fe053d0d174a1368bb08a67f99fee146a41a7ab Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 8 Sep 2022 08:26:07 +0000 Subject: [PATCH 4/6] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- xarray/backends/netCDF4_.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/xarray/backends/netCDF4_.py b/xarray/backends/netCDF4_.py index 4f945e4de1e..d6412c11cc6 100644 --- a/xarray/backends/netCDF4_.py +++ b/xarray/backends/netCDF4_.py @@ -253,10 +253,9 @@ def _extract_nc4_variable_encoding( if backend == "netCDF4" and has_netcdf4: # if using netCDF4 check the versions are updated enough for supporting the new # compression - if ( - Version(netCDF4.__version__) < Version("1.6.0") or - Version(netCDF4.getlibversion().split(" ")[0]) < Version("4.9.0") - ): + if Version(netCDF4.__version__) < Version("1.6.0") or Version( + netCDF4.getlibversion().split(" ")[0] + ) < Version("4.9.0"): valid_encodings.remove("compression") if not raise_on_invalid and encoding.get("chunksizes") is not None: From 01bf86732102bf85d418d3278b8ba9b72f48974e Mon Sep 17 00:00:00 2001 From: garciam Date: Mon, 31 Oct 2022 12:44:32 +0100 Subject: [PATCH 5/6] Remove checking netCDF4 and libnetcdf versions. We let them fail if they do not match compression/zlib arguments. --- xarray/backends/netCDF4_.py | 9 --------- xarray/tests/test_backends.py | 8 -------- 2 files changed, 17 deletions(-) diff --git a/xarray/backends/netCDF4_.py b/xarray/backends/netCDF4_.py index d6412c11cc6..1b528276415 100644 --- a/xarray/backends/netCDF4_.py +++ b/xarray/backends/netCDF4_.py @@ -6,7 +6,6 @@ from contextlib import suppress import numpy as np -from packaging.version import Version from .. import coding from ..coding.variables import pop_to @@ -250,14 +249,6 @@ def _extract_nc4_variable_encoding( if h5py_okay: valid_encodings.add("compression_opts") - if backend == "netCDF4" and has_netcdf4: - # if using netCDF4 check the versions are updated enough for supporting the new - # compression - if Version(netCDF4.__version__) < Version("1.6.0") or Version( - netCDF4.getlibversion().split(" ")[0] - ) < Version("4.9.0"): - valid_encodings.remove("compression") - if not raise_on_invalid and encoding.get("chunksizes") is not None: # It's possible to get encoded chunksizes larger than a dimension size # if the original file had an unlimited dimension. This is problematic diff --git a/xarray/tests/test_backends.py b/xarray/tests/test_backends.py index 8b01ebf91ce..84acce36616 100644 --- a/xarray/tests/test_backends.py +++ b/xarray/tests/test_backends.py @@ -4893,15 +4893,7 @@ def test_extract_nc4_variable_encoding_netcdf4(self, monkeypatch): # New netCDF4 1.6.0 compression argument. var = xr.Variable(("x",), [1, 2, 3], {}, {"compression": "szlib"}) _extract_nc4_variable_encoding(var, backend="netCDF4", raise_on_invalid=True) - # Check that fails for older versions - import netCDF4 - with monkeypatch.context() as m: - m.setattr(netCDF4, "__version__", "1.5.0") - with pytest.raises(ValueError): - _extract_nc4_variable_encoding( - var, backend="netCDF4", raise_on_invalid=True - ) def test_extract_h5nc_encoding(self): # not supported with h5netcdf (yet) From de79377602322940d202fd03900c1b31f655c697 Mon Sep 17 00:00:00 2001 From: garciam Date: Mon, 31 Oct 2022 13:00:12 +0100 Subject: [PATCH 6/6] Added test after merging with the new version of the file. --- xarray/tests/test_backends.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/xarray/tests/test_backends.py b/xarray/tests/test_backends.py index 64030b3f595..4971f878d22 100644 --- a/xarray/tests/test_backends.py +++ b/xarray/tests/test_backends.py @@ -4949,6 +4949,12 @@ def test_extract_nc4_variable_encoding(self) -> None: encoding = _extract_nc4_variable_encoding(var, unlimited_dims=("x",)) assert {} == encoding + @requires_netCDF4 + def test_extract_nc4_variable_encoding_netcdf4(self, monkeypatch): + # New netCDF4 1.6.0 compression argument. + var = xr.Variable(("x",), [1, 2, 3], {}, {"compression": "szlib"}) + _extract_nc4_variable_encoding(var, backend="netCDF4", raise_on_invalid=True) + def test_extract_h5nc_encoding(self) -> None: # not supported with h5netcdf (yet) var = xr.Variable(("x",), [1, 2, 3], {}, {"least_sigificant_digit": 2})