diff --git a/environment.yml b/environment.yml index 361dacb2..cf1cb7b9 100644 --- a/environment.yml +++ b/environment.yml @@ -1,6 +1,6 @@ channels: - conda-forge dependencies: - - iris>=2 + - iris>=2.4 - python-eccodes>=0.9.1,<2 - pep8 diff --git a/iris_grib/__init__.py b/iris_grib/__init__.py index dcc32799..9e4a6e6b 100644 --- a/iris_grib/__init__.py +++ b/iris_grib/__init__.py @@ -20,12 +20,13 @@ import numpy as np import numpy.ma as ma +# NOTE: careful here, to avoid circular imports (as iris imports grib) import iris from iris._lazy_data import as_lazy_data import iris.coord_systems as coord_systems from iris.exceptions import TranslationError, NotYetImplementedError +from iris.util import _array_slice_ifempty -# NOTE: careful here, to avoid circular imports (as iris imports grib) from . import grib_phenom_translation as gptx from . import _save_rules from ._load_convert import convert as load_convert @@ -97,13 +98,19 @@ def ndim(self): return len(self.shape) def __getitem__(self, keys): - with open(self.path, 'rb') as grib_fh: - grib_fh.seek(self.offset) - grib_message = gribapi.grib_new_from_file(grib_fh) - data = _message_values(grib_message, self.shape) - gribapi.grib_release(grib_message) - - return data.__getitem__(keys) + # Avoid fetching file data just to return an 'empty' result. + # Needed because of how dask.array.from_array behaves since Dask v2.0. + result = _array_slice_ifempty(keys, self.shape, self.dtype) + if result is None: + with open(self.path, 'rb') as grib_fh: + grib_fh.seek(self.offset) + grib_message = gribapi.grib_new_from_file(grib_fh) + data = _message_values(grib_message, self.shape) + gribapi.grib_release(grib_message) + + result = data.__getitem__(keys) + + return result def __repr__(self): msg = '<{self.__class__.__name__} shape={self.shape} ' \ diff --git a/iris_grib/message.py b/iris_grib/message.py index 7f2b8598..06642f24 100644 --- a/iris_grib/message.py +++ b/iris_grib/message.py @@ -12,6 +12,7 @@ import re import gribapi +from iris_grib import _array_slice_ifempty import numpy as np import numpy.ma as ma @@ -228,29 +229,35 @@ def _bitmap(self, bitmap_section): def __getitem__(self, keys): # NB. Currently assumes that the validity of this interpretation # is checked before this proxy is created. - message = self.recreate_raw() - sections = message.sections - bitmap_section = sections[6] - bitmap = self._bitmap(bitmap_section) - data = sections[7]['codedValues'] - - if bitmap is not None: - # Note that bitmap and data are both 1D arrays at this point. - if np.count_nonzero(bitmap) == data.shape[0]: - # Only the non-masked values are included in codedValues. - _data = np.empty(shape=bitmap.shape) - _data[bitmap.astype(bool)] = data - # `ma.masked_array` masks where input = 1, the opposite of - # the behaviour specified by the GRIB spec. - data = ma.masked_array(_data, mask=np.logical_not(bitmap), - fill_value=np.nan) - else: - msg = 'Shapes of data and bitmap do not match.' - raise TranslationError(msg) - data = data.reshape(self.shape) + # Avoid fetching file data just to return an 'empty' result. + # Needed because of how dask.array.from_array behaves since Dask v2.0. + result = _array_slice_ifempty(keys, self.shape, self.dtype) + if result is None: + message = self.recreate_raw() + sections = message.sections + bitmap_section = sections[6] + bitmap = self._bitmap(bitmap_section) + data = sections[7]['codedValues'] + + if bitmap is not None: + # Note that bitmap and data are both 1D arrays at this point. + if np.count_nonzero(bitmap) == data.shape[0]: + # Only the non-masked values are included in codedValues. + _data = np.empty(shape=bitmap.shape) + _data[bitmap.astype(bool)] = data + # `ma.masked_array` masks where input = 1, the opposite of + # the behaviour specified by the GRIB spec. + data = ma.masked_array(_data, mask=np.logical_not(bitmap), + fill_value=np.nan) + else: + msg = 'Shapes of data and bitmap do not match.' + raise TranslationError(msg) + + data = data.reshape(self.shape) + result = data.__getitem__(keys) - return data.__getitem__(keys) + return result def __repr__(self): msg = '<{self.__class__.__name__} shape={self.shape} ' \ diff --git a/iris_grib/tests/unit/message/test__DataProxy.py b/iris_grib/tests/unit/message/test__DataProxy.py index 3b6dc6a3..d7494d5e 100644 --- a/iris_grib/tests/unit/message/test__DataProxy.py +++ b/iris_grib/tests/unit/message/test__DataProxy.py @@ -12,6 +12,8 @@ # importing anything else. import iris_grib.tests as tests +from unittest import mock + import numpy as np from numpy.random import randint @@ -41,5 +43,31 @@ def test_bitmap__invalid_indicator(self): data_proxy._bitmap(section_6) +class Test_emptyfetch(tests.IrisGribTest): + # See : + # iris.tests.unit.fileformats.pp.test_PPDataProxy.Test__getitem__slicing + # In this case, test *only* the no-data-read effect, not the method which + # is part of Iris. + def test_empty_slice(self): + # Check behaviour of the getitem call with an 'empty' slicing. + # This is necessary because, since Dask 2.0, the "from_array" function + # takes a zero-length slice of its array argument, to capture array + # metadata, and in those cases we want to avoid file access. + test_dtype = np.dtype(np.float32) + mock_datafetch = mock.MagicMock() + proxy = _DataProxy(shape=(3, 4), + dtype=np.dtype(np.float32), + recreate_raw=mock_datafetch) + + # Test the special no-data indexing operation. + result = proxy[0:0, 0:0] + + # Check the behaviour and results were as expected. + self.assertEqual(mock_datafetch.call_count, 0) + self.assertIsInstance(result, np.ndarray) + self.assertEqual(result.dtype, test_dtype) + self.assertEqual(result.shape, (0, 0)) + + if __name__ == '__main__': tests.main() diff --git a/setup.py b/setup.py index b36ed2a9..5db3255b 100644 --- a/setup.py +++ b/setup.py @@ -80,7 +80,7 @@ def file_walk_relative(top, remove=''): # NOTE: The Python 3 bindings to eccodes (eccodes-python) is available on # PyPI, but the user is required to install eccodes itself manually. See # ECMWF ecCodes installation documentation for more information. - install_requires=['scitools-iris>=2.0.*'] + ['eccodes-python'], + install_requires=['scitools-iris>=2.4.*'] + ['eccodes-python'], test_suite = 'iris_grib.tests', )