From 4634453cddf4d9d9021e209e1f56be74fa563fae Mon Sep 17 00:00:00 2001 From: Eddy Comyn-Platt <53045993+EddyCMWF@users.noreply.github.com> Date: Mon, 18 Sep 2023 20:50:02 +0100 Subject: [PATCH] only import ecCodes dependency where essential (#192) * only import ecCodes dependency where essential * Add ci tests with no ecCodes available --------- Co-authored-by: Sandor Kertesz --- .github/workflows/ci.yml | 5 ++++ earthkit/data/sources/file.py | 3 ++- pytest.ini | 1 + tests/indexing/indexing_fixtures.py | 32 ----------------------- tests/readers/test_netcdf_reader.py | 16 +++++++----- tests/utils/test_module_inputs_wrapper.py | 7 ++++- 6 files changed, 23 insertions(+), 41 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4ef239eb..4893d17e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -60,6 +60,11 @@ jobs: - name: Run tests run: | make unit-tests + - name: Run tests without eccodes + run: | + micromamba remove eccodes + python -m pytest -v -m 'no_eccodes' + type-check: needs: [unit-tests] diff --git a/earthkit/data/sources/file.py b/earthkit/data/sources/file.py index 842007c4..cd80e48a 100644 --- a/earthkit/data/sources/file.py +++ b/earthkit/data/sources/file.py @@ -15,7 +15,6 @@ from earthkit.data import from_source from earthkit.data.core.caching import CACHE from earthkit.data.readers import reader -from earthkit.data.sources.file_indexed import FileIndexedSource from . import Source @@ -53,6 +52,8 @@ def mutate(self): # here we must have a file or a directory if self._kwargs.get("indexing", False): + from earthkit.data.sources.file_indexed import FileIndexedSource + kw = dict(self._kwargs) kw.pop("indexing", None) return FileIndexedSource(self.path, filter=filter, merger=self.merger, **kw) diff --git a/pytest.ini b/pytest.ini index c2cb6780..09ce5741 100644 --- a/pytest.ini +++ b/pytest.ini @@ -6,4 +6,5 @@ markers = ftp: test that used FTP. FTP is an old protocol and is not supported by most recent firewalls. notebook: testing notebooks can be slow. But needs to be performed to ensure that the documention is tested. no_cache_init: a test where the cache is not initialised. Must be run with --forked. + no_eccodes: a test which should pass when ecCodes is not installed testpaths = tests diff --git a/tests/indexing/indexing_fixtures.py b/tests/indexing/indexing_fixtures.py index 425ac478..383435a4 100644 --- a/tests/indexing/indexing_fixtures.py +++ b/tests/indexing/indexing_fixtures.py @@ -11,10 +11,8 @@ import os import shutil -import warnings from earthkit.data.core.temporary import temp_directory, temp_file -from earthkit.data.readers.grib.index import GribFieldList from earthkit.data.testing import ( earthkit_examples_file, earthkit_file, @@ -78,36 +76,6 @@ def list_of_dicts(): ] -class GribIndexFromDicts(GribFieldList): - def __init__(self, list_of_dicts, *args, **kwargs): - self.list_of_dicts = list_of_dicts - print(f"KWARGS={kwargs}") - super().__init__(*args, **kwargs) - - def __getitem__(self, n): - class _VirtualGribField(dict): - def metadata(_self, n, **kwargs): - try: - if n == "level": - n = "levelist" - if n == "shortName": - n = "param" - if n == "paramId": - n = "_param_id" - return _self[n] - except KeyError: - warnings.warn("Cannot find all metadata keys.") - - @property - def values(self, n): - return self["values"] - - return _VirtualGribField(self.list_of_dicts[n]) - - def __len__(self): - return len(self.list_of_dicts) - - def get_tmp_fixture(input_mode): tmp = { "directory": unique_grib_dir, diff --git a/tests/readers/test_netcdf_reader.py b/tests/readers/test_netcdf_reader.py index 5db8812c..8eeddf78 100644 --- a/tests/readers/test_netcdf_reader.py +++ b/tests/readers/test_netcdf_reader.py @@ -32,16 +32,15 @@ def check_array(v, shape=None, first=None, last=None, meanv=None, eps=1e-3): assert np.isclose(v.mean(), meanv, eps) -def test_netcdf(): - for s in from_source("file", earthkit_file("docs/examples/test.nc")): - s is not None - - -def test_dummy_netcdf_reader_1(): +@pytest.mark.no_eccodes +def test_netcdf_reader(): ds = from_source("file", earthkit_file("docs/examples/test.nc")) # assert str(ds).startswith("NetCDFReader"), r assert len(ds) == 2 - assert isinstance(ds[1], NetCDFField), ds + assert isinstance(ds[0], NetCDFField) + assert isinstance(ds[1], NetCDFField) + for f in from_source("file", earthkit_file("docs/examples/test.nc")): + assert isinstance(f, NetCDFField) @pytest.mark.parametrize("attribute", ["coordinates", "bounds", "grid_mapping"]) @@ -130,6 +129,7 @@ def test_netcdf_multi_cds(): source.to_xarray() +@pytest.mark.no_eccodes def test_netcdf_multi_sources(): path = earthkit_test_data_file("era5_2t_1.nc") s1 = from_source("file", path) @@ -167,6 +167,7 @@ def test_netcdf_multi_sources(): s3.to_xarray() +@pytest.mark.no_eccodes def test_netcdf_multi_files(): ds = from_source( "file", @@ -199,6 +200,7 @@ def test_netcdf_multi_files(): ds.to_xarray() +@pytest.mark.no_eccodes def test_get_fields_missing_standard_name_attr_in_coord_array(): """test _get_fields() can handle a missing 'standard_name' attr in coordinate data arrays""" diff --git a/tests/utils/test_module_inputs_wrapper.py b/tests/utils/test_module_inputs_wrapper.py index 3df34b81..e39c1572 100644 --- a/tests/utils/test_module_inputs_wrapper.py +++ b/tests/utils/test_module_inputs_wrapper.py @@ -28,7 +28,6 @@ TEST_DS = TEST_DA.to_dataset() TEST_DS["test2"] = TEST_DA2 -EK_GRIB_READER = from_source("file", "tests/data/test_single.grib") EK_XARRAY_WRAPPER = from_object(TEST_DS) EK_NUMPY_WRAPPER = from_object(TEST_NP) @@ -59,6 +58,7 @@ def test_transform_function_inputs_reader_to_xarray(): # Check EK GribReader object + EK_GRIB_READER = from_source("file", "tests/data/test_single.grib") ek_reader_result = WRAPPED_XR_ONES_LIKE(EK_GRIB_READER) # Will return a DataSet becuase that is first value in kwarg_types assert isinstance(ek_reader_result, xr.Dataset) @@ -67,6 +67,7 @@ def test_transform_function_inputs_reader_to_xarray(): def test_transform_function_inputs_reader_to_xarray_typesetting(): # Check EK GribReader object + EK_GRIB_READER = from_source("file", "tests/data/test_single.grib") ek_reader_result = WRAPPED_XR_ONES_LIKE_TYPE_SETTING(EK_GRIB_READER) # Will return a dataarray because that is first value in type-set Union assert isinstance(ek_reader_result, xr.DataArray) @@ -75,6 +76,7 @@ def test_transform_function_inputs_reader_to_xarray_typesetting(): def test_transform_module_inputs_reader_to_xarray(): # Check EK GribReader object + EK_GRIB_READER = from_source("file", "tests/data/test_single.grib") ek_reader_result = WRAPPED_DUMMY_MODULE.xarray_ones_like(EK_GRIB_READER) # Data array because type-setting of function has dataarray first assert isinstance(ek_reader_result, xr.DataArray) @@ -106,12 +108,14 @@ def test_transform_module_inputs_wrapper_to_xarray(): def test_transform_function_inputs_reader_to_numpy(): # Test with Earthkit.data GribReader object + EK_GRIB_READER = from_source("file", "tests/data/test_single.grib") assert WRAPPED_NP_MEAN(EK_GRIB_READER) == np.mean(EK_GRIB_READER.to_numpy()) assert isinstance(WRAPPED_NP_MEAN(EK_GRIB_READER), np.float64) def test_transform_function_inputs_reader_to_numpy_typesetting(): # Test with Earthkit.data GribReader object + EK_GRIB_READER = from_source("file", "tests/data/test_single.grib") result = WRAPPED_NP_MEAN_TYPE_SETTING(EK_GRIB_READER) assert result == np.mean(EK_GRIB_READER.to_numpy()) assert isinstance(result, np.float64) @@ -119,6 +123,7 @@ def test_transform_function_inputs_reader_to_numpy_typesetting(): def test_transform_module_inputs_reader_to_numpy(): # Test with Earthkit.data GribReader object + EK_GRIB_READER = from_source("file", "tests/data/test_single.grib") result = WRAPPED_DUMMY_MODULE.numpy_mean(EK_GRIB_READER) assert result == np.mean(EK_GRIB_READER.to_numpy()) assert isinstance(result, np.float64)