only import ecCodes dependency where essential (#192)

* only import ecCodes dependency where essential

* Add ci tests with no ecCodes available

---------

Co-authored-by: Sandor Kertesz <Sandor.Kertesz@ecmwf.int>

.github/workflows/ci.yml

@@ -60,6 +60,11 @@ jobs:
     - name: Run tests
       run: |
         make unit-tests
+    - name: Run tests without eccodes
+      run: |
+        micromamba remove eccodes
+        python -m pytest -v -m 'no_eccodes'
+
 
   type-check:
     needs: [unit-tests]

earthkit/data/sources/file.py

@@ -15,7 +15,6 @@
 from earthkit.data import from_source
 from earthkit.data.core.caching import CACHE
 from earthkit.data.readers import reader
-from earthkit.data.sources.file_indexed import FileIndexedSource
 
 from . import Source
 
@@ -53,6 +52,8 @@ def mutate(self):
 
         # here we must have a file or a directory
         if self._kwargs.get("indexing", False):
+            from earthkit.data.sources.file_indexed import FileIndexedSource
+
             kw = dict(self._kwargs)
             kw.pop("indexing", None)
             return FileIndexedSource(self.path, filter=filter, merger=self.merger, **kw)

pytest.ini

@@ -6,4 +6,5 @@ markers =
     ftp: test that used FTP. FTP is an old protocol and is not supported by most recent firewalls.
     notebook: testing notebooks can be slow. But needs to be performed to ensure that the documention is tested.
     no_cache_init: a test where the cache is not initialised. Must be run with --forked.
+    no_eccodes: a test which should pass when ecCodes is not installed
 testpaths = tests

tests/indexing/indexing_fixtures.py

@@ -11,10 +11,8 @@
 
 import os
 import shutil
-import warnings
 
 from earthkit.data.core.temporary import temp_directory, temp_file
-from earthkit.data.readers.grib.index import GribFieldList
 from earthkit.data.testing import (
     earthkit_examples_file,
     earthkit_file,
@@ -78,36 +76,6 @@ def list_of_dicts():
     ]
 
 
-class GribIndexFromDicts(GribFieldList):
-    def __init__(self, list_of_dicts, *args, **kwargs):
-        self.list_of_dicts = list_of_dicts
-        print(f"KWARGS={kwargs}")
-        super().__init__(*args, **kwargs)
-
-    def __getitem__(self, n):
-        class _VirtualGribField(dict):
-            def metadata(_self, n, **kwargs):
-                try:
-                    if n == "level":
-                        n = "levelist"
-                    if n == "shortName":
-                        n = "param"
-                    if n == "paramId":
-                        n = "_param_id"
-                    return _self[n]
-                except KeyError:
-                    warnings.warn("Cannot find all metadata keys.")
-
-            @property
-            def values(self, n):
-                return self["values"]
-
-        return _VirtualGribField(self.list_of_dicts[n])
-
-    def __len__(self):
-        return len(self.list_of_dicts)
-
-
 def get_tmp_fixture(input_mode):
     tmp = {
         "directory": unique_grib_dir,

tests/readers/test_netcdf_reader.py

@@ -32,16 +32,15 @@ def check_array(v, shape=None, first=None, last=None, meanv=None, eps=1e-3):
     assert np.isclose(v.mean(), meanv, eps)
 
 
-def test_netcdf():
-    for s in from_source("file", earthkit_file("docs/examples/test.nc")):
-        s is not None
-
-
-def test_dummy_netcdf_reader_1():
+@pytest.mark.no_eccodes
+def test_netcdf_reader():
     ds = from_source("file", earthkit_file("docs/examples/test.nc"))
     # assert str(ds).startswith("NetCDFReader"), r
     assert len(ds) == 2
-    assert isinstance(ds[1], NetCDFField), ds
+    assert isinstance(ds[0], NetCDFField)
+    assert isinstance(ds[1], NetCDFField)
+    for f in from_source("file", earthkit_file("docs/examples/test.nc")):
+        assert isinstance(f, NetCDFField)
 
 
 @pytest.mark.parametrize("attribute", ["coordinates", "bounds", "grid_mapping"])
@@ -130,6 +129,7 @@ def test_netcdf_multi_cds():
     source.to_xarray()
 
 
+@pytest.mark.no_eccodes
 def test_netcdf_multi_sources():
     path = earthkit_test_data_file("era5_2t_1.nc")
     s1 = from_source("file", path)
@@ -167,6 +167,7 @@ def test_netcdf_multi_sources():
     s3.to_xarray()
 
 
+@pytest.mark.no_eccodes
 def test_netcdf_multi_files():
     ds = from_source(
         "file",
@@ -199,6 +200,7 @@ def test_netcdf_multi_files():
     ds.to_xarray()
 
 
+@pytest.mark.no_eccodes
 def test_get_fields_missing_standard_name_attr_in_coord_array():
     """test _get_fields() can handle a missing 'standard_name' attr in coordinate data arrays"""
 

tests/utils/test_module_inputs_wrapper.py

@@ -28,7 +28,6 @@
 TEST_DS = TEST_DA.to_dataset()
 TEST_DS["test2"] = TEST_DA2
 
-EK_GRIB_READER = from_source("file", "tests/data/test_single.grib")
 EK_XARRAY_WRAPPER = from_object(TEST_DS)
 EK_NUMPY_WRAPPER = from_object(TEST_NP)
 
@@ -59,6 +58,7 @@
 
 def test_transform_function_inputs_reader_to_xarray():
     # Check EK GribReader object
+    EK_GRIB_READER = from_source("file", "tests/data/test_single.grib")
     ek_reader_result = WRAPPED_XR_ONES_LIKE(EK_GRIB_READER)
     # Will return a DataSet becuase that is first value in kwarg_types
     assert isinstance(ek_reader_result, xr.Dataset)
@@ -67,6 +67,7 @@ def test_transform_function_inputs_reader_to_xarray():
 
 def test_transform_function_inputs_reader_to_xarray_typesetting():
     # Check EK GribReader object
+    EK_GRIB_READER = from_source("file", "tests/data/test_single.grib")
     ek_reader_result = WRAPPED_XR_ONES_LIKE_TYPE_SETTING(EK_GRIB_READER)
     # Will return a dataarray because that is first value in type-set Union
     assert isinstance(ek_reader_result, xr.DataArray)
@@ -75,6 +76,7 @@ def test_transform_function_inputs_reader_to_xarray_typesetting():
 
 def test_transform_module_inputs_reader_to_xarray():
     # Check EK GribReader object
+    EK_GRIB_READER = from_source("file", "tests/data/test_single.grib")
     ek_reader_result = WRAPPED_DUMMY_MODULE.xarray_ones_like(EK_GRIB_READER)
     # Data array because type-setting of function has dataarray first
     assert isinstance(ek_reader_result, xr.DataArray)
@@ -106,19 +108,22 @@ def test_transform_module_inputs_wrapper_to_xarray():
 
 def test_transform_function_inputs_reader_to_numpy():
     # Test with Earthkit.data GribReader object
+    EK_GRIB_READER = from_source("file", "tests/data/test_single.grib")
     assert WRAPPED_NP_MEAN(EK_GRIB_READER) == np.mean(EK_GRIB_READER.to_numpy())
     assert isinstance(WRAPPED_NP_MEAN(EK_GRIB_READER), np.float64)
 
 
 def test_transform_function_inputs_reader_to_numpy_typesetting():
     # Test with Earthkit.data GribReader object
+    EK_GRIB_READER = from_source("file", "tests/data/test_single.grib")
     result = WRAPPED_NP_MEAN_TYPE_SETTING(EK_GRIB_READER)
     assert result == np.mean(EK_GRIB_READER.to_numpy())
     assert isinstance(result, np.float64)
 
 
 def test_transform_module_inputs_reader_to_numpy():
     # Test with Earthkit.data GribReader object
+    EK_GRIB_READER = from_source("file", "tests/data/test_single.grib")
     result = WRAPPED_DUMMY_MODULE.numpy_mean(EK_GRIB_READER)
     assert result == np.mean(EK_GRIB_READER.to_numpy())
     assert isinstance(result, np.float64)