Feature/merge from main

docs/release_notes/version_0.11_updates.rst

@@ -2,6 +2,29 @@ Version 0.11 Updates
 /////////////////////////
 
 
+Version 0.11.3
+===============
+
+Fixes
+++++++
+
+- Fixed issue when :py:meth:`to_numpy` could not be called on a FieldList (:pr:`551`).
+- Fixed issue when the :ref:`data-sources-lod` source did not raise an exception when the geography was not correctly specified (:pr:`549`).
+- Increased the minimum version of ``multiurl`` to 0.3.3
+
+
+
+
+Version 0.11.2
+===============
+
+Fixes
+++++++
+
+- Allowed encoding PL arrays for :py:class:`GribCoder` (:pr:`546`).
+
+
+
 Version 0.11.1
 ===============
 

environment.yml

@@ -22,7 +22,7 @@ dependencies:
 - cdsapi>=0.7.2
 - hda
 - pip:
-  - multiurl>=0.3.0
+  - multiurl>=0.3.3
   - pyfdb>=0.1.0
   - ecmwf-opendata>=0.1.2
   - polytope-client>=0.7.4
@@ -56,3 +56,4 @@ dependencies:
 - nbsphinx
 - ipykernel
 - geopandas
+- array-api-compat

pyproject.toml

@@ -38,7 +38,7 @@ dependencies = [
   "jsonschema",
   "lru-dict",
   "markdown",
-  "multiurl>=0.3.1",
+  "multiurl>=0.3.3",
   "netcdf4",
   "pandas",
   "pdbufr>=0.11",

src/earthkit/data/core/fieldlist.py

@@ -1045,7 +1045,7 @@ def _vals(f):
         if n > 0:
             it = iter(self)
             first = next(it)
-            is_property = isinstance(getattr(first.__class__, accessor), property)
+            is_property = not callable(getattr(first, accessor, None))
             vals = _vals(first)
             ns = array_namespace(vals)
             shape = (n, *vals.shape)

src/earthkit/data/utils/metadata/dict.py

@@ -53,9 +53,9 @@ def make_geography(metadata):
         if lon is None:
             raise ValueError("No longitudes or distinctLongitudes found")
         if len(lat.shape) != 1:
-            return ValueError(f"distinct latitudes must be 1D array! shape={lat.shape} unsupported")
+            raise ValueError(f"distinct latitudes must be 1D array! shape={lat.shape} unsupported")
         if len(lon.shape) != 1:
-            return ValueError(f"distinctLongitudes must be 1D array! shape={lon.shape} unsupported")
+            raise ValueError(f"distinctLongitudes must be 1D array! shape={lon.shape} unsupported")
         if lat.size * lon.size != val.size:
             raise ValueError(
                 (
@@ -70,11 +70,11 @@ def make_geography(metadata):
         lon = np.asarray(lon, dtype=float)
         if lat.size * lon.size == val.size:
             if len(lat.shape) != 1:
-                return ValueError(
+                raise ValueError(
                     f"latitudes must be a 1D array when holding distinct values! shape={lat.shape} unsupported"
                 )
             if len(lon.shape) != 1:
-                return ValueError(
+                raise ValueError(
                     f"longitudes must be a 1D array when holding distinct values! shape={lon.shape} unsupported"
                 )
             distinct = True

tests/ai-models/test_ai_warpped_field.py

@@ -0,0 +1,76 @@
+#!/usr/bin/env python3
+
+# (C) Copyright 2020 ECMWF.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+#
+
+import logging
+
+import numpy as np
+
+from earthkit.data import SimpleFieldList
+from earthkit.data import from_source
+from earthkit.data.testing import earthkit_examples_file
+
+LOG = logging.getLogger(__name__)
+
+
+class WrappedField:
+    def __init__(self, field):
+        self._field = field
+
+    def __getattr__(self, name):
+        return getattr(self._field, name)
+
+    def __repr__(self) -> str:
+        return repr(self._field)
+
+
+class NewDataField(WrappedField):
+    def __init__(self, field, data):
+        super().__init__(field)
+        self._data = data
+        self.shape = data.shape
+
+    def to_numpy(self, flatten=False, dtype=None, index=None):
+        data = self._data
+        if dtype is not None:
+            data = data.astype(dtype)
+        if flatten:
+            data = data.flatten()
+        if index is not None:
+            data = data[index]
+        return data
+
+
+class NewMetadataField(WrappedField):
+    def __init__(self, field, **kwargs):
+        super().__init__(field)
+        self._metadata = kwargs
+
+    def metadata(self, *args, **kwargs):
+        if len(args) == 1 and args[0] in self._metadata:
+            return self._metadata[args[0]]
+        return self._field.metadata(*args, **kwargs)
+
+
+def test_ai_models_wrapped_field_grib():
+
+    ds = from_source("file", earthkit_examples_file("test.grib"))
+    np_ref = ds.to_numpy()
+    vals_ref = ds.values
+
+    f1 = NewMetadataField(ds[0], shortName="sstk")
+    f2 = NewMetadataField(ds[1], shortName="2d")
+
+    r = SimpleFieldList([f1, f2])
+
+    assert r[0].metadata("shortName") == "sstk"
+    assert r[1].metadata("shortName") == "2d"
+    assert np.allclose(r.to_numpy(), np_ref)
+    assert np.allclose(r.values, vals_ref)

tests/environment-unit-tests.yml

@@ -24,7 +24,7 @@ dependencies:
 - hda
 - jsonschema
 - pip:
-  - multiurl>=0.3.0
+  - multiurl>=0.3.3
   - pyfdb>=0.1.0
   - ecmwf-opendata>=0.3.3
   - polytope-client>=0.7.4
@@ -58,3 +58,4 @@ dependencies:
 - nbsphinx
 - ipykernel
 - geopandas
+- array-api-compat