tests: test data on s3

ecml_tools/create/functions/__init__.py

@@ -38,8 +38,16 @@ def load_source(self, *args, **kwargs):
             return None
         path = os.path.join(self.read_directory, self.get_filename([args, kwargs]))
         if os.path.exists(path):
-            print(f"Loading to {path} for {args}, {kwargs}")
+            print(f"Loading path {path} for {args}, {kwargs}")
             return load_source("file", path)
+        elif path.startswith("http"):
+            import requests
+
+            print(f"Loading url {path} for {args}, {kwargs}")
+            try:
+                return load_source("url", path)
+            except requests.exceptions.HTTPError:
+                pass
         return None
 
 

tests/create/test_create.py

@@ -12,6 +12,7 @@
 import pytest
 
 from ecml_tools.create import Creator
+from ecml_tools.create.functions import enable_read_mars
 from ecml_tools.data import open_dataset
 from ecml_tools.data import open_zarr
 
@@ -29,6 +30,7 @@
 assert NAMES, "No yaml files found in " + HERE
 
 TEST_DATA_ROOT = "https://object-store.os-api.cci1.ecmwf.int/ml-tests/test-data/anemoi-datasets/create/"
+enable_read_mars("https://object-store.os-api.cci1.ecmwf.int/ml-tests/test-data/anemoi-datasets/create")
 
 
 def compare_dot_zattrs(a, b):