Tests: Timeaverage Unit tests

.github/workflows/CI-test.yaml

@@ -35,9 +35,16 @@ jobs:
           flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
           # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
           flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
-      - name: Test with pytest
+      - name: Test if data will work (Meta-Test)
         run: |
-          pytest -v --cov
+          # -s turns on printing to screen, even for passing tests
+          pytest -vvv --cov -s tests/meta/*.py
+      - name: Test with pytest (Unit)
+        run: |
+          pytest -vvv --cov tests/unit/*.py
+      - name: Test with pytest (Integration)
+        run: |
+          pytest -vvv --cov tests/integration/*.py
       - name: Test with doctest
         run: |
           PYTHONPATH=src pytest -v --doctest-modules src/

src/pymorize/cmorizer.py

@@ -220,6 +220,7 @@ def _post_init_inherit_rules(self):
                 rule.set(rule_attr, rule_value)
 
     def validate(self):
+        """Performs validation on files if they are suitable for use with the pipeline requirements"""
         # Sanity Checks:
         # :PS: @PG the following functions are not defined yet
         # self._check_rules_for_table()

src/pymorize/timeaverage.py

@@ -36,7 +36,8 @@
 
 import pandas as pd
 import xarray as xr
-
+import numpy as np
+import functools
 from .logging import logger
 
 
@@ -135,23 +136,19 @@ def _frequency_from_approx_interval(interval: str):
         ("year", lambda x: f"{x}YE", 365),
         ("month", lambda x: f"{x}ME", 30),
         ("day", lambda x: f"{x}D", 1),
-        ("hour", lambda x: f"{x}H", 24),
-        ("minute", lambda x: f"{x}min", 24 * 60),
-        ("second", lambda x: f"{x}s", 24 * 60 * 60),
-        ("millisecond", lambda x: f"{x}ms", 24 * 60 * 60 * 1000),
+        ("hour", lambda x: f"{x}H", 1 / 24),
+        ("minute", lambda x: f"{x}min", 1.0 / (24 * 60)),
+        ("second", lambda x: f"{x}s", 1.0 / (24 * 60 * 60)),
+        ("millisecond", lambda x: f"{x}ms", 1.0 / (24 * 60 * 60 * 1000)),
     ]
     try:
         interval = float(interval)
     except ValueError:
         return interval
-    to_divide = {"decade", "year", "month", "day"}
+    isclose = functools.partial(np.isclose, rtol=1e-3)
     for name, func, val in notation:
-        if name in to_divide:
-            value = interval // val
-        else:
-            value = interval * val
-        if value >= 1:
-            value = round(value)
+        if (interval >= val) or isclose(interval, val):
+            value = round(interval / val)
             value = "" if value == 1 else value
             return func(value)
 
@@ -313,7 +310,5 @@ def compute_average(da: xr.DataArray, rule):
 time: mean
 time: mean grid_longitude: mean
 time: point
-""".strip().split(
-    "\n"
-)
+""".strip().split("\n")
 """list: cell_methods to ignore when calculating time averages"""

tests/integration/test_basic_pipeline.py

@@ -10,6 +10,14 @@
 from pymorize.cmorizer import CMORizer
 from pymorize.logging import logger
 
+def test_init(test_config):
+    logger.info(f"Processing {test_config}")
+    with open(test_config, "r") as f:
+        cfg = yaml.safe_load(f)
+    CMORizer.from_dict(cfg)
+    # If we get this far, it was possible to construct
+    # the object, so this test passes:
+    assert True
 
 @pytest.mark.skipif(
     shutil.which("sbatch") is None, reason="sbatch is not available on this host"
@@ -22,8 +30,3 @@ def test_process(test_config):
     cmorizer.process()
 
 
-def test_init(test_config):
-    logger.info(f"Processing {test_config}")
-    with open(test_config, "r") as f:
-        cfg = yaml.safe_load(f)
-    cmorizer = CMORizer.from_dict(cfg)

tests/integration/test_fesom_2p6_pimesh_esm_tools.py

@@ -5,6 +5,21 @@
 from pymorize.logging import logger
 
 
+def test_init(fesom_2p6_pimesh_esm_tools_config, fesom_2p6_pimesh_esm_tools_data):
+    logger.info(f"Processing {fesom_2p6_pimesh_esm_tools_config}")
+    with open(fesom_2p6_pimesh_esm_tools_config, "r") as f:
+        cfg = yaml.safe_load(f)
+    for rule in cfg["rules"]:
+        for input in rule["inputs"]:
+            input["path"] = input["path"].replace(
+                "REPLACE_ME", str(fesom_2p6_pimesh_esm_tools_data)
+            )
+    CMORizer.from_dict(cfg)
+    # If we get this far, it was possible to construct
+    # the object, so this test passes:
+    assert True
+
+
 def test_process(fesom_2p6_pimesh_esm_tools_config, fesom_2p6_pimesh_esm_tools_data):
     logger.info(f"Processing {fesom_2p6_pimesh_esm_tools_config}")
     with open(fesom_2p6_pimesh_esm_tools_config, "r") as f:

tests/meta/test_xarray_open_mfdataset.py

@@ -0,0 +1,13 @@
+import xarray as xr
+
+
+def test_open_fesom_2p6_pimesh_esm_tools(fesom_2p6_pimesh_esm_tools_data):
+    matching_files = [
+        f
+        for f in (fesom_2p6_pimesh_esm_tools_data / "outdata/fesom/").iterdir()
+        if f.name.startswith("temp.fesom")
+    ]
+    assert len(matching_files) > 0
+    print(matching_files)
+    ds = xr.open_mfdataset(matching_files)
+    print(ds)