Use aerovaldb for writing json in ExperimentOutput

pyaerocom/aeroval/experiment_output.py

@@ -4,6 +4,8 @@
 import shutil
 from collections import namedtuple
 
+import aerovaldb
+
 from pyaerocom import const
 from pyaerocom._lowlevel_helpers import DirLoc, StrType, TypeValidator, sort_dict_by_name
 from pyaerocom.aeroval.collections import ObsCollection
@@ -16,12 +18,11 @@
     var_ranges_defaults,
     var_web_info,
 )
-from pyaerocom.aeroval.json_utils import check_make_json, read_json, write_json
 from pyaerocom.aeroval.modelentry import ModelEntry
 from pyaerocom.aeroval.setupclasses import EvalSetup
 from pyaerocom.aeroval.varinfo_web import VarinfoWeb
 from pyaerocom.exceptions import EntryNotAvailable, VariableDefinitionError
-from pyaerocom.stats.mda8.const import MDA8_INPUT_VARS, MDA8_OUTPUT_VARS
+from pyaerocom.stats.mda8.const import MDA8_OUTPUT_VARS
 from pyaerocom.stats.stats import _init_stats_dummy
 from pyaerocom.variable_helpers import get_aliases
 
@@ -45,6 +46,8 @@
         self.proj_id = proj_id
         self.json_basedir = json_basedir
 
+        self.avdb = aerovaldb.open(f"json_files:{json_basedir}")
+
     @property
     def proj_dir(self) -> str:
         """Project directory"""
@@ -58,15 +61,14 @@
     def experiments_file(self) -> str:
         """json file containing region specifications"""
         fp = os.path.join(self.proj_dir, "experiments.json")
-        fp = check_make_json(fp)
         return fp
 
     @property
     def available_experiments(self) -> list:
         """
         List of available experiments
         """
-        return list(read_json(self.experiments_file))
+        return list(self.avdb.get_experiments(self.proj_id, default={}))
 
 
 class ExperimentOutput(ProjectOutput):
@@ -100,28 +102,24 @@
     def regions_file(self) -> str:
         """json file containing region specifications"""
         fp = os.path.join(self.exp_dir, "regions.json")
-        fp = check_make_json(fp)
         return fp
 
     @property
     def statistics_file(self) -> str:
         """json file containing region specifications"""
         fp = os.path.join(self.exp_dir, "statistics.json")
-        fp = check_make_json(fp)
         return fp
 
     @property
     def var_ranges_file(self) -> str:
         """json file containing region specifications"""
         fp = os.path.join(self.exp_dir, "ranges.json")
-        check_make_json(fp)
         return fp
 
     @property
     def menu_file(self) -> str:
         """json file containing region specifications"""
         fp = os.path.join(self.exp_dir, "menu.json")
-        check_make_json(fp)
         return fp
 
     @property
@@ -159,7 +157,7 @@
         """
         avail = self._create_menu_dict()
         avail = self._sort_menu_entries(avail)
-        write_json(avail, self.menu_file, indent=4)
+        self.avdb.put_menu(avail, self.proj_id, self.exp_id)
 
     def update_interface(self) -> None:
         """
@@ -193,16 +191,16 @@
         # AeroVal frontend needs periods to be set in config json file...
         # make sure they are
         self.cfg._check_time_config()
-        self.cfg.to_json(self.exp_dir)
+        self.avdb.put_config(self.cfg.json_repr(), self.proj_id, self.exp_id)
 
     def _sync_heatmaps_with_menu_and_regions(self) -> None:
         """
         Synchronise content of heatmap json files with content of menu.json
         """
-        menu = read_json(self.menu_file)
-        all_regions = read_json(self.regions_file)
-        for fp in self._get_json_output_files("hm"):
-            data = read_json(fp)
+        menu = self.avdb.get_menu(self.proj_id, self.exp_id, default={})
+        all_regions = self.avdb.get_regions(self.proj_id, self.exp_id, default={})
+        for fp in self.avdb.list_glob_stats(self.proj_id, self.exp_id):
+            data = self.avdb.get_by_uuid(fp)
             hm = {}
             for vardisp, info in menu.items():
                 obs_dict = info["obs"]
@@ -222,7 +220,7 @@
                             hm_data = self._check_hm_all_regions_avail(all_regions, hm_data)
                             hm[vardisp][obs][vert_code][mod][modvar] = hm_data
 
-            write_json(hm, fp, ignore_nan=True)
+            self.avdb.put_by_uuid(hm, fp)
 
     def _check_hm_all_regions_avail(self, all_regions, hm_data) -> dict:
         if all([x in hm_data for x in all_regions]):
@@ -385,7 +383,7 @@
             return True
 
         try:
-            data = read_json(fp)
+            data = self.avdb.get_by_uuid(fp)
         except Exception:
             logger.exception(f"FATAL: detected corrupt json file: {fp}. Removing file...")
             os.remove(fp)
@@ -405,7 +403,7 @@
                 modified = True
                 logger.info(f"Removing data for model {mod_name} from ts file: {fp}")
 
-        write_json(data_new, fp)
+        self.avdb.put_by_uuid(data_new, fp)
         return modified
 
     def _clean_modelmap_files(self) -> list[str]:
@@ -448,20 +446,12 @@
 
         Parameters
         ----------
-        base_dir : str, optional
-            basic output direcory (containing subdirs of all projects)
-        proj_name : str, optional
-            name of project, if None, then this project is used
-        exp_name : str, optional
-            name experiment, if None, then this project is used
         also_coldata : bool
             if True and if output directory for colocated data is default and
             specific for input experiment ID, then also all associated colocated
             NetCDF files are deleted. Defaults to True.
         """
-        if os.path.exists(self.exp_dir):
-            logger.info(f"Deleting everything under {self.exp_dir}")
-            shutil.rmtree(self.exp_dir)
+        self.avdb.rm_experiment_data(self.proj_id, self.exp_id)
 
         if also_coldata:
             coldir = self.cfg.path_manager.get_coldata_dir()
@@ -525,16 +515,14 @@
         return info
 
     def _create_var_ranges_json(self) -> None:
-        try:
-            ranges = read_json(self.var_ranges_file)
-        except FileNotFoundError:
-            ranges = {}
+        ranges = self.avdb.get_ranges(self.proj_id, self.exp_id, default={})
+
         avail = self._results_summary()
         all_vars = list(set(avail["ovar"] + avail["mvar"]))
         for var in all_vars:
             if not var in ranges or ranges[var]["scale"] == []:
                 ranges[var] = self._get_cmap_info(var)
-        write_json(ranges, self.var_ranges_file, indent=4)
+        self.avdb.put_ranges(ranges, self.proj_id, self.exp_id)
 
     def _create_statistics_json(self) -> None:
         if self.cfg.statistics_opts.obs_only_stats:
@@ -563,7 +551,8 @@
                 stats_info.update(obs_statistics_trend)
             else:
                 stats_info.update(statistics_trend)
-        write_json(stats_info, self.statistics_file, indent=4)
+
+        self.avdb.put_statistics(stats_info, self.proj_id, self.exp_id)
 
     def _get_var_name_and_type(self, var_name: str) -> VariableInfo:
         """Get menu name and type of observation variable
@@ -801,15 +790,12 @@
                     new_sorted[var]["obs"][obs_name][vert_code] = models_sorted
         return new_sorted
 
-    def _add_entry_experiments_json(self, exp_id, data) -> None:
-        fp = self.experiments_file
-        current = read_json(fp)
+    def _add_entry_experiments_json(self, exp_id: str, data) -> None:
+        current = self.avdb.get_experiments(self.proj_id, default={})
+
         current[exp_id] = data
-        write_json(
-            current,
-            self.experiments_file,
-            indent=4,
-        )
+
+        self.avdb.put_experiments(current, self.proj_id)
 
     def _del_entry_experiments_json(self, exp_id) -> None:
         """
@@ -825,16 +811,13 @@
         None
 
         """
-        current = read_json(self.experiments_file)
+        current = self.avdb.get_experiments(self.proj_id, default={})
+
         try:
             del current[exp_id]
         except KeyError:
             logger.warning(f"no such experiment registered: {exp_id}")
-        write_json(
-            current,
-            self.experiments_file,
-            indent=4,
-        )
+        self.avdb.put_experiments(current, self.proj_id)
 
     def reorder_experiments(self, exp_order=None) -> None:
         """Reorder experiment order in evaluation interface
@@ -851,10 +834,8 @@
             exp_order = []
         elif not isinstance(exp_order, list):
             raise ValueError("need list as input")
-        current = read_json(self.experiments_file)
+
+        current = self.avdb.get_experiments(self.proj_id, default={})
+
         current = sort_dict_by_name(current, pref_list=exp_order)
-        write_json(
-            current,
-            self.experiments_file,
-            indent=4,
-        )
+        self.avdb.put_experiments(current, self.proj_id)

pyaerocom/aeroval/json_utils.py

@@ -89,34 +89,3 @@ def write_json(data_dict, file_path, **kwargs):
         data_dict = round_floats(in_data=data_dict)
     with open(file_path, "w") as f:
         simplejson.dump(data_dict, f, allow_nan=True, **kwargs)
-
-
-def check_make_json(fp, indent=4) -> str:
-    """
-    Make sure input json file exists
-
-    Parameters
-    ----------
-    fp : str
-        filepath to be checked (must end with .json)
-    indent : int
-        indentation of json file
-
-    Raises
-    ------
-    ValueError
-        if filepath does not exist.
-
-    Returns
-    -------
-    str
-        input filepath.
-
-    """
-    fp = str(fp)
-    if not fp.endswith(".json"):
-        raise ValueError("Input filepath must end with .json")
-    if not os.path.exists(fp):
-        logger.info(f"Creating empty json file: {fp}")
-        write_json({}, fp, indent=indent)
-    return fp

pyaerocom_env.yml

@@ -31,6 +31,7 @@ dependencies:
     - geojsoncontour
     - geocoder_reverse_natural_earth >= 0.0.2
     - pyaro >= 0.0.10
+    - git+https://github.com/metno/aerovaldb.git@v0.0.6
 ## testing
   - pytest >=7.4
   - pytest-dependency

pyproject.toml

@@ -21,6 +21,7 @@ classifiers = [
 ]
 requires-python = ">=3.10"
 dependencies = [
+    "aerovaldb@git+https://github.com/metno/aerovaldb.git@v0.0.6",
     "scitools-iris>=3.8.1",
     "xarray>=2022.10.0",
     "cartopy>=0.21.1",

tests/aeroval/test_experiment_output.py

@@ -61,7 +61,6 @@ def test_ProjectOutput_experiments_file(tmp_path: Path):
     val = ProjectOutput("test", str(tmp_path))
     path = tmp_path / "test" / "experiments.json"
     assert Path(val.experiments_file) == path
-    assert path.exists()
 
 
 def test_ProjectOutput_available_experiments(tmp_path: Path):

tests/aeroval/test_json_utils.py

@@ -5,7 +5,6 @@
 import pytest
 
 from pyaerocom.aeroval.json_utils import (
-    check_make_json,
     read_json,
     round_floats,
     set_float_serialization_precision,
@@ -77,16 +76,3 @@ def test_write_json_error(json_path: Path):
     with pytest.raises(TypeError) as e:
         write_json({"bla": 42}, json_path, bla=42)
     assert str(e.value).endswith("unexpected keyword argument 'bla'")
-
-
-def test_check_make_json(json_path: Path):
-    json = check_make_json(json_path)
-    assert Path(json).exists()
-
-
-def test_check_make_json_error(tmp_path: Path):
-    path = tmp_path / "bla.txt"
-    assert not path.exists()
-    with pytest.raises(ValueError) as e:
-        check_make_json(path)
-    assert str(e.value) == "Input filepath must end with .json"