diff --git a/pyaerocom/aeroval/experiment_output.py b/pyaerocom/aeroval/experiment_output.py index cffdab848..1ab586ba5 100644 --- a/pyaerocom/aeroval/experiment_output.py +++ b/pyaerocom/aeroval/experiment_output.py @@ -4,6 +4,8 @@ import shutil from collections import namedtuple +import aerovaldb + from pyaerocom import const from pyaerocom._lowlevel_helpers import DirLoc, StrType, TypeValidator, sort_dict_by_name from pyaerocom.aeroval.collections import ObsCollection @@ -16,12 +18,11 @@ var_ranges_defaults, var_web_info, ) -from pyaerocom.aeroval.json_utils import check_make_json, read_json, write_json from pyaerocom.aeroval.modelentry import ModelEntry from pyaerocom.aeroval.setupclasses import EvalSetup from pyaerocom.aeroval.varinfo_web import VarinfoWeb from pyaerocom.exceptions import EntryNotAvailable, VariableDefinitionError -from pyaerocom.stats.mda8.const import MDA8_INPUT_VARS, MDA8_OUTPUT_VARS +from pyaerocom.stats.mda8.const import MDA8_OUTPUT_VARS from pyaerocom.stats.stats import _init_stats_dummy from pyaerocom.variable_helpers import get_aliases @@ -45,6 +46,8 @@ def __init__(self, proj_id: str, json_basedir: str): self.proj_id = proj_id self.json_basedir = json_basedir + self.avdb = aerovaldb.open(f"json_files:{json_basedir}") + @property def proj_dir(self) -> str: """Project directory""" @@ -58,7 +61,6 @@ def proj_dir(self) -> str: def experiments_file(self) -> str: """json file containing region specifications""" fp = os.path.join(self.proj_dir, "experiments.json") - fp = check_make_json(fp) return fp @property @@ -66,7 +68,7 @@ def available_experiments(self) -> list: """ List of available experiments """ - return list(read_json(self.experiments_file)) + return list(self.avdb.get_experiments(self.proj_id, default={})) class ExperimentOutput(ProjectOutput): @@ -100,28 +102,24 @@ def exp_dir(self) -> str: def regions_file(self) -> str: """json file containing region specifications""" fp = os.path.join(self.exp_dir, "regions.json") - fp = check_make_json(fp) return fp @property def statistics_file(self) -> str: """json file containing region specifications""" fp = os.path.join(self.exp_dir, "statistics.json") - fp = check_make_json(fp) return fp @property def var_ranges_file(self) -> str: """json file containing region specifications""" fp = os.path.join(self.exp_dir, "ranges.json") - check_make_json(fp) return fp @property def menu_file(self) -> str: """json file containing region specifications""" fp = os.path.join(self.exp_dir, "menu.json") - check_make_json(fp) return fp @property @@ -159,7 +157,7 @@ def update_menu(self) -> None: """ avail = self._create_menu_dict() avail = self._sort_menu_entries(avail) - write_json(avail, self.menu_file, indent=4) + self.avdb.put_menu(avail, self.proj_id, self.exp_id) def update_interface(self) -> None: """ @@ -193,16 +191,16 @@ def update_interface(self) -> None: # AeroVal frontend needs periods to be set in config json file... # make sure they are self.cfg._check_time_config() - self.cfg.to_json(self.exp_dir) + self.avdb.put_config(self.cfg.json_repr(), self.proj_id, self.exp_id) def _sync_heatmaps_with_menu_and_regions(self) -> None: """ Synchronise content of heatmap json files with content of menu.json """ - menu = read_json(self.menu_file) - all_regions = read_json(self.regions_file) - for fp in self._get_json_output_files("hm"): - data = read_json(fp) + menu = self.avdb.get_menu(self.proj_id, self.exp_id, default={}) + all_regions = self.avdb.get_regions(self.proj_id, self.exp_id, default={}) + for fp in self.avdb.list_glob_stats(self.proj_id, self.exp_id): + data = self.avdb.get_by_uuid(fp) hm = {} for vardisp, info in menu.items(): obs_dict = info["obs"] @@ -222,7 +220,7 @@ def _sync_heatmaps_with_menu_and_regions(self) -> None: hm_data = self._check_hm_all_regions_avail(all_regions, hm_data) hm[vardisp][obs][vert_code][mod][modvar] = hm_data - write_json(hm, fp, ignore_nan=True) + self.avdb.put_by_uuid(hm, fp) def _check_hm_all_regions_avail(self, all_regions, hm_data) -> dict: if all([x in hm_data for x in all_regions]): @@ -385,7 +383,7 @@ def _check_clean_ts_file(self, fp) -> bool: return True try: - data = read_json(fp) + data = self.avdb.get_by_uuid(fp) except Exception: logger.exception(f"FATAL: detected corrupt json file: {fp}. Removing file...") os.remove(fp) @@ -405,7 +403,7 @@ def _check_clean_ts_file(self, fp) -> bool: modified = True logger.info(f"Removing data for model {mod_name} from ts file: {fp}") - write_json(data_new, fp) + self.avdb.put_by_uuid(data_new, fp) return modified def _clean_modelmap_files(self) -> list[str]: @@ -448,20 +446,12 @@ def delete_experiment_data(self, also_coldata=True) -> None: Parameters ---------- - base_dir : str, optional - basic output direcory (containing subdirs of all projects) - proj_name : str, optional - name of project, if None, then this project is used - exp_name : str, optional - name experiment, if None, then this project is used also_coldata : bool if True and if output directory for colocated data is default and specific for input experiment ID, then also all associated colocated NetCDF files are deleted. Defaults to True. """ - if os.path.exists(self.exp_dir): - logger.info(f"Deleting everything under {self.exp_dir}") - shutil.rmtree(self.exp_dir) + self.avdb.rm_experiment_data(self.proj_id, self.exp_id) if also_coldata: coldir = self.cfg.path_manager.get_coldata_dir() @@ -525,16 +515,14 @@ def _get_cmap_info(self, var) -> list[float]: return info def _create_var_ranges_json(self) -> None: - try: - ranges = read_json(self.var_ranges_file) - except FileNotFoundError: - ranges = {} + ranges = self.avdb.get_ranges(self.proj_id, self.exp_id, default={}) + avail = self._results_summary() all_vars = list(set(avail["ovar"] + avail["mvar"])) for var in all_vars: if not var in ranges or ranges[var]["scale"] == []: ranges[var] = self._get_cmap_info(var) - write_json(ranges, self.var_ranges_file, indent=4) + self.avdb.put_ranges(ranges, self.proj_id, self.exp_id) def _create_statistics_json(self) -> None: if self.cfg.statistics_opts.obs_only_stats: @@ -563,7 +551,8 @@ def _create_statistics_json(self) -> None: stats_info.update(obs_statistics_trend) else: stats_info.update(statistics_trend) - write_json(stats_info, self.statistics_file, indent=4) + + self.avdb.put_statistics(stats_info, self.proj_id, self.exp_id) def _get_var_name_and_type(self, var_name: str) -> VariableInfo: """Get menu name and type of observation variable @@ -801,15 +790,12 @@ def _sort_menu_entries(self, avail: dict) -> dict: new_sorted[var]["obs"][obs_name][vert_code] = models_sorted return new_sorted - def _add_entry_experiments_json(self, exp_id, data) -> None: - fp = self.experiments_file - current = read_json(fp) + def _add_entry_experiments_json(self, exp_id: str, data) -> None: + current = self.avdb.get_experiments(self.proj_id, default={}) + current[exp_id] = data - write_json( - current, - self.experiments_file, - indent=4, - ) + + self.avdb.put_experiments(current, self.proj_id) def _del_entry_experiments_json(self, exp_id) -> None: """ @@ -825,16 +811,13 @@ def _del_entry_experiments_json(self, exp_id) -> None: None """ - current = read_json(self.experiments_file) + current = self.avdb.get_experiments(self.proj_id, default={}) + try: del current[exp_id] except KeyError: logger.warning(f"no such experiment registered: {exp_id}") - write_json( - current, - self.experiments_file, - indent=4, - ) + self.avdb.put_experiments(current, self.proj_id) def reorder_experiments(self, exp_order=None) -> None: """Reorder experiment order in evaluation interface @@ -851,10 +834,8 @@ def reorder_experiments(self, exp_order=None) -> None: exp_order = [] elif not isinstance(exp_order, list): raise ValueError("need list as input") - current = read_json(self.experiments_file) + + current = self.avdb.get_experiments(self.proj_id, default={}) + current = sort_dict_by_name(current, pref_list=exp_order) - write_json( - current, - self.experiments_file, - indent=4, - ) + self.avdb.put_experiments(current, self.proj_id) diff --git a/pyaerocom/aeroval/json_utils.py b/pyaerocom/aeroval/json_utils.py index 6173f1989..40955ab57 100644 --- a/pyaerocom/aeroval/json_utils.py +++ b/pyaerocom/aeroval/json_utils.py @@ -89,34 +89,3 @@ def write_json(data_dict, file_path, **kwargs): data_dict = round_floats(in_data=data_dict) with open(file_path, "w") as f: simplejson.dump(data_dict, f, allow_nan=True, **kwargs) - - -def check_make_json(fp, indent=4) -> str: - """ - Make sure input json file exists - - Parameters - ---------- - fp : str - filepath to be checked (must end with .json) - indent : int - indentation of json file - - Raises - ------ - ValueError - if filepath does not exist. - - Returns - ------- - str - input filepath. - - """ - fp = str(fp) - if not fp.endswith(".json"): - raise ValueError("Input filepath must end with .json") - if not os.path.exists(fp): - logger.info(f"Creating empty json file: {fp}") - write_json({}, fp, indent=indent) - return fp diff --git a/pyaerocom_env.yml b/pyaerocom_env.yml index 7bb7f4504..532f1b651 100644 --- a/pyaerocom_env.yml +++ b/pyaerocom_env.yml @@ -31,6 +31,7 @@ dependencies: - geojsoncontour - geocoder_reverse_natural_earth >= 0.0.2 - pyaro >= 0.0.10 + - git+https://github.com/metno/aerovaldb.git@v0.0.6 ## testing - pytest >=7.4 - pytest-dependency diff --git a/pyproject.toml b/pyproject.toml index 92945085b..59c8c7420 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,6 +21,7 @@ classifiers = [ ] requires-python = ">=3.10" dependencies = [ + "aerovaldb@git+https://github.com/metno/aerovaldb.git@v0.0.6", "scitools-iris>=3.8.1", "xarray>=2022.10.0", "cartopy>=0.21.1", diff --git a/tests/aeroval/test_experiment_output.py b/tests/aeroval/test_experiment_output.py index 4f65320f4..d7cd424e8 100644 --- a/tests/aeroval/test_experiment_output.py +++ b/tests/aeroval/test_experiment_output.py @@ -61,7 +61,6 @@ def test_ProjectOutput_experiments_file(tmp_path: Path): val = ProjectOutput("test", str(tmp_path)) path = tmp_path / "test" / "experiments.json" assert Path(val.experiments_file) == path - assert path.exists() def test_ProjectOutput_available_experiments(tmp_path: Path): diff --git a/tests/aeroval/test_json_utils.py b/tests/aeroval/test_json_utils.py index 50e2fe29b..30243148e 100644 --- a/tests/aeroval/test_json_utils.py +++ b/tests/aeroval/test_json_utils.py @@ -5,7 +5,6 @@ import pytest from pyaerocom.aeroval.json_utils import ( - check_make_json, read_json, round_floats, set_float_serialization_precision, @@ -77,16 +76,3 @@ def test_write_json_error(json_path: Path): with pytest.raises(TypeError) as e: write_json({"bla": 42}, json_path, bla=42) assert str(e.value).endswith("unexpected keyword argument 'bla'") - - -def test_check_make_json(json_path: Path): - json = check_make_json(json_path) - assert Path(json).exists() - - -def test_check_make_json_error(tmp_path: Path): - path = tmp_path / "bla.txt" - assert not path.exists() - with pytest.raises(ValueError) as e: - check_make_json(path) - assert str(e.value) == "Input filepath must end with .json"