From 7573a97d99e18981ae56ddc83ce7549ece25b66d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?thorbjoernl=20=28Thorbj=C3=B8rn=29?= Date: Wed, 10 Jul 2024 11:25:16 +0000 Subject: [PATCH 1/8] WIP --- pyaerocom/aeroval/experiment_output.py | 90 ++++++++++++++------------ pyaerocom_env.yml | 1 + pyproject.toml | 1 + 3 files changed, 50 insertions(+), 42 deletions(-) diff --git a/pyaerocom/aeroval/experiment_output.py b/pyaerocom/aeroval/experiment_output.py index cffdab848..28d6da173 100644 --- a/pyaerocom/aeroval/experiment_output.py +++ b/pyaerocom/aeroval/experiment_output.py @@ -4,6 +4,8 @@ import shutil from collections import namedtuple +import aerovaldb + from pyaerocom import const from pyaerocom._lowlevel_helpers import DirLoc, StrType, TypeValidator, sort_dict_by_name from pyaerocom.aeroval.collections import ObsCollection @@ -16,7 +18,7 @@ var_ranges_defaults, var_web_info, ) -from pyaerocom.aeroval.json_utils import check_make_json, read_json, write_json +from pyaerocom.aeroval.json_utils import check_make_json, write_json from pyaerocom.aeroval.modelentry import ModelEntry from pyaerocom.aeroval.setupclasses import EvalSetup from pyaerocom.aeroval.varinfo_web import VarinfoWeb @@ -45,6 +47,8 @@ def __init__(self, proj_id: str, json_basedir: str): self.proj_id = proj_id self.json_basedir = json_basedir + self.avdb = aerovaldb.open(f"json_files:{json_basedir}") + @property def proj_dir(self) -> str: """Project directory""" @@ -66,7 +70,12 @@ def available_experiments(self) -> list: """ List of available experiments """ - return list(read_json(self.experiments_file)) + try: + experiments = list(self.avdb.get_experiments(self.proj_id)) + except FileNotFoundError: + experiments = [] + + return experiments class ExperimentOutput(ProjectOutput): @@ -159,7 +168,7 @@ def update_menu(self) -> None: """ avail = self._create_menu_dict() avail = self._sort_menu_entries(avail) - write_json(avail, self.menu_file, indent=4) + self.avdb.put_menu(avail, self.proj_id, self.exp_id) def update_interface(self) -> None: """ @@ -193,16 +202,22 @@ def update_interface(self) -> None: # AeroVal frontend needs periods to be set in config json file... # make sure they are self.cfg._check_time_config() - self.cfg.to_json(self.exp_dir) + self.avdb.put_config(self.cfg.json_repr(), self.proj_id, self.exp_id) def _sync_heatmaps_with_menu_and_regions(self) -> None: """ Synchronise content of heatmap json files with content of menu.json """ - menu = read_json(self.menu_file) - all_regions = read_json(self.regions_file) - for fp in self._get_json_output_files("hm"): - data = read_json(fp) + try: + menu = self.avdb.get_menu(self.proj_id, self.exp_id) + except FileNotFoundError: + menu = {} + try: + all_regions = self.avdb.get_regions(self.proj_id, self.exp_id) + except FileNotFoundError: + all_regions = {} + for fp in self.avdb.list_glob_stats(self.proj_id, self.exp_id): + data = self.avdb.get_by_uuid(fp) hm = {} for vardisp, info in menu.items(): obs_dict = info["obs"] @@ -222,7 +237,7 @@ def _sync_heatmaps_with_menu_and_regions(self) -> None: hm_data = self._check_hm_all_regions_avail(all_regions, hm_data) hm[vardisp][obs][vert_code][mod][modvar] = hm_data - write_json(hm, fp, ignore_nan=True) + self.avdb.put_by_uuid(hm, fp) def _check_hm_all_regions_avail(self, all_regions, hm_data) -> dict: if all([x in hm_data for x in all_regions]): @@ -385,7 +400,7 @@ def _check_clean_ts_file(self, fp) -> bool: return True try: - data = read_json(fp) + data = self.avdb.get_by_uuid(fp) except Exception: logger.exception(f"FATAL: detected corrupt json file: {fp}. Removing file...") os.remove(fp) @@ -448,20 +463,12 @@ def delete_experiment_data(self, also_coldata=True) -> None: Parameters ---------- - base_dir : str, optional - basic output direcory (containing subdirs of all projects) - proj_name : str, optional - name of project, if None, then this project is used - exp_name : str, optional - name experiment, if None, then this project is used also_coldata : bool if True and if output directory for colocated data is default and specific for input experiment ID, then also all associated colocated NetCDF files are deleted. Defaults to True. """ - if os.path.exists(self.exp_dir): - logger.info(f"Deleting everything under {self.exp_dir}") - shutil.rmtree(self.exp_dir) + self.avdb.rm_experiment_data(self.proj_id, self.exp_id) if also_coldata: coldir = self.cfg.path_manager.get_coldata_dir() @@ -526,7 +533,7 @@ def _get_cmap_info(self, var) -> list[float]: def _create_var_ranges_json(self) -> None: try: - ranges = read_json(self.var_ranges_file) + ranges = self.avdb.get_ranges(self.proj_id, self.exp_id) except FileNotFoundError: ranges = {} avail = self._results_summary() @@ -534,7 +541,7 @@ def _create_var_ranges_json(self) -> None: for var in all_vars: if not var in ranges or ranges[var]["scale"] == []: ranges[var] = self._get_cmap_info(var) - write_json(ranges, self.var_ranges_file, indent=4) + self.avdb.put_ranges(ranges, self.proj_id, self.exp_id) def _create_statistics_json(self) -> None: if self.cfg.statistics_opts.obs_only_stats: @@ -563,7 +570,8 @@ def _create_statistics_json(self) -> None: stats_info.update(obs_statistics_trend) else: stats_info.update(statistics_trend) - write_json(stats_info, self.statistics_file, indent=4) + + self.avdb.put_statistics(stats_info, self.proj_id, self.exp_id) def _get_var_name_and_type(self, var_name: str) -> VariableInfo: """Get menu name and type of observation variable @@ -801,15 +809,14 @@ def _sort_menu_entries(self, avail: dict) -> dict: new_sorted[var]["obs"][obs_name][vert_code] = models_sorted return new_sorted - def _add_entry_experiments_json(self, exp_id, data) -> None: - fp = self.experiments_file - current = read_json(fp) + def _add_entry_experiments_json(self, exp_id: str, data) -> None: + try: + current = self.avdb.get_experiments(self.proj_id) + except FileNotFoundError: + current = {} current[exp_id] = data - write_json( - current, - self.experiments_file, - indent=4, - ) + + self.avdb.put_experiments(current, self.proj_id) def _del_entry_experiments_json(self, exp_id) -> None: """ @@ -825,16 +832,15 @@ def _del_entry_experiments_json(self, exp_id) -> None: None """ - current = read_json(self.experiments_file) + try: + current = self.avdb.get_experiments(self.proj_id) + except FileNotFoundError: + current = {} try: del current[exp_id] except KeyError: logger.warning(f"no such experiment registered: {exp_id}") - write_json( - current, - self.experiments_file, - indent=4, - ) + self.avdb.put_experiments(current, self.proj_id) def reorder_experiments(self, exp_order=None) -> None: """Reorder experiment order in evaluation interface @@ -851,10 +857,10 @@ def reorder_experiments(self, exp_order=None) -> None: exp_order = [] elif not isinstance(exp_order, list): raise ValueError("need list as input") - current = read_json(self.experiments_file) + + try: + current = self.avdb.get_experiments(self.proj_id) + except FileNotFoundError: + current = {} current = sort_dict_by_name(current, pref_list=exp_order) - write_json( - current, - self.experiments_file, - indent=4, - ) + self.avdb.put_experiments(current, self.proj_id) diff --git a/pyaerocom_env.yml b/pyaerocom_env.yml index 7bb7f4504..7aa6b2514 100644 --- a/pyaerocom_env.yml +++ b/pyaerocom_env.yml @@ -31,6 +31,7 @@ dependencies: - geojsoncontour - geocoder_reverse_natural_earth >= 0.0.2 - pyaro >= 0.0.10 + - git+https://github.com/metno/aerovaldb.git@list-glob-stats ## testing - pytest >=7.4 - pytest-dependency diff --git a/pyproject.toml b/pyproject.toml index 92945085b..68f73e6b2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,6 +21,7 @@ classifiers = [ ] requires-python = ">=3.10" dependencies = [ + "aerovaldb@git+https://github.com/metno/aerovaldb.git@list-glob-stats", "scitools-iris>=3.8.1", "xarray>=2022.10.0", "cartopy>=0.21.1", From fad7a87bf6bfbd7c3ea1ae4af8250f777c8782bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?thorbjoernl=20=28Thorbj=C3=B8rn=29?= Date: Wed, 10 Jul 2024 11:41:20 +0000 Subject: [PATCH 2/8] WIP --- pyaerocom/aeroval/experiment_output.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyaerocom/aeroval/experiment_output.py b/pyaerocom/aeroval/experiment_output.py index 28d6da173..19e48dade 100644 --- a/pyaerocom/aeroval/experiment_output.py +++ b/pyaerocom/aeroval/experiment_output.py @@ -18,7 +18,7 @@ var_ranges_defaults, var_web_info, ) -from pyaerocom.aeroval.json_utils import check_make_json, write_json +from pyaerocom.aeroval.json_utils import check_make_json from pyaerocom.aeroval.modelentry import ModelEntry from pyaerocom.aeroval.setupclasses import EvalSetup from pyaerocom.aeroval.varinfo_web import VarinfoWeb @@ -420,7 +420,7 @@ def _check_clean_ts_file(self, fp) -> bool: modified = True logger.info(f"Removing data for model {mod_name} from ts file: {fp}") - write_json(data_new, fp) + self.avdb.put_by_uuid(data_new, fp) return modified def _clean_modelmap_files(self) -> list[str]: From e75217c770ba5dba8143c5bd1fe6862785dfe8af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?thorbjoernl=20=28Thorbj=C3=B8rn=29?= Date: Wed, 10 Jul 2024 13:28:13 +0000 Subject: [PATCH 3/8] Simplify try-except with default return values --- pyaerocom/aeroval/experiment_output.py | 41 +++++++------------------- 1 file changed, 11 insertions(+), 30 deletions(-) diff --git a/pyaerocom/aeroval/experiment_output.py b/pyaerocom/aeroval/experiment_output.py index 19e48dade..75e105669 100644 --- a/pyaerocom/aeroval/experiment_output.py +++ b/pyaerocom/aeroval/experiment_output.py @@ -70,12 +70,7 @@ def available_experiments(self) -> list: """ List of available experiments """ - try: - experiments = list(self.avdb.get_experiments(self.proj_id)) - except FileNotFoundError: - experiments = [] - - return experiments + return list(self.avdb.get_experiments(self.proj_id, default={})) class ExperimentOutput(ProjectOutput): @@ -208,14 +203,8 @@ def _sync_heatmaps_with_menu_and_regions(self) -> None: """ Synchronise content of heatmap json files with content of menu.json """ - try: - menu = self.avdb.get_menu(self.proj_id, self.exp_id) - except FileNotFoundError: - menu = {} - try: - all_regions = self.avdb.get_regions(self.proj_id, self.exp_id) - except FileNotFoundError: - all_regions = {} + menu = self.avdb.get_menu(self.proj_id, self.exp_id, default={}) + all_regions = self.avdb.get_regions(self.proj_id, self.exp_id, default={}) for fp in self.avdb.list_glob_stats(self.proj_id, self.exp_id): data = self.avdb.get_by_uuid(fp) hm = {} @@ -532,10 +521,8 @@ def _get_cmap_info(self, var) -> list[float]: return info def _create_var_ranges_json(self) -> None: - try: - ranges = self.avdb.get_ranges(self.proj_id, self.exp_id) - except FileNotFoundError: - ranges = {} + ranges = self.avdb.get_ranges(self.proj_id, self.exp_id, default={}) + avail = self._results_summary() all_vars = list(set(avail["ovar"] + avail["mvar"])) for var in all_vars: @@ -810,10 +797,8 @@ def _sort_menu_entries(self, avail: dict) -> dict: return new_sorted def _add_entry_experiments_json(self, exp_id: str, data) -> None: - try: - current = self.avdb.get_experiments(self.proj_id) - except FileNotFoundError: - current = {} + current = self.avdb.get_experiments(self.proj_id, default={}) + current[exp_id] = data self.avdb.put_experiments(current, self.proj_id) @@ -832,10 +817,8 @@ def _del_entry_experiments_json(self, exp_id) -> None: None """ - try: - current = self.avdb.get_experiments(self.proj_id) - except FileNotFoundError: - current = {} + current = self.avdb.get_experiments(self.proj_id, default={}) + try: del current[exp_id] except KeyError: @@ -858,9 +841,7 @@ def reorder_experiments(self, exp_order=None) -> None: elif not isinstance(exp_order, list): raise ValueError("need list as input") - try: - current = self.avdb.get_experiments(self.proj_id) - except FileNotFoundError: - current = {} + current = self.avdb.get_experiments(self.proj_id, default={}) + current = sort_dict_by_name(current, pref_list=exp_order) self.avdb.put_experiments(current, self.proj_id) From 572c5e0bc3f7a42ce54aa248cfb19061eb0fe066 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?thorbjoernl=20=28Thorbj=C3=B8rn=29?= Date: Thu, 11 Jul 2024 07:46:05 +0000 Subject: [PATCH 4/8] Remove check_make_json() --- pyaerocom/aeroval/experiment_output.py | 8 +------ pyaerocom/aeroval/json_utils.py | 31 ------------------------- tests/aeroval/test_experiment_output.py | 1 - tests/aeroval/test_json_utils.py | 14 ----------- 4 files changed, 1 insertion(+), 53 deletions(-) diff --git a/pyaerocom/aeroval/experiment_output.py b/pyaerocom/aeroval/experiment_output.py index 75e105669..1ab586ba5 100644 --- a/pyaerocom/aeroval/experiment_output.py +++ b/pyaerocom/aeroval/experiment_output.py @@ -18,12 +18,11 @@ var_ranges_defaults, var_web_info, ) -from pyaerocom.aeroval.json_utils import check_make_json from pyaerocom.aeroval.modelentry import ModelEntry from pyaerocom.aeroval.setupclasses import EvalSetup from pyaerocom.aeroval.varinfo_web import VarinfoWeb from pyaerocom.exceptions import EntryNotAvailable, VariableDefinitionError -from pyaerocom.stats.mda8.const import MDA8_INPUT_VARS, MDA8_OUTPUT_VARS +from pyaerocom.stats.mda8.const import MDA8_OUTPUT_VARS from pyaerocom.stats.stats import _init_stats_dummy from pyaerocom.variable_helpers import get_aliases @@ -62,7 +61,6 @@ def proj_dir(self) -> str: def experiments_file(self) -> str: """json file containing region specifications""" fp = os.path.join(self.proj_dir, "experiments.json") - fp = check_make_json(fp) return fp @property @@ -104,28 +102,24 @@ def exp_dir(self) -> str: def regions_file(self) -> str: """json file containing region specifications""" fp = os.path.join(self.exp_dir, "regions.json") - fp = check_make_json(fp) return fp @property def statistics_file(self) -> str: """json file containing region specifications""" fp = os.path.join(self.exp_dir, "statistics.json") - fp = check_make_json(fp) return fp @property def var_ranges_file(self) -> str: """json file containing region specifications""" fp = os.path.join(self.exp_dir, "ranges.json") - check_make_json(fp) return fp @property def menu_file(self) -> str: """json file containing region specifications""" fp = os.path.join(self.exp_dir, "menu.json") - check_make_json(fp) return fp @property diff --git a/pyaerocom/aeroval/json_utils.py b/pyaerocom/aeroval/json_utils.py index 6173f1989..40955ab57 100644 --- a/pyaerocom/aeroval/json_utils.py +++ b/pyaerocom/aeroval/json_utils.py @@ -89,34 +89,3 @@ def write_json(data_dict, file_path, **kwargs): data_dict = round_floats(in_data=data_dict) with open(file_path, "w") as f: simplejson.dump(data_dict, f, allow_nan=True, **kwargs) - - -def check_make_json(fp, indent=4) -> str: - """ - Make sure input json file exists - - Parameters - ---------- - fp : str - filepath to be checked (must end with .json) - indent : int - indentation of json file - - Raises - ------ - ValueError - if filepath does not exist. - - Returns - ------- - str - input filepath. - - """ - fp = str(fp) - if not fp.endswith(".json"): - raise ValueError("Input filepath must end with .json") - if not os.path.exists(fp): - logger.info(f"Creating empty json file: {fp}") - write_json({}, fp, indent=indent) - return fp diff --git a/tests/aeroval/test_experiment_output.py b/tests/aeroval/test_experiment_output.py index 4f65320f4..d7cd424e8 100644 --- a/tests/aeroval/test_experiment_output.py +++ b/tests/aeroval/test_experiment_output.py @@ -61,7 +61,6 @@ def test_ProjectOutput_experiments_file(tmp_path: Path): val = ProjectOutput("test", str(tmp_path)) path = tmp_path / "test" / "experiments.json" assert Path(val.experiments_file) == path - assert path.exists() def test_ProjectOutput_available_experiments(tmp_path: Path): diff --git a/tests/aeroval/test_json_utils.py b/tests/aeroval/test_json_utils.py index 50e2fe29b..30243148e 100644 --- a/tests/aeroval/test_json_utils.py +++ b/tests/aeroval/test_json_utils.py @@ -5,7 +5,6 @@ import pytest from pyaerocom.aeroval.json_utils import ( - check_make_json, read_json, round_floats, set_float_serialization_precision, @@ -77,16 +76,3 @@ def test_write_json_error(json_path: Path): with pytest.raises(TypeError) as e: write_json({"bla": 42}, json_path, bla=42) assert str(e.value).endswith("unexpected keyword argument 'bla'") - - -def test_check_make_json(json_path: Path): - json = check_make_json(json_path) - assert Path(json).exists() - - -def test_check_make_json_error(tmp_path: Path): - path = tmp_path / "bla.txt" - assert not path.exists() - with pytest.raises(ValueError) as e: - check_make_json(path) - assert str(e.value) == "Input filepath must end with .json" From 788ae0501ad0629d62b12cc7f7569f9dddf010bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?thorbjoernl=20=28Thorbj=C3=B8rn=29?= Date: Thu, 11 Jul 2024 08:02:50 +0000 Subject: [PATCH 5/8] Pin aerovaldb version --- pyaerocom_env.yml | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyaerocom_env.yml b/pyaerocom_env.yml index 7aa6b2514..daa4ec552 100644 --- a/pyaerocom_env.yml +++ b/pyaerocom_env.yml @@ -31,7 +31,7 @@ dependencies: - geojsoncontour - geocoder_reverse_natural_earth >= 0.0.2 - pyaro >= 0.0.10 - - git+https://github.com/metno/aerovaldb.git@list-glob-stats + - git+https://github.com/metno/aerovaldb.git@v0.0.4 ## testing - pytest >=7.4 - pytest-dependency diff --git a/pyproject.toml b/pyproject.toml index 68f73e6b2..3666219b1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,7 +21,7 @@ classifiers = [ ] requires-python = ">=3.10" dependencies = [ - "aerovaldb@git+https://github.com/metno/aerovaldb.git@list-glob-stats", + "aerovaldb@git+https://github.com/metno/aerovaldb.git@v0.0.4", "scitools-iris>=3.8.1", "xarray>=2022.10.0", "cartopy>=0.21.1", From fbe764cd0a2513e4a278aa0a9447a0719b718e63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?thorbjoernl=20=28Thorbj=C3=B8rn=29?= Date: Thu, 11 Jul 2024 08:54:13 +0000 Subject: [PATCH 6/8] fix: issue with avdb --- pyaerocom_env.yml | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyaerocom_env.yml b/pyaerocom_env.yml index daa4ec552..8078a9cf2 100644 --- a/pyaerocom_env.yml +++ b/pyaerocom_env.yml @@ -31,7 +31,7 @@ dependencies: - geojsoncontour - geocoder_reverse_natural_earth >= 0.0.2 - pyaro >= 0.0.10 - - git+https://github.com/metno/aerovaldb.git@v0.0.4 + - git+https://github.com/metno/aerovaldb.git@v0.0.5 ## testing - pytest >=7.4 - pytest-dependency diff --git a/pyproject.toml b/pyproject.toml index 3666219b1..aafd2af55 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,7 +21,7 @@ classifiers = [ ] requires-python = ">=3.10" dependencies = [ - "aerovaldb@git+https://github.com/metno/aerovaldb.git@v0.0.4", + "aerovaldb@git+https://github.com/metno/aerovaldb.git@v0.0.5", "scitools-iris>=3.8.1", "xarray>=2022.10.0", "cartopy>=0.21.1", From de16fad98defa9d1581f86bfde2511c59df6b319 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?thorbjoernl=20=28Thorbj=C3=B8rn=29?= Date: Fri, 12 Jul 2024 07:54:56 +0000 Subject: [PATCH 7/8] pin avdb @0.0.6 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index aafd2af55..59c8c7420 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,7 +21,7 @@ classifiers = [ ] requires-python = ">=3.10" dependencies = [ - "aerovaldb@git+https://github.com/metno/aerovaldb.git@v0.0.5", + "aerovaldb@git+https://github.com/metno/aerovaldb.git@v0.0.6", "scitools-iris>=3.8.1", "xarray>=2022.10.0", "cartopy>=0.21.1", From 816b2cccc4c2ccf0d546f4da61a21e3a68fadd58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?thorbjoernl=20=28Thorbj=C3=B8rn=29?= Date: Fri, 12 Jul 2024 08:04:07 +0000 Subject: [PATCH 8/8] Same as above but for env.yml --- pyaerocom_env.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyaerocom_env.yml b/pyaerocom_env.yml index 8078a9cf2..532f1b651 100644 --- a/pyaerocom_env.yml +++ b/pyaerocom_env.yml @@ -31,7 +31,7 @@ dependencies: - geojsoncontour - geocoder_reverse_natural_earth >= 0.0.2 - pyaro >= 0.0.10 - - git+https://github.com/metno/aerovaldb.git@v0.0.5 + - git+https://github.com/metno/aerovaldb.git@v0.0.6 ## testing - pytest >=7.4 - pytest-dependency