From 4084c360075dd4fcd4acb932e72b3ca8f404d5ef Mon Sep 17 00:00:00 2001 From: Lewis Blake Date: Tue, 15 Oct 2024 12:58:10 +0200 Subject: [PATCH 01/27] starting --- pyaerocom/aeroval/collections.py | 2 +- pyaerocom/aeroval/experiment_output.py | 3 +- pyaerocom/aeroval/modelentry.py | 132 ++++++++++++++++++++++++- pyaerocom/aeroval/utils.py | 2 +- 4 files changed, 133 insertions(+), 6 deletions(-) diff --git a/pyaerocom/aeroval/collections.py b/pyaerocom/aeroval/collections.py index 96d442b28..dc67e3fc0 100644 --- a/pyaerocom/aeroval/collections.py +++ b/pyaerocom/aeroval/collections.py @@ -214,7 +214,7 @@ def get_entry(self, key) -> ModelEntry: """ try: entry = self[key] - entry["model_name"] = key + entry.model_name = key return entry except (KeyError, AttributeError): raise EntryNotAvailable(f"no such entry {key}") diff --git a/pyaerocom/aeroval/experiment_output.py b/pyaerocom/aeroval/experiment_output.py index 1dd43f9a6..9e6ce980e 100644 --- a/pyaerocom/aeroval/experiment_output.py +++ b/pyaerocom/aeroval/experiment_output.py @@ -784,9 +784,8 @@ def _create_menu_dict(self) -> dict: if mod_name not in new[var]["obs"][obs_name][vert_code]: new[var]["obs"][obs_name][vert_code][mod_name] = {} - model_id = mcfg["model_id"] new[var]["obs"][obs_name][vert_code][mod_name] = { - "model_id": model_id, + "model_id": mcfg.model_id, "model_var": mod_var, "obs_var": obs_var, } diff --git a/pyaerocom/aeroval/modelentry.py b/pyaerocom/aeroval/modelentry.py index b04f02ab0..1e37e37b4 100644 --- a/pyaerocom/aeroval/modelentry.py +++ b/pyaerocom/aeroval/modelentry.py @@ -1,11 +1,18 @@ import inspect from copy import deepcopy -from pyaerocom._lowlevel_helpers import BrowseDict, DictStrKeysListVals, DictType, StrType +from pydantic import BaseModel + +from pyaerocom._lowlevel_helpers import ( + BrowseDict, + DictStrKeysListVals, + DictType, + StrType, +) from pyaerocom.aeroval.aux_io_helpers import check_aux_info -class ModelEntry(BrowseDict): +class ModelEntry_old(BrowseDict): """Modeln configuration for evaluation (dictionary) Note @@ -138,3 +145,124 @@ def prep_dict_analysis(self, funs=None) -> dict: if self.aux_funs_required: output["model_read_aux"].update(self._get_aux_funcs_setup(funs)) return output + + +class ModelEntry(BaseModel): + """Model configuration for evaluation (BaseModel) + + Note + ----model_read_aux + Only :attr:`model_id` is mandatory, the rest is optional. + + Attributes + ---------- + model_id : str + ID of model run in AeroCom database (e.g. 'ECMWF_CAMS_REAN') + model_ts_type_read : str or dict, optional + may be specified to explicitly define the reading frequency of the + model data. Not to be confused with :attr:`ts_type`, which specifies + the frequency used for colocation. Can be specified variable specific + by providing a dictionary. + model_use_vars : dict + dictionary that specifies mapping of model variables. Keys are + observation variables, values are strings specifying the corresponding + model variable to be used + (e.g. model_use_vars=dict(od550aer='od550csaer')) + model_add_vars : dict + dictionary that specifies additional model variables. Keys are + observation variables, values are lists of strings specifying the + corresponding model variables to be used + (e.g. model_use_vars=dict(od550aer=['od550csaer', 'od550so4'])) + model_rename_vars : dict + key / value pairs specifying new variable names for model variables + in the output json files (is applied after co-location). + model_read_aux : dict + may be used to specify additional computation methods of variables from + models. Keys are obs variables, values are dictionaries with keys + `vars_required` (list of required variables for computation of var + and `fun` (method that takes list of read data objects and computes + and returns var) + """ + + # ## Pydantic ConfigDict + # model_config = ConfigDict( + # arbitrary_types_allowed=True, + # extra="allow", + # validate_assignment=True, + # ) + + model_id: str + model_name: str | None = None + model_use_vars: dict | None = None + model_add_vars: dict[str, tuple[str, ...]] | None = None + model_read_aux: dict | None = None + model_rename_vars: dict | None = None + + @property + def aux_funs_required(self): + """ + Boolean specifying whether this entry requires auxiliary variables + """ + return True if bool(self.model_read_aux) else False + + def json_repr(self) -> dict: + raise NotImplementedError + # return self.model_dump() + + def get_vars_to_process(self, obs_vars: tuple) -> tuple: + """ + Get lists of obs / mod variables to be processed + + Parameters + ---------- + obs_vars : tuple + tuple of observation variables + + Returns + ------- + list + list of observation variables (potentially extended from input + list) + list + corresponding model variables which are mapped based on content + of :attr:`model_add_vars` and :attr:`model_use_vars`. + + """ + obsout, modout = [], [] + for obsvar in obs_vars: + obsout.append(obsvar) + if obsvar in self.model_use_vars: + modout.append(self.model_use_vars[obsvar]) + else: + modout.append(obsvar) + + for ovar, mvars in self.model_add_vars.items(): + # if not isinstance(mvars, tuple): + # raise AttributeError( + # f"values of model_add_vars need to be lists, even if " + # f"only single variables are to be added: " + # f"{self.model_add_vars}" + # ) + for mvar in mvars: + obsout.append(ovar) + modout.append(mvar) + return (obsout, modout) + + def get_varname_web(self, mod_var, obs_var): + if obs_var in self.model_add_vars and mod_var in self.model_add_vars[obs_var]: + return mod_var + return obs_var + + def _get_aux_funcs_setup(self, funs): + mra = {} + for var, aux_info in self.model_read_aux.items(): + mra[var] = check_aux_info(funcs=funs, **aux_info) + return mra + + def prep_dict_analysis(self, funs=None) -> dict: + if funs is None: + funs = {} + output = deepcopy(self.model_dump()) + if self.aux_funs_required: + output["model_read_aux"].update(self._get_aux_funcs_setup(funs)) + return output diff --git a/pyaerocom/aeroval/utils.py b/pyaerocom/aeroval/utils.py index e6e50a027..8f680fe05 100644 --- a/pyaerocom/aeroval/utils.py +++ b/pyaerocom/aeroval/utils.py @@ -148,7 +148,7 @@ def compute_model_average_and_diversity( for mname in models: logger.info(f"Adding {mname} ({var_name})") - mid = cfg.cfg.model_cfg.get_entry(mname)["model_id"] + mid = cfg.cfg.model_cfg.get_entry(mname).model_id if mid == data_id or mname == data_id: continue From ac90e76168ab73f12578585cd116d36f4d93d06e Mon Sep 17 00:00:00 2001 From: Lewis Blake Date: Tue, 15 Oct 2024 14:03:25 +0200 Subject: [PATCH 02/27] passing all but three aeroval tests --- pyaerocom/aeroval/modelentry.py | 28 +++++++++++++++------------- pyaerocom/colocation/colocator.py | 14 +++++++++++--- 2 files changed, 26 insertions(+), 16 deletions(-) diff --git a/pyaerocom/aeroval/modelentry.py b/pyaerocom/aeroval/modelentry.py index 1e37e37b4..15f284eaa 100644 --- a/pyaerocom/aeroval/modelentry.py +++ b/pyaerocom/aeroval/modelentry.py @@ -1,7 +1,7 @@ import inspect from copy import deepcopy -from pydantic import BaseModel +from pydantic import BaseModel, ConfigDict from pyaerocom._lowlevel_helpers import ( BrowseDict, @@ -184,19 +184,22 @@ class ModelEntry(BaseModel): and returns var) """ - # ## Pydantic ConfigDict - # model_config = ConfigDict( - # arbitrary_types_allowed=True, - # extra="allow", - # validate_assignment=True, - # ) + ## Pydantic ConfigDict + model_config = ConfigDict( + # arbitrary_types_allowed=True, + # extra="allow", + validate_assignment=True, + protected_namespaces=(), + ) model_id: str + model_ts_type_read: str | dict = "" # TODO: see if can make None model_name: str | None = None - model_use_vars: dict | None = None - model_add_vars: dict[str, tuple[str, ...]] | None = None - model_read_aux: dict | None = None - model_rename_vars: dict | None = None + model_use_vars: dict = {} + model_add_vars: dict[str, tuple[str, ...]] = {} + model_read_aux: dict = {} + model_rename_vars: dict = {} + flex_ts_type: bool = False @property def aux_funs_required(self): @@ -206,8 +209,7 @@ def aux_funs_required(self): return True if bool(self.model_read_aux) else False def json_repr(self) -> dict: - raise NotImplementedError - # return self.model_dump() + return self.model_dump() def get_vars_to_process(self, obs_vars: tuple) -> tuple: """ diff --git a/pyaerocom/colocation/colocator.py b/pyaerocom/colocation/colocator.py index 61612065f..21fa3bf6e 100644 --- a/pyaerocom/colocation/colocator.py +++ b/pyaerocom/colocation/colocator.py @@ -22,7 +22,11 @@ colocate_gridded_ungridded, correct_model_stp_coldata, ) -from pyaerocom.exceptions import ColocationError, ColocationSetupError, DataCoverageError +from pyaerocom.exceptions import ( + ColocationError, + ColocationSetupError, + DataCoverageError, +) from pyaerocom.helpers import ( get_lowest_resolution, start_stop, @@ -373,7 +377,7 @@ def run(self, var_list: list = None): calc_mda8 = True data_out = defaultdict(lambda: dict()) - # ToDo: see if the following could be solved via custom context manager + # TODO: see if the following could be solved via custom context manager try: vars_to_process = self.prepare_run(var_list) except Exception as ex: @@ -401,7 +405,11 @@ def run(self, var_list: list = None): logger.debug(e) else: self._save_coldata(mda8) - logger.info("Successfully calculated mda8 for [%s, %s].", obs_var, mod_var) + logger.info( + "Successfully calculated mda8 for [%s, %s].", + obs_var, + mod_var, + ) data_out[f"{mod_var}mda8"][f"{obs_var}mda8"] = mda8 self._processing_status.append([mod_var, obs_var, 1]) From e57d3f971e51ff3876eef630708248a3c506a2fa Mon Sep 17 00:00:00 2001 From: Lewis Blake Date: Tue, 15 Oct 2024 14:33:45 +0200 Subject: [PATCH 03/27] WIP, breaks some cams2_83 stuff --- pyaerocom/_lowlevel_helpers.py | 35 ++-- pyaerocom/aeroval/modelentry.py | 280 ++++++++++++++++---------------- 2 files changed, 160 insertions(+), 155 deletions(-) diff --git a/pyaerocom/_lowlevel_helpers.py b/pyaerocom/_lowlevel_helpers.py index 18fb48deb..60a89b3a9 100644 --- a/pyaerocom/_lowlevel_helpers.py +++ b/pyaerocom/_lowlevel_helpers.py @@ -126,11 +126,11 @@ def validate(self, val): return val -class DictType(Validator): - def validate(self, val): - if not isinstance(val, dict): - raise ValueError(f"need dict, got {val}") - return val +# class DictType(Validator): +# def validate(self, val): +# if not isinstance(val, dict): +# raise ValueError(f"need dict, got {val}") +# return val class FlexList(Validator): @@ -170,15 +170,15 @@ def validate(self, val): return val -class DictStrKeysListVals(Validator): - def validate(self, val: dict): - if not isinstance(val, dict): - raise ValueError(f"need dict, got {val}") - if any(not isinstance(x, str) for x in val): - raise ValueError(f"all keys need to be str type in {val}") - if any(not isinstance(x, list) for x in val.values()): - raise ValueError(f"all values need to be list type in {val}") - return val +# class DictStrKeysListVals(Validator): +# def validate(self, val: dict): +# if not isinstance(val, dict): +# raise ValueError(f"need dict, got {val}") +# if any(not isinstance(x, str) for x in val): +# raise ValueError(f"all keys need to be str type in {val}") +# if any(not isinstance(x, list) for x in val.values()): +# raise ValueError(f"all values need to be list type in {val}") +# return val class Loc(abc.ABC): @@ -194,7 +194,12 @@ class Loc(abc.ABC): """ def __init__( - self, default=None, assert_exists=False, auto_create=False, tooltip=None, logger=None + self, + default=None, + assert_exists=False, + auto_create=False, + tooltip=None, + logger=None, ): self.assert_exists = assert_exists self.auto_create = auto_create diff --git a/pyaerocom/aeroval/modelentry.py b/pyaerocom/aeroval/modelentry.py index 15f284eaa..799f49f0f 100644 --- a/pyaerocom/aeroval/modelentry.py +++ b/pyaerocom/aeroval/modelentry.py @@ -1,150 +1,150 @@ -import inspect +# import inspect from copy import deepcopy from pydantic import BaseModel, ConfigDict -from pyaerocom._lowlevel_helpers import ( - BrowseDict, - DictStrKeysListVals, - DictType, - StrType, -) +# from pyaerocom._lowlevel_helpers import ( +# BrowseDict, +# # DictStrKeysListVals, +# # DictType, +# StrType, +# ) from pyaerocom.aeroval.aux_io_helpers import check_aux_info -class ModelEntry_old(BrowseDict): - """Modeln configuration for evaluation (dictionary) - - Note - ----model_read_aux - Only :attr:`model_id` is mandatory, the rest is optional. - - Attributes - ---------- - model_id : str - ID of model run in AeroCom database (e.g. 'ECMWF_CAMS_REAN') - model_ts_type_read : str or dict, optional - may be specified to explicitly define the reading frequency of the - model data. Not to be confused with :attr:`ts_type`, which specifies - the frequency used for colocation. Can be specified variable specific - by providing a dictionary. - model_use_vars : dict - dictionary that specifies mapping of model variables. Keys are - observation variables, values are strings specifying the corresponding - model variable to be used - (e.g. model_use_vars=dict(od550aer='od550csaer')) - model_add_vars : dict - dictionary that specifies additional model variables. Keys are - observation variables, values are lists of strings specifying the - corresponding model variables to be used - (e.g. model_use_vars=dict(od550aer=['od550csaer', 'od550so4'])) - model_rename_vars : dict - key / value pairs specifying new variable names for model variables - in the output json files (is applied after co-location). - model_read_aux : dict - may be used to specify additional computation methods of variables from - models. Keys are obs variables, values are dictionaries with keys - `vars_required` (list of required variables for computation of var - and `fun` (method that takes list of read data objects and computes - and returns var) - """ - - model_id = StrType() - model_use_vars = DictType() - model_add_vars = DictStrKeysListVals() - model_read_aux = DictType() - model_rename_vars = DictType() - - def __init__(self, model_id, **kwargs): - self.model_id = model_id - self.model_ts_type_read = "" - self.model_use_vars = {} - self.model_add_vars = {} - self.model_rename_vars = {} - self.model_read_aux = {} - - self.kwargs = kwargs - - self.update(**kwargs) - - @property - def aux_funs_required(self): - """ - Boolean specifying whether this entry requires auxiliary variables - """ - return True if bool(self.model_read_aux) else False - - def json_repr(self) -> dict: - sup_rep = super().json_repr() - - # a little hacky, but makes the cams2-82 configs work - try: - for key in sup_rep["model_read_aux"]: - sup_rep["model_read_aux"][key]["fun"] = inspect.getsource( - deepcopy(sup_rep["model_read_aux"][key]["fun"]) - ) - except TypeError: - pass - - return sup_rep - - def get_vars_to_process(self, obs_vars: list) -> tuple: - """ - Get lists of obs / mod variables to be processed - - Parameters - ---------- - obs_vars : list - list of observation variables - - Returns - ------- - list - list of observation variables (potentially extended from input - list) - list - corresponding model variables which are mapped based on content - of :attr:`model_add_vars` and :attr:`model_use_vars`. - - """ - obsout, modout = [], [] - for obsvar in obs_vars: - obsout.append(obsvar) - if obsvar in self.model_use_vars: - modout.append(self.model_use_vars[obsvar]) - else: - modout.append(obsvar) - - for ovar, mvars in self.model_add_vars.items(): - if not isinstance(mvars, list): - raise AttributeError( - f"values of model_add_vars need to be lists, even if " - f"only single variables are to be added: " - f"{self.model_add_vars}" - ) - for mvar in mvars: - obsout.append(ovar) - modout.append(mvar) - return (obsout, modout) - - def get_varname_web(self, mod_var, obs_var): - if obs_var in self.model_add_vars and mod_var in self.model_add_vars[obs_var]: - return mod_var - return obs_var - - def _get_aux_funcs_setup(self, funs): - mra = {} - for var, aux_info in self.model_read_aux.items(): - mra[var] = check_aux_info(funcs=funs, **aux_info) - return mra - - def prep_dict_analysis(self, funs=None) -> dict: - if funs is None: - funs = {} - output = deepcopy(self.to_dict()) - if self.aux_funs_required: - output["model_read_aux"].update(self._get_aux_funcs_setup(funs)) - return output +# class ModelEntry_old(BrowseDict): +# """Modeln configuration for evaluation (dictionary) + +# Note +# ----model_read_aux +# Only :attr:`model_id` is mandatory, the rest is optional. + +# Attributes +# ---------- +# model_id : str +# ID of model run in AeroCom database (e.g. 'ECMWF_CAMS_REAN') +# model_ts_type_read : str or dict, optional +# may be specified to explicitly define the reading frequency of the +# model data. Not to be confused with :attr:`ts_type`, which specifies +# the frequency used for colocation. Can be specified variable specific +# by providing a dictionary. +# model_use_vars : dict +# dictionary that specifies mapping of model variables. Keys are +# observation variables, values are strings specifying the corresponding +# model variable to be used +# (e.g. model_use_vars=dict(od550aer='od550csaer')) +# model_add_vars : dict +# dictionary that specifies additional model variables. Keys are +# observation variables, values are lists of strings specifying the +# corresponding model variables to be used +# (e.g. model_use_vars=dict(od550aer=['od550csaer', 'od550so4'])) +# model_rename_vars : dict +# key / value pairs specifying new variable names for model variables +# in the output json files (is applied after co-location). +# model_read_aux : dict +# may be used to specify additional computation methods of variables from +# models. Keys are obs variables, values are dictionaries with keys +# `vars_required` (list of required variables for computation of var +# and `fun` (method that takes list of read data objects and computes +# and returns var) +# """ + +# model_id = StrType() +# model_use_vars = DictType() +# model_add_vars = DictStrKeysListVals() +# model_read_aux = DictType() +# model_rename_vars = DictType() + +# def __init__(self, model_id, **kwargs): +# self.model_id = model_id +# self.model_ts_type_read = "" +# self.model_use_vars = {} +# self.model_add_vars = {} +# self.model_rename_vars = {} +# self.model_read_aux = {} + +# self.kwargs = kwargs + +# self.update(**kwargs) + +# @property +# def aux_funs_required(self): +# """ +# Boolean specifying whether this entry requires auxiliary variables +# """ +# return True if bool(self.model_read_aux) else False + +# def json_repr(self) -> dict: +# sup_rep = super().json_repr() + +# # a little hacky, but makes the cams2-82 configs work +# try: +# for key in sup_rep["model_read_aux"]: +# sup_rep["model_read_aux"][key]["fun"] = inspect.getsource( +# deepcopy(sup_rep["model_read_aux"][key]["fun"]) +# ) +# except TypeError: +# pass + +# return sup_rep + +# def get_vars_to_process(self, obs_vars: list) -> tuple: +# """ +# Get lists of obs / mod variables to be processed + +# Parameters +# ---------- +# obs_vars : list +# list of observation variables + +# Returns +# ------- +# list +# list of observation variables (potentially extended from input +# list) +# list +# corresponding model variables which are mapped based on content +# of :attr:`model_add_vars` and :attr:`model_use_vars`. + +# """ +# obsout, modout = [], [] +# for obsvar in obs_vars: +# obsout.append(obsvar) +# if obsvar in self.model_use_vars: +# modout.append(self.model_use_vars[obsvar]) +# else: +# modout.append(obsvar) + +# for ovar, mvars in self.model_add_vars.items(): +# if not isinstance(mvars, list): +# raise AttributeError( +# f"values of model_add_vars need to be lists, even if " +# f"only single variables are to be added: " +# f"{self.model_add_vars}" +# ) +# for mvar in mvars: +# obsout.append(ovar) +# modout.append(mvar) +# return (obsout, modout) + +# def get_varname_web(self, mod_var, obs_var): +# if obs_var in self.model_add_vars and mod_var in self.model_add_vars[obs_var]: +# return mod_var +# return obs_var + +# def _get_aux_funcs_setup(self, funs): +# mra = {} +# for var, aux_info in self.model_read_aux.items(): +# mra[var] = check_aux_info(funcs=funs, **aux_info) +# return mra + +# def prep_dict_analysis(self, funs=None) -> dict: +# if funs is None: +# funs = {} +# output = deepcopy(self.to_dict()) +# if self.aux_funs_required: +# output["model_read_aux"].update(self._get_aux_funcs_setup(funs)) +# return output class ModelEntry(BaseModel): From eb556577d155f11ddc93d58f8de3c946ded75daf Mon Sep 17 00:00:00 2001 From: lewisblake Date: Thu, 31 Oct 2024 14:19:56 +0000 Subject: [PATCH 04/27] model_ts_type_read allowed to be None --- pyaerocom/aeroval/modelentry.py | 2 +- pyaerocom/ungriddeddata.py | 35 ++++++++++++++++++++++++++++----- 2 files changed, 31 insertions(+), 6 deletions(-) diff --git a/pyaerocom/aeroval/modelentry.py b/pyaerocom/aeroval/modelentry.py index 799f49f0f..0d8e46db3 100644 --- a/pyaerocom/aeroval/modelentry.py +++ b/pyaerocom/aeroval/modelentry.py @@ -193,7 +193,7 @@ class ModelEntry(BaseModel): ) model_id: str - model_ts_type_read: str | dict = "" # TODO: see if can make None + model_ts_type_read: str | dict | None = "" # TODO: see if can make None model_name: str | None = None model_use_vars: dict = {} model_add_vars: dict[str, tuple[str, ...]] = {} diff --git a/pyaerocom/ungriddeddata.py b/pyaerocom/ungriddeddata.py index 7c32ad8c8..e547a064f 100644 --- a/pyaerocom/ungriddeddata.py +++ b/pyaerocom/ungriddeddata.py @@ -970,7 +970,11 @@ def to_station_data( continue if freq is not None: stat.resample_time( - var, freq, how=resample_how, min_num_obs=min_num_obs, inplace=True + var, + freq, + how=resample_how, + min_num_obs=min_num_obs, + inplace=True, ) elif insert_nans: stat.insert_nans_timeseries(var) @@ -1249,7 +1253,13 @@ def to_station_data_all( - longitude: list of longitude coordinates """ - out_data = {"stats": [], "station_name": [], "latitude": [], "failed": [], "longitude": []} + out_data = { + "stats": [], + "station_name": [], + "latitude": [], + "failed": [], + "longitude": [], + } _iter = self._generate_station_index(by_station_name, ignore_index) for idx in _iter: @@ -1575,7 +1585,13 @@ def set_flags_nan(self, inplace=False): # TODO: check, confirm and remove Beta version note in docstring def remove_outliers( - self, var_name, inplace=False, low=None, high=None, unit_ref=None, move_to_trash=True + self, + var_name, + inplace=False, + low=None, + high=None, + unit_ref=None, + move_to_trash=True, ): """Method that can be used to remove outliers from data @@ -1900,7 +1916,10 @@ def apply_filters(self, var_outlier_ranges=None, **filter_attributes): var_outlier_ranges = {} for var in data.contains_vars: - lower, upper = None, None # uses pyaerocom default specified in variables.ini + lower, upper = ( + None, + None, + ) # uses pyaerocom default specified in variables.ini if var in var_outlier_ranges: lower, upper = var_outlier_ranges[var] data = data.remove_outliers( @@ -2907,7 +2926,13 @@ def plot_station_coordinates( kwargs["label"] = info_str ax = plot_coordinates( - lons, lats, color=color, marker=marker, markersize=markersize, legend=legend, **kwargs + lons, + lats, + color=color, + marker=marker, + markersize=markersize, + legend=legend, + **kwargs, ) if "title" in kwargs: From e17bb210f4e5090d275d1b874fc854c30f6b02fd Mon Sep 17 00:00:00 2001 From: Lewis Blake Date: Tue, 15 Oct 2024 16:18:15 +0200 Subject: [PATCH 05/27] Update README.rst --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index d1d06e7c8..7e8cd315c 100644 --- a/README.rst +++ b/README.rst @@ -25,4 +25,4 @@ pyaerocom :target: https://codecov.io/gh/metno/pyaerocom .. image:: https://zenodo.org/badge/DOI/10.5281/zenodo.13927979.svg - :target: https://doi.org/10.5281/zenodo.13927979 + :target: https://doi.org/10.5281/zenodo.13927979 From f9f0ea15893a159b4e88f2a7cf1da9f8b88d1a46 Mon Sep 17 00:00:00 2001 From: Lewis Blake Date: Wed, 16 Oct 2024 14:12:31 +0200 Subject: [PATCH 06/27] Update README.rst --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 7e8cd315c..e709ed69d 100644 --- a/README.rst +++ b/README.rst @@ -24,5 +24,5 @@ pyaerocom .. |Coverage| image:: https://codecov.io/gh/metno/pyaerocom/branch/main-dev/graph/badge.svg?token=A0AdX8YciZ :target: https://codecov.io/gh/metno/pyaerocom -.. image:: https://zenodo.org/badge/DOI/10.5281/zenodo.13927979.svg +.. |DOI| image:: https://zenodo.org/badge/DOI/10.5281/zenodo.13927979.svg :target: https://doi.org/10.5281/zenodo.13927979 From fa66ae3679bc80fa3cd02bb962793b288c071d0e Mon Sep 17 00:00:00 2001 From: Jan Griesfeller Date: Mon, 14 Oct 2024 15:42:28 +0200 Subject: [PATCH 07/27] add ratpm10pm25 to EMEP calculated variables --- .../aeroval/config/ratpm10pm25/__init__.py | 0 .../aeroval/config/ratpm10pm25/base_config.py | 600 ++++++++++++++++++ pyaerocom/io/mscw_ctm/additional_variables.py | 22 + pyaerocom/io/mscw_ctm/reader.py | 6 +- 4 files changed, 626 insertions(+), 2 deletions(-) create mode 100644 pyaerocom/aeroval/config/ratpm10pm25/__init__.py create mode 100644 pyaerocom/aeroval/config/ratpm10pm25/base_config.py diff --git a/pyaerocom/aeroval/config/ratpm10pm25/__init__.py b/pyaerocom/aeroval/config/ratpm10pm25/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/pyaerocom/aeroval/config/ratpm10pm25/base_config.py b/pyaerocom/aeroval/config/ratpm10pm25/base_config.py new file mode 100644 index 000000000..0d7beac41 --- /dev/null +++ b/pyaerocom/aeroval/config/ratpm10pm25/base_config.py @@ -0,0 +1,600 @@ +""" +Global config for ratio pm10 vs pm25 +""" + +import copy +import logging +import os + +logger = logging.getLogger(__name__) + +# Constraints +DEFAULT_RESAMPLE_CONSTRAINTS = dict( + yearly=dict(monthly=9), + monthly=dict( + daily=21, + weekly=3, + ), + daily=dict(hourly=18), +) + +DEFAULT_RESAMPLE_CONSTRAINTS_DAILY = dict( + daily=dict(hourly=18), +) + +# ODCSFUN_EEANRT = "EEAAQeRep.NRT;concpm10/EEAAQeRep.NRT;concpm25" +ODCSFUN_EEAV2 = "EEAAQeRep.v2;concpm10/EEAAQeRep.v2;concpm25" +ODCSFUN_EBAS = "EBASMC;concpm10/EBASMC;concpm25" + + +def get_CFG(reportyear, year, model_dir) -> dict: + """create aeroval configuration dict to run the variable + ratpm10pm25 (ratio pm10 vspm25) + + :returns: a dict of a model configuration usable for EvalSetup + """ + # get current path for reference to local gridded_io_aux.py + base_conf_path = os.path.dirname(__file__) + + CFG = dict( + json_basedir=os.path.abspath("/home/jang/data/aeroval-local-web/data"), + coldata_basedir=os.path.abspath("/home/jang/data/aeroval-local-web/coldata"), + # io_aux_file=os.path.abspath("/home/jang/data/aeroval-local-web/gridded_io_aux.py"), not needed for ReadMscwCtm + # io_aux_file=os.path.join(base_conf_path, "gridded_io_aux.py"), + # var_scale_colmap_file=os.path.abspath( + # "/home/jang/data/aeroval-local-web/pyaerocom-config/config_files/CAMEO/user_var_scale_colmap.ini" + # ), + # if True, existing colocated data files will be deleted and contours will be overwritten + reanalyse_existing=True, + only_json=False, + add_model_maps=True, + only_model_maps=False, + modelmaps_opts=dict(maps_freq="yearly", maps_res_deg=5), + clear_existing_json=False, + # if True, the analysis will stop whenever an error occurs (else, errors that + # occurred will be written into the logfiles) + raise_exceptions=True, + # Regional filter for analysis + filter_name="ALL-wMOUNTAINS", + # colocation frequency (no statistics in higher resolution can be computed) + ts_type="daily", + map_zoom="Europe", + freqs=["yearly", "monthly", "weekly", "daily", "hourly"], + periods=[f"{year}"], + main_freq="daily", + zeros_to_nan=False, + use_diurnal=True, + min_num_obs=DEFAULT_RESAMPLE_CONSTRAINTS, + colocate_time=True, + resample_how={"vmro3max": {"daily": {"hourly": "max"}}}, + obs_remove_outliers=False, + model_remove_outliers=False, + harmonise_units=True, + regions_how="country", + annual_stats_constrained=True, + proj_id="emep", + exp_id=f"{reportyear}-reporting", + exp_name=f"Evaluation of EMEP runs for {reportyear} EMEP reporting", + exp_descr=( + f"Evaluation of EMEP runs for {reportyear} EMEP reporting. The EMEP model, simulated for {year}, is compared against observations from EEA and EBAS." + ), + exp_pi="emep.mscw@met.no", + public=True, + # directory where colocated data files are supposed to be stored + weighted_stats=True, + var_order_menu=[ + # Gases + "ratpm10pm25", + "concNno", + "concNno2", + "concNtno3", + "concNhno3", + "concNtnh", + "concNnh3", + "concnh4", + "concSso2", + "concso4t", + "concso4c", + "vmro3", + "vmro3max", + "vmro3mda8", + "vmrox", + "vmrco", + # PMs + "concpm10", + "concpm25", + "concno3pm10", + "concno3pm25", + "concnh4pm25", + "concso4pm25", + "concCecpm10", + "concCecpm25", + "concCocpm10", # SURF_ugC_PM_OMCOARSE missing in model-output + "concCocpm25", + "concsspm10", + "concsspm25", + # Depositions + "wetrdn", + "wetoxs", + "wetoxn", + "prmm", + ], + ) + + CFG["model_cfg"] = { + "EMEP.cameo": dict( + model_id="EMEP,", + model_data_dir=model_dir, + # gridded_reader_id={"model": "ReadMscwCtm"}, + # model_read_aux={}, + # model_ts_type_read="daily", + ), + } + + """ + Filters + """ + + # OBS SPECIFIC FILTERS (combination of the above and more) + EEA_RURAL_FILTER = { + "station_classification": ["background"], + "area_classification": [ + "rural", + "rural-nearcity", + "rural-regional", + "rural-remote", + ], + } + + BASE_FILTER = { + "latitude": [30, 82], + "longitude": [-30, 90], + } + + EBAS_FILTER = { + **BASE_FILTER, + "data_level": [None, 2], + "set_flags_nan": True, + } + + EEA_FILTER = { + **BASE_FILTER, + **EEA_RURAL_FILTER, + } + + EEA_FILTER_ALL = { + **BASE_FILTER, + } + + AERONET_FILTER = { + **BASE_FILTER, # Forandring fra Daniel + "altitude": [-20, 1000], + } + + # Station filters + + eea_species = [ + "concpm10", + "concpm25", + "ratpm10pm25", + ] + + ebas_species = [ + "concpm10", + "concpm25", + "ratpm10pm25", + ] + + # no new sites with 2021 observations (comment Svetlana T.) + height_ignore_ebas = [ + "AT0034G", + "AT0038R", + "AT0049R", + "BG0001R", + "CH0001G", + "CH0018R", + "CH0024R", + "CH0031R", + "CH0033R", + "DE0054R", + "DE0057G", + "DE0075R", + "ES0005R", + "ES0022R", + "FR0019R", + "FR0030R", + "FR0031R", + "FR0038U", + "FR0039U", + "FR0100G", + "GR0003R", + "GR0101R", + "HR0002R", + "HR0004R", + "IT0002R", + "IT0009R", + "IT0019R", + "IT0020U", + "IT0021U", + "IT0024R", + "KG0001R", + "KG0002U", + "NO0036R", + "NO0039R", + "NO0211R", + "NO0214R", + "NO0225R", + "NO0226R", + "NO0227R", + "NO0229R", + "NO0796R", + "NO0802R", + "NO0907R", + "NO2073R", + "NO2079R", + "NO2085R", + "NO2096R", + "NO2156R", + "NO2210R", + "NO2216R", + "NO2219R", + "NO2233R", + "NO2239R", + "NO2257R", + "NO2263R", + "NO2274R", + "NO2280R", + "NO2288R", + "NO2362R", + "NO2380R", + "NO2397R", + "NO2411R", + "PL0003R", + "PT0005R", + "PT0007R", + "PT0012R", + "RO0002R", + "RO0003R", + "SE0093R", + "SE0094R", + "SI0032R", + "SK0002R", + ] + + height_ignore_eea = [ + "FR33220", + "TR0047A", + "AT72619", + "ES1982A", + "IT0983A", + "IS0040A", + "IT2099A", + "BG0080A", + "IT2159A", + "IT0906A", + "AT72821", + "IT1190A", + "IT1976A", + "AT56072", + "IT2178A", + "IS0044A", + "IT1335A", + "AT0SON1", + "IT0703A", + "AT72227", + "DEUB044", + "AT55032", + "HR0013A", + "FR33120", + "AT60182", + "IT0908A", + "ES1673A", + "AT55019", + "SK0042A", + "SI0032R", + "ES0005R", + "FR33720", + "DEBY196", + "AT60177", + "IT2128A", + "AT2SP18", + "FR15045", + "R160421", + "IT2234A", + "TR0118A", + "DEST039", + "E165168", + "AT72110", + "FR15013", + "ES1348A", + "E165169", + "AL0206A", + "AT72822", + "DEBY123", + "FR15031", + "AT72538", + "IS0042A", + "FR33114", + "AT52300", + "IT1859A", + "FR33232", + "IT2239A", + "IS0043A", + "PL0003R", + "FR31027", + "FR33113", + "FR15048", + "AT54057", + "TR0046A", + "FR33111", + "IT2284A", + "AT72550", + "IT1037A", + "FR33121", + "E165167", + "IT1847A", + "AT72912", + "RS0047A", + "R610613", + "TR0110A", + "R160512", + "IT1191A", + "IT1963A", + "FR15053", + "RO0009R", + "IT0508A", + "IT2233A", + "MK0041A", + "AT72519", + "BG0079A", + "IT1696A", + "IT1619A", + "IT2267A", + "TR0107A", + "AT56071", + "FR29440", + "AT4S235", + "AD0945A", + "IS0038A", + "E165166", + "PT01047", + "AT55018", + "SK0002R", + "IT0499A", + "HR0014A", + "IT0591A", + "IT0507A", + "AT72315", + "E165170", + "ES1432A", + "IT1166A", + "AT4S254", + "IT1967A", + "AT2VL52", + "IT1930A", + "AT72115", + "AT82708", + "IT0988A", + "FR15038", + "AT82801", + "IT2285A", + "NO0039R", + "TR0020A", + "IT2096A", + "AD0942A", + "TR0071A", + "E165165", + "ES0354A", + "AT72910", + "ES1882A", + "IT1725A", + "AT60150", + "CH0024A", + "IT1114A", + "AT72113", + "IT1852A", + "IS0048A", + "FR15017", + "FR15039", + "IT0980A", + "IT0502A", + "IT1678A", + "IT1334A", + "IT0978A", + "FR15043", + "IT2279A", + "IT0775A", + "IT1539A", + "AT72123", + "IT2014A", + "XK0005A", + "AT2WO15", + "FR33122", + "XK0007A", + "AT60196", + "CH0033A", + "IT1385A", + "GR0405A", + "AT52000", + "IT2266A", + "FR15046", + "AT72223", + "FR24024", + "IT0979A", + "AT2SP10", + "IT2179A", + "IT0977A", + "AT72530", + "ES1248A", + "AT72106", + "IT0753A", + ] + + EEA_FILTER_ALL = { + key: dict( + **EEA_FILTER_ALL, + station_name=height_ignore_eea, + negate="station_name", + ) + for key in eea_species + } + + OBS_GROUNDBASED = { + ################## + # EBAS + ################## + "EBAS-m": dict( + obs_id="EBASMC", + web_interface_name="EBAS-m", + obs_vars=ebas_species, + obs_vert_type="Surface", + colocate_time=True, + ts_type="monthly", + obs_filters=EBAS_FILTER, + obs_merge_how={ + "ratpm10pm25": "eval", + }, + obs_aux_requires={ + "ratpm10pm25": { + "EBASMC": [ + "concpm10", + "concpm25", + ], + } + }, + obs_aux_funs={ + "vmrox": + # variables used in computation method need to be based on AeroCom + # units, since the colocated StationData objects (from which the + # new UngriddedData is computed, will perform AeroCom unit check + # and conversion) + "(EBASMC;concpm10/EBASMC;concpm25)" + }, + obs_aux_units={"ratpm10pm25": "1"}, + ), + "EBAS-d": dict( + obs_id="EBASMC", + web_interface_name="EBAS-d", + obs_vars=ebas_species, + obs_vert_type="Surface", + colocate_time=True, + min_num_obs=DEFAULT_RESAMPLE_CONSTRAINTS, + ts_type="daily", + obs_filters=EBAS_FILTER, + obs_merge_how={ + "ratpm10pm25": "eval", + }, + obs_aux_requires={ + "ratpm10pm25": { + "EBASMC": [ + "concpm10", + "concpm25", + ], + } + }, + obs_aux_funs={ + "vmrox": + # variables used in computation method need to be based on AeroCom + # units, since the colocated StationData objects (from which the + # new UngriddedData is computed, will perform AeroCom unit check + # and conversion) + "(EBASMC;concpm10/EBASMC;concpm25)" + }, + obs_aux_units={"ratpm10pm25": "1"}, + ), + # Diurnal + # "EBAS-h-diurnal": dict( + # obs_id="EBASMC", + # web_interface_name="EBAS-h", + # obs_vars=[ + # "concNno2", + # "concNno", + # "vmro3", + # "concpm10", + # "concpm25", + # ], + # obs_vert_type="Surface", + # ts_type="hourly", + # # diurnal_only=True, + # resample_how="mean", + # obs_filters={**EBAS_FILTER, "ts_type": "hourly"}, + # ), + # OX + ################ + # EEA-rural + ################ + "EEA-d-rural": dict( + obs_id="EEAAQeRep.v2", + obs_vars=[ + "concpm10", + "concpm25", + "ratpm10pm25", + ], + web_interface_name="EEA-rural", + obs_vert_type="Surface", + obs_filters=EEA_FILTER, + obs_merge_how={ + "ratpm10pm25": "eval", + }, + obs_aux_requires={ + "ratpm10pm25": { + "EBASMC": [ + "concpm10", + "concpm25", + ], + } + }, + obs_aux_funs={ + "vmrox": + # variables used in computation method need to be based on AeroCom + # units, since the colocated StationData objects (from which the + # new UngriddedData is computed, will perform AeroCom unit check + # and conversion) + "(EBASMC;concpm10/EBASMC;concpm25)" + }, + obs_aux_units={"ratpm10pm25": "1"}, + ), + ################ + # EEA-all + ################ + "EEA-d-all": dict( + obs_id="EEAAQeRep.v2", + obs_vars=[ + "concpm10", + "concpm25", + "ratpm10pm25", + ], + web_interface_name="EEA-all", + obs_vert_type="Surface", + obs_filters=EEA_FILTER_ALL, + obs_merge_how={ + "ratpm10pm25": "eval", + }, + obs_aux_requires={ + "ratpm10pm25": { + "EBASMC": [ + "concpm10", + "concpm25", + ], + } + }, + obs_aux_funs={ + "vmrox": + # variables used in computation method need to be based on AeroCom + # units, since the colocated StationData objects (from which the + # new UngriddedData is computed, will perform AeroCom unit check + # and conversion) + "(EBASMC;concpm10/EBASMC;concpm25)" + }, + obs_aux_units={"ratpm10pm25": "1"}, + ), + } + + # Setup for supported satellite evaluations + OBS_SAT = {} + + OBS_CFG = {**OBS_GROUNDBASED, **OBS_SAT} + + CFG["obs_cfg"] = OBS_CFG + + return copy.deepcopy(CFG) diff --git a/pyaerocom/io/mscw_ctm/additional_variables.py b/pyaerocom/io/mscw_ctm/additional_variables.py index 883550722..bf893ae2c 100644 --- a/pyaerocom/io/mscw_ctm/additional_variables.py +++ b/pyaerocom/io/mscw_ctm/additional_variables.py @@ -382,3 +382,25 @@ def calc_concpolyol(concspores): concpolyol = concspores.copy(deep=True) * factor concpolyol.attrs["units"] = "ug m-3" return concpolyol + + +def calc_ratpm10pm25(concpm10: xr.DataArray, concpm25: xr.DataArray) -> xr.DataArray: + """ + Calculate ratio of pm10 and pm25 + + Parameters + ---------- + concpm10 : xr.DataArray + mass concentration pm10 + concpm25 : xr.DataArray + mass concentration of pm25 + + Returns + ------- + xr.DataArray + ratio of concpm10 / concpm25 in units of 1 + + """ + ratpm10pm25 = concpm10 / concpm25 + ratpm10pm25.attrs["units"] = "1" + return ratpm10pm25 diff --git a/pyaerocom/io/mscw_ctm/reader.py b/pyaerocom/io/mscw_ctm/reader.py index 6fb3c016e..4b332edd6 100755 --- a/pyaerocom/io/mscw_ctm/reader.py +++ b/pyaerocom/io/mscw_ctm/reader.py @@ -13,7 +13,6 @@ from pyaerocom.io.gridded_reader import GriddedReader from pyaerocom.projection_information import ProjectionInformation from pyaerocom.units_helpers import UALIASES - from .additional_variables import ( add_dataarrays, calc_concNhno3, @@ -36,6 +35,7 @@ identity, subtract_dataarrays, update_EC_units, + calc_ratpm10pm25, ) from .model_variables import emep_variables @@ -101,6 +101,7 @@ class ReadMscwCtm(GriddedReader): "concNno2": ["concno2"], "concSso2": ["concso2"], "vmro3": ["conco3"], + "ratpm10pm25": ["concpm10", "concpm25"], # For Pollen # "concpolyol": ["concspores"], } @@ -145,6 +146,7 @@ class ReadMscwCtm(GriddedReader): "concNno2": calc_concNno2, "concSso2": calc_concSso2, "vmro3": calc_vmro3, + "ratpm10pm25": calc_ratpm10pm25, # "concpolyol": calc_concpolyol, } @@ -702,7 +704,7 @@ def read_var(self, var_name, ts_type=None, **kwargs): proj_info=proj_info, ) - #!obsolete + # !obsolete # if var.is_deposition: # implicit_to_explicit_rates(gridded, ts_type) From bf6612196005ad03f0ff218930d780a9991386f8 Mon Sep 17 00:00:00 2001 From: Jan Griesfeller Date: Wed, 16 Oct 2024 15:08:30 +0200 Subject: [PATCH 08/27] add config example for ratpm10pm25 and ratpm25pm10 --- .../aeroval/config/ratpm10pm25/base_config.py | 356 +++--------------- pyaerocom/io/mscw_ctm/additional_variables.py | 22 ++ pyaerocom/io/mscw_ctm/reader.py | 5 +- 3 files changed, 74 insertions(+), 309 deletions(-) diff --git a/pyaerocom/aeroval/config/ratpm10pm25/base_config.py b/pyaerocom/aeroval/config/ratpm10pm25/base_config.py index 0d7beac41..748d78c22 100644 --- a/pyaerocom/aeroval/config/ratpm10pm25/base_config.py +++ b/pyaerocom/aeroval/config/ratpm10pm25/base_config.py @@ -47,26 +47,29 @@ def get_CFG(reportyear, year, model_dir) -> dict: # if True, existing colocated data files will be deleted and contours will be overwritten reanalyse_existing=True, only_json=False, - add_model_maps=True, + add_model_maps=False, only_model_maps=False, - modelmaps_opts=dict(maps_freq="yearly", maps_res_deg=5), + modelmaps_opts=dict(maps_freq="monthly", maps_res_deg=5), clear_existing_json=False, # if True, the analysis will stop whenever an error occurs (else, errors that # occurred will be written into the logfiles) - raise_exceptions=True, + raise_exceptions=False, # Regional filter for analysis filter_name="ALL-wMOUNTAINS", # colocation frequency (no statistics in higher resolution can be computed) ts_type="daily", map_zoom="Europe", - freqs=["yearly", "monthly", "weekly", "daily", "hourly"], + freqs=[ + "yearly", + "monthly", + "daily", + ], periods=[f"{year}"], - main_freq="daily", + main_freq="monthly", zeros_to_nan=False, use_diurnal=True, min_num_obs=DEFAULT_RESAMPLE_CONSTRAINTS, colocate_time=True, - resample_how={"vmro3max": {"daily": {"hourly": "max"}}}, obs_remove_outliers=False, model_remove_outliers=False, harmonise_units=True, @@ -85,6 +88,7 @@ def get_CFG(reportyear, year, model_dir) -> dict: var_order_menu=[ # Gases "ratpm10pm25", + "ratpm25pm10", "concNno", "concNno2", "concNtno3", @@ -122,30 +126,28 @@ def get_CFG(reportyear, year, model_dir) -> dict: ) CFG["model_cfg"] = { - "EMEP.cameo": dict( + "EMEPcameo": dict( model_id="EMEP,", model_data_dir=model_dir, - # gridded_reader_id={"model": "ReadMscwCtm"}, + gridded_reader_id={"model": "ReadMscwCtm"}, # model_read_aux={}, - # model_ts_type_read="daily", + model_ts_type_read="daily", ), + # "EMEP": dict( + # model_id="EMEP.ratpm25pm10.testing", + # # model_read_aux={ + # + # # "ratpm10pm25": dict( + # # vars_required=["sconcpm10", "sconcpm25"], fun="calc_ratpm10pm25" + # # ) + # # }, + # ), } """ Filters """ - # OBS SPECIFIC FILTERS (combination of the above and more) - EEA_RURAL_FILTER = { - "station_classification": ["background"], - "area_classification": [ - "rural", - "rural-nearcity", - "rural-regional", - "rural-remote", - ], - } - BASE_FILTER = { "latitude": [30, 82], "longitude": [-30, 90], @@ -157,15 +159,6 @@ def get_CFG(reportyear, year, model_dir) -> dict: "set_flags_nan": True, } - EEA_FILTER = { - **BASE_FILTER, - **EEA_RURAL_FILTER, - } - - EEA_FILTER_ALL = { - **BASE_FILTER, - } - AERONET_FILTER = { **BASE_FILTER, # Forandring fra Daniel "altitude": [-20, 1000], @@ -173,16 +166,11 @@ def get_CFG(reportyear, year, model_dir) -> dict: # Station filters - eea_species = [ - "concpm10", - "concpm25", - "ratpm10pm25", - ] - ebas_species = [ "concpm10", "concpm25", "ratpm10pm25", + "ratpm25pm10", ] # no new sites with 2021 observations (comment Svetlana T.) @@ -261,196 +249,20 @@ def get_CFG(reportyear, year, model_dir) -> dict: "SK0002R", ] - height_ignore_eea = [ - "FR33220", - "TR0047A", - "AT72619", - "ES1982A", - "IT0983A", - "IS0040A", - "IT2099A", - "BG0080A", - "IT2159A", - "IT0906A", - "AT72821", - "IT1190A", - "IT1976A", - "AT56072", - "IT2178A", - "IS0044A", - "IT1335A", - "AT0SON1", - "IT0703A", - "AT72227", - "DEUB044", - "AT55032", - "HR0013A", - "FR33120", - "AT60182", - "IT0908A", - "ES1673A", - "AT55019", - "SK0042A", - "SI0032R", - "ES0005R", - "FR33720", - "DEBY196", - "AT60177", - "IT2128A", - "AT2SP18", - "FR15045", - "R160421", - "IT2234A", - "TR0118A", - "DEST039", - "E165168", - "AT72110", - "FR15013", - "ES1348A", - "E165169", - "AL0206A", - "AT72822", - "DEBY123", - "FR15031", - "AT72538", - "IS0042A", - "FR33114", - "AT52300", - "IT1859A", - "FR33232", - "IT2239A", - "IS0043A", - "PL0003R", - "FR31027", - "FR33113", - "FR15048", - "AT54057", - "TR0046A", - "FR33111", - "IT2284A", - "AT72550", - "IT1037A", - "FR33121", - "E165167", - "IT1847A", - "AT72912", - "RS0047A", - "R610613", - "TR0110A", - "R160512", - "IT1191A", - "IT1963A", - "FR15053", - "RO0009R", - "IT0508A", - "IT2233A", - "MK0041A", - "AT72519", - "BG0079A", - "IT1696A", - "IT1619A", - "IT2267A", - "TR0107A", - "AT56071", - "FR29440", - "AT4S235", - "AD0945A", - "IS0038A", - "E165166", - "PT01047", - "AT55018", - "SK0002R", - "IT0499A", - "HR0014A", - "IT0591A", - "IT0507A", - "AT72315", - "E165170", - "ES1432A", - "IT1166A", - "AT4S254", - "IT1967A", - "AT2VL52", - "IT1930A", - "AT72115", - "AT82708", - "IT0988A", - "FR15038", - "AT82801", - "IT2285A", - "NO0039R", - "TR0020A", - "IT2096A", - "AD0942A", - "TR0071A", - "E165165", - "ES0354A", - "AT72910", - "ES1882A", - "IT1725A", - "AT60150", - "CH0024A", - "IT1114A", - "AT72113", - "IT1852A", - "IS0048A", - "FR15017", - "FR15039", - "IT0980A", - "IT0502A", - "IT1678A", - "IT1334A", - "IT0978A", - "FR15043", - "IT2279A", - "IT0775A", - "IT1539A", - "AT72123", - "IT2014A", - "XK0005A", - "AT2WO15", - "FR33122", - "XK0007A", - "AT60196", - "CH0033A", - "IT1385A", - "GR0405A", - "AT52000", - "IT2266A", - "FR15046", - "AT72223", - "FR24024", - "IT0979A", - "AT2SP10", - "IT2179A", - "IT0977A", - "AT72530", - "ES1248A", - "AT72106", - "IT0753A", - ] - - EEA_FILTER_ALL = { - key: dict( - **EEA_FILTER_ALL, - station_name=height_ignore_eea, - negate="station_name", - ) - for key in eea_species - } - OBS_GROUNDBASED = { ################## # EBAS ################## - "EBAS-m": dict( - obs_id="EBASMC", - web_interface_name="EBAS-m", - obs_vars=ebas_species, + "EBAS-d-10": dict( + obs_id="EBASratd10", + web_interface_name="EBAS-d", + obs_vars=["ratpm10pm25"], obs_vert_type="Surface", colocate_time=True, - ts_type="monthly", + min_num_obs=DEFAULT_RESAMPLE_CONSTRAINTS, + ts_type="daily", obs_filters=EBAS_FILTER, + obs_type="ungridded", obs_merge_how={ "ratpm10pm25": "eval", }, @@ -463,81 +275,30 @@ def get_CFG(reportyear, year, model_dir) -> dict: } }, obs_aux_funs={ - "vmrox": + "ratpm10pm25": # variables used in computation method need to be based on AeroCom # units, since the colocated StationData objects (from which the # new UngriddedData is computed, will perform AeroCom unit check # and conversion) - "(EBASMC;concpm10/EBASMC;concpm25)" + "(EBASMC;concpm10/EBASMC;concpm25)" }, obs_aux_units={"ratpm10pm25": "1"}, ), - "EBAS-d": dict( - obs_id="EBASMC", + "EBAS-d-25": dict( + obs_id="EBASratd25", web_interface_name="EBAS-d", - obs_vars=ebas_species, + obs_vars=["ratpm25pm10"], obs_vert_type="Surface", colocate_time=True, min_num_obs=DEFAULT_RESAMPLE_CONSTRAINTS, ts_type="daily", obs_filters=EBAS_FILTER, + obs_type="ungridded", obs_merge_how={ - "ratpm10pm25": "eval", - }, - obs_aux_requires={ - "ratpm10pm25": { - "EBASMC": [ - "concpm10", - "concpm25", - ], - } - }, - obs_aux_funs={ - "vmrox": - # variables used in computation method need to be based on AeroCom - # units, since the colocated StationData objects (from which the - # new UngriddedData is computed, will perform AeroCom unit check - # and conversion) - "(EBASMC;concpm10/EBASMC;concpm25)" - }, - obs_aux_units={"ratpm10pm25": "1"}, - ), - # Diurnal - # "EBAS-h-diurnal": dict( - # obs_id="EBASMC", - # web_interface_name="EBAS-h", - # obs_vars=[ - # "concNno2", - # "concNno", - # "vmro3", - # "concpm10", - # "concpm25", - # ], - # obs_vert_type="Surface", - # ts_type="hourly", - # # diurnal_only=True, - # resample_how="mean", - # obs_filters={**EBAS_FILTER, "ts_type": "hourly"}, - # ), - # OX - ################ - # EEA-rural - ################ - "EEA-d-rural": dict( - obs_id="EEAAQeRep.v2", - obs_vars=[ - "concpm10", - "concpm25", - "ratpm10pm25", - ], - web_interface_name="EEA-rural", - obs_vert_type="Surface", - obs_filters=EEA_FILTER, - obs_merge_how={ - "ratpm10pm25": "eval", + "ratpm25pm10": "eval", }, obs_aux_requires={ - "ratpm10pm25": { + "ratpm25pm10": { "EBASMC": [ "concpm10", "concpm25", @@ -545,48 +306,27 @@ def get_CFG(reportyear, year, model_dir) -> dict: } }, obs_aux_funs={ - "vmrox": + "ratpm25pm10": # variables used in computation method need to be based on AeroCom # units, since the colocated StationData objects (from which the # new UngriddedData is computed, will perform AeroCom unit check # and conversion) - "(EBASMC;concpm10/EBASMC;concpm25)" + "(EBASMC;concpm25/EBASMC;concpm10)" }, - obs_aux_units={"ratpm10pm25": "1"}, + obs_aux_units={"ratpm25pm10": "1"}, ), - ################ - # EEA-all - ################ - "EEA-d-all": dict( - obs_id="EEAAQeRep.v2", + "EBAS-d-tc": dict( + obs_id="EBASMC", + web_interface_name="EBAS-d", obs_vars=[ "concpm10", "concpm25", - "ratpm10pm25", ], - web_interface_name="EEA-all", obs_vert_type="Surface", - obs_filters=EEA_FILTER_ALL, - obs_merge_how={ - "ratpm10pm25": "eval", - }, - obs_aux_requires={ - "ratpm10pm25": { - "EBASMC": [ - "concpm10", - "concpm25", - ], - } - }, - obs_aux_funs={ - "vmrox": - # variables used in computation method need to be based on AeroCom - # units, since the colocated StationData objects (from which the - # new UngriddedData is computed, will perform AeroCom unit check - # and conversion) - "(EBASMC;concpm10/EBASMC;concpm25)" - }, - obs_aux_units={"ratpm10pm25": "1"}, + colocate_time=True, + min_num_obs=DEFAULT_RESAMPLE_CONSTRAINTS, + ts_type="daily", + obs_filters=EBAS_FILTER, ), } diff --git a/pyaerocom/io/mscw_ctm/additional_variables.py b/pyaerocom/io/mscw_ctm/additional_variables.py index bf893ae2c..9208141b7 100644 --- a/pyaerocom/io/mscw_ctm/additional_variables.py +++ b/pyaerocom/io/mscw_ctm/additional_variables.py @@ -404,3 +404,25 @@ def calc_ratpm10pm25(concpm10: xr.DataArray, concpm25: xr.DataArray) -> xr.DataA ratpm10pm25 = concpm10 / concpm25 ratpm10pm25.attrs["units"] = "1" return ratpm10pm25 + + +def calc_ratpm25pm10(concpm25: xr.DataArray, concpm10: xr.DataArray) -> xr.DataArray: + """ + Calculate ratio of pm10 and pm25 + + Parameters + ---------- + concpm10 : xr.DataArray + mass concentration pm10 + concpm25 : xr.DataArray + mass concentration of pm25 + + Returns + ------- + xr.DataArray + ratio of concpm25 / concpm10 in units of 1 + + """ + ratpm25pm10 = concpm25 / concpm10 + ratpm25pm10.attrs["units"] = "1" + return ratpm25pm10 diff --git a/pyaerocom/io/mscw_ctm/reader.py b/pyaerocom/io/mscw_ctm/reader.py index 4b332edd6..ed4790081 100755 --- a/pyaerocom/io/mscw_ctm/reader.py +++ b/pyaerocom/io/mscw_ctm/reader.py @@ -6,13 +6,13 @@ import numpy as np import xarray as xr - from pyaerocom import const from pyaerocom.exceptions import VarNotAvailableError from pyaerocom.griddeddata import GriddedData from pyaerocom.io.gridded_reader import GriddedReader from pyaerocom.projection_information import ProjectionInformation from pyaerocom.units_helpers import UALIASES + from .additional_variables import ( add_dataarrays, calc_concNhno3, @@ -36,6 +36,7 @@ subtract_dataarrays, update_EC_units, calc_ratpm10pm25, + calc_ratpm25pm10, ) from .model_variables import emep_variables @@ -102,6 +103,7 @@ class ReadMscwCtm(GriddedReader): "concSso2": ["concso2"], "vmro3": ["conco3"], "ratpm10pm25": ["concpm10", "concpm25"], + "ratpm25pm10": ["concpm25", "concpm10"], # For Pollen # "concpolyol": ["concspores"], } @@ -147,6 +149,7 @@ class ReadMscwCtm(GriddedReader): "concSso2": calc_concSso2, "vmro3": calc_vmro3, "ratpm10pm25": calc_ratpm10pm25, + "ratpm25pm10": calc_ratpm25pm10, # "concpolyol": calc_concpolyol, } From 28f8e7bb8b2c6d85a6f9e7933b4a880ebf7ed6b4 Mon Sep 17 00:00:00 2001 From: Jan Griesfeller Date: Wed, 16 Oct 2024 15:20:01 +0200 Subject: [PATCH 09/27] rename directory for clearness --- pyaerocom/aeroval/config/{ratpm10pm25 => pmratios}/__init__.py | 0 pyaerocom/aeroval/config/{ratpm10pm25 => pmratios}/base_config.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename pyaerocom/aeroval/config/{ratpm10pm25 => pmratios}/__init__.py (100%) rename pyaerocom/aeroval/config/{ratpm10pm25 => pmratios}/base_config.py (100%) diff --git a/pyaerocom/aeroval/config/ratpm10pm25/__init__.py b/pyaerocom/aeroval/config/pmratios/__init__.py similarity index 100% rename from pyaerocom/aeroval/config/ratpm10pm25/__init__.py rename to pyaerocom/aeroval/config/pmratios/__init__.py diff --git a/pyaerocom/aeroval/config/ratpm10pm25/base_config.py b/pyaerocom/aeroval/config/pmratios/base_config.py similarity index 100% rename from pyaerocom/aeroval/config/ratpm10pm25/base_config.py rename to pyaerocom/aeroval/config/pmratios/base_config.py From 44cc55fd06bc1a7d105ddd1f255585a28bfb3250 Mon Sep 17 00:00:00 2001 From: Jan Griesfeller Date: Wed, 16 Oct 2024 15:53:44 +0200 Subject: [PATCH 10/27] add some documentation --- docs/_static/aeroval/sample_pm_ratios.py | 45 ++++++++++++++++++++++++ docs/aeroval-examples.rst | 7 +++- 2 files changed, 51 insertions(+), 1 deletion(-) create mode 100644 docs/_static/aeroval/sample_pm_ratios.py diff --git a/docs/_static/aeroval/sample_pm_ratios.py b/docs/_static/aeroval/sample_pm_ratios.py new file mode 100644 index 000000000..044146a7b --- /dev/null +++ b/docs/_static/aeroval/sample_pm_ratios.py @@ -0,0 +1,45 @@ +# Example aeroval configuration for pm ratios +# using the EMEP reader (which has a built in pm ratio calculation) + +import os +from getpass import getuser + +from pyaerocom.aeroval import EvalSetup, ExperimentProcessor +from pyaerocom.aeroval.config.pmratios.base_config import get_CFG + +reportyear = year = 2019 +CFG = get_CFG(reportyear=reportyear, year=year, + # model directory + model_dir="/lustre/storeB/project/fou/kl/CAMEO/u8_cams0201/" + ) +user = getuser() + +CFG.update(dict( + json_basedir=os.path.abspath(f"/home/{user}/data/aeroval-local-web/data"), # always adjust to your environment + coldata_basedir=os.path.abspath(f"/home/{user}/data/aeroval-local-web/coldata"), + # always adjust to your environment + clear_existing_json=True, + add_model_maps=True, + # if True, the analysis will stop whenever an error occurs (else, errors that + # occurred will be written into the logfiles) + raise_exceptions=False, + modelmaps_opts=dict(maps_freq="monthly", maps_res_deg=5), + # Regional filter for analysis + periods=[f"{year}"], + proj_id="RATPM", + exp_id=f"ratpm testing {year}", + exp_name=f"Evaluation of EMEP runs for {year}", + exp_descr=( + f"Evaluation of EMEP runs for {year} for CAMEO. The EMEP model, is compared against observations from EEA and EBAS." + ), + exp_pi="jang@met.no", + public=True, + # directory where colocated data files are supposed to be stored + weighted_stats=True, )) + +stp = EvalSetup(**CFG) + +ana = ExperimentProcessor(stp) +ana.update_interface() + +res = ana.run() diff --git a/docs/aeroval-examples.rst b/docs/aeroval-examples.rst index 713b3e82f..b86ac101e 100644 --- a/docs/aeroval-examples.rst +++ b/docs/aeroval-examples.rst @@ -8,7 +8,7 @@ detailed explanations of the setup parameters. A configuration could be run as t The code blocks below are the Python configuruation files *cfg_examples_example1.py* and *sample_gridded_io_aux.py*. -Example 1 +Example 1: NorESM, CAMS reanalysis against AERONET --------- NorESM2 and CAMS reanalysis vs AERONET and merged satellite AOD dataset. @@ -21,3 +21,8 @@ Example IO aux file for model reading .. literalinclude:: _static/aeroval/sample_gridded_io_aux.py +Example for pm ratios compared to EMEP mode data +--------------------- + +.. literalinclude:: _static/aeroval/sample_pm_ratios.py + From b00ea07334a47bbc9852338e27ca29a1c2d5c940 Mon Sep 17 00:00:00 2001 From: Jan Griesfeller Date: Wed, 16 Oct 2024 16:09:38 +0200 Subject: [PATCH 11/27] add pm ratio tests --- tests/io/mscw_ctm/test_additional_variables.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/io/mscw_ctm/test_additional_variables.py b/tests/io/mscw_ctm/test_additional_variables.py index 99dccba70..eaf137443 100644 --- a/tests/io/mscw_ctm/test_additional_variables.py +++ b/tests/io/mscw_ctm/test_additional_variables.py @@ -6,6 +6,8 @@ calc_concNno3pm25, calc_concNtnh, calc_conNtno3, + calc_ratpm10pm25, + calc_ratpm25pm10, update_EC_units, ) from tests.fixtures.mscw_ctm import create_fake_MSCWCtm_data @@ -80,6 +82,22 @@ def test_calc_concNtnh(): assert (concNtnh == concNtnh_from_func).all() +def test_calc_concpm10pm25(): + concpm10 = create_fake_MSCWCtm_data() + concpm25 = create_fake_MSCWCtm_data() + + ratpm10pm25_from_func = calc_ratpm10pm25(concpm10, concpm25) + assert ratpm10pm25_from_func.attrs["units"] == "1" + + +def test_calc_concpm25pm10(): + concpm10 = create_fake_MSCWCtm_data() + concpm25 = create_fake_MSCWCtm_data() + + ratpm10pm25_from_func = calc_ratpm25pm10(concpm25, concpm10) + assert ratpm10pm25_from_func.attrs["units"] == "1" + + def test_calc_concNnh3(): concnh3 = create_fake_MSCWCtm_data() From 2f5f2a0a1af551c343dcb29c071eb0b742c8b3f7 Mon Sep 17 00:00:00 2001 From: Jan Griesfeller Date: Wed, 16 Oct 2024 16:12:28 +0200 Subject: [PATCH 12/27] linting --- tests/io/mscw_ctm/test_additional_variables.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/io/mscw_ctm/test_additional_variables.py b/tests/io/mscw_ctm/test_additional_variables.py index eaf137443..bccbb057f 100644 --- a/tests/io/mscw_ctm/test_additional_variables.py +++ b/tests/io/mscw_ctm/test_additional_variables.py @@ -94,8 +94,8 @@ def test_calc_concpm25pm10(): concpm10 = create_fake_MSCWCtm_data() concpm25 = create_fake_MSCWCtm_data() - ratpm10pm25_from_func = calc_ratpm25pm10(concpm25, concpm10) - assert ratpm10pm25_from_func.attrs["units"] == "1" + ratpm25pm10_from_func = calc_ratpm25pm10(concpm25, concpm10) + assert ratpm25pm10_from_func.attrs["units"] == "1" def test_calc_concNnh3(): From bd55ef572b534ee0a4e1b1653ea6a3a415e71827 Mon Sep 17 00:00:00 2001 From: Jan Griesfeller Date: Wed, 16 Oct 2024 16:14:50 +0200 Subject: [PATCH 13/27] linting --- pyaerocom/aeroval/config/pmratios/base_config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyaerocom/aeroval/config/pmratios/base_config.py b/pyaerocom/aeroval/config/pmratios/base_config.py index 748d78c22..6973e2a60 100644 --- a/pyaerocom/aeroval/config/pmratios/base_config.py +++ b/pyaerocom/aeroval/config/pmratios/base_config.py @@ -280,7 +280,7 @@ def get_CFG(reportyear, year, model_dir) -> dict: # units, since the colocated StationData objects (from which the # new UngriddedData is computed, will perform AeroCom unit check # and conversion) - "(EBASMC;concpm10/EBASMC;concpm25)" + "(EBASMC;concpm10/EBASMC;concpm25)" }, obs_aux_units={"ratpm10pm25": "1"}, ), @@ -311,7 +311,7 @@ def get_CFG(reportyear, year, model_dir) -> dict: # units, since the colocated StationData objects (from which the # new UngriddedData is computed, will perform AeroCom unit check # and conversion) - "(EBASMC;concpm25/EBASMC;concpm10)" + "(EBASMC;concpm25/EBASMC;concpm10)" }, obs_aux_units={"ratpm25pm10": "1"}, ), From c7785f8c9521ae7ea4463bdad2a853c1b063ff17 Mon Sep 17 00:00:00 2001 From: Jan Griesfeller Date: Wed, 16 Oct 2024 16:20:12 +0200 Subject: [PATCH 14/27] ruff --- .../aeroval/config/pmratios/base_config.py | 104 +----------------- 1 file changed, 2 insertions(+), 102 deletions(-) diff --git a/pyaerocom/aeroval/config/pmratios/base_config.py b/pyaerocom/aeroval/config/pmratios/base_config.py index 6973e2a60..4f0526520 100644 --- a/pyaerocom/aeroval/config/pmratios/base_config.py +++ b/pyaerocom/aeroval/config/pmratios/base_config.py @@ -34,7 +34,6 @@ def get_CFG(reportyear, year, model_dir) -> dict: :returns: a dict of a model configuration usable for EvalSetup """ # get current path for reference to local gridded_io_aux.py - base_conf_path = os.path.dirname(__file__) CFG = dict( json_basedir=os.path.abspath("/home/jang/data/aeroval-local-web/data"), @@ -133,15 +132,6 @@ def get_CFG(reportyear, year, model_dir) -> dict: # model_read_aux={}, model_ts_type_read="daily", ), - # "EMEP": dict( - # model_id="EMEP.ratpm25pm10.testing", - # # model_read_aux={ - # - # # "ratpm10pm25": dict( - # # vars_required=["sconcpm10", "sconcpm25"], fun="calc_ratpm10pm25" - # # ) - # # }, - # ), } """ @@ -159,96 +149,6 @@ def get_CFG(reportyear, year, model_dir) -> dict: "set_flags_nan": True, } - AERONET_FILTER = { - **BASE_FILTER, # Forandring fra Daniel - "altitude": [-20, 1000], - } - - # Station filters - - ebas_species = [ - "concpm10", - "concpm25", - "ratpm10pm25", - "ratpm25pm10", - ] - - # no new sites with 2021 observations (comment Svetlana T.) - height_ignore_ebas = [ - "AT0034G", - "AT0038R", - "AT0049R", - "BG0001R", - "CH0001G", - "CH0018R", - "CH0024R", - "CH0031R", - "CH0033R", - "DE0054R", - "DE0057G", - "DE0075R", - "ES0005R", - "ES0022R", - "FR0019R", - "FR0030R", - "FR0031R", - "FR0038U", - "FR0039U", - "FR0100G", - "GR0003R", - "GR0101R", - "HR0002R", - "HR0004R", - "IT0002R", - "IT0009R", - "IT0019R", - "IT0020U", - "IT0021U", - "IT0024R", - "KG0001R", - "KG0002U", - "NO0036R", - "NO0039R", - "NO0211R", - "NO0214R", - "NO0225R", - "NO0226R", - "NO0227R", - "NO0229R", - "NO0796R", - "NO0802R", - "NO0907R", - "NO2073R", - "NO2079R", - "NO2085R", - "NO2096R", - "NO2156R", - "NO2210R", - "NO2216R", - "NO2219R", - "NO2233R", - "NO2239R", - "NO2257R", - "NO2263R", - "NO2274R", - "NO2280R", - "NO2288R", - "NO2362R", - "NO2380R", - "NO2397R", - "NO2411R", - "PL0003R", - "PT0005R", - "PT0007R", - "PT0012R", - "RO0002R", - "RO0003R", - "SE0093R", - "SE0094R", - "SI0032R", - "SK0002R", - ] - OBS_GROUNDBASED = { ################## # EBAS @@ -280,7 +180,7 @@ def get_CFG(reportyear, year, model_dir) -> dict: # units, since the colocated StationData objects (from which the # new UngriddedData is computed, will perform AeroCom unit check # and conversion) - "(EBASMC;concpm10/EBASMC;concpm25)" + "(EBASMC;concpm10/EBASMC;concpm25)" }, obs_aux_units={"ratpm10pm25": "1"}, ), @@ -311,7 +211,7 @@ def get_CFG(reportyear, year, model_dir) -> dict: # units, since the colocated StationData objects (from which the # new UngriddedData is computed, will perform AeroCom unit check # and conversion) - "(EBASMC;concpm25/EBASMC;concpm10)" + "(EBASMC;concpm25/EBASMC;concpm10)" }, obs_aux_units={"ratpm25pm10": "1"}, ), From d987fd613952e9db1b16c3faf229e70479f50d78 Mon Sep 17 00:00:00 2001 From: Jan Griesfeller Date: Wed, 16 Oct 2024 16:37:36 +0200 Subject: [PATCH 15/27] add coverage --- tests/io/mscw_ctm/test_additional_variables.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/io/mscw_ctm/test_additional_variables.py b/tests/io/mscw_ctm/test_additional_variables.py index bccbb057f..7af78c44c 100644 --- a/tests/io/mscw_ctm/test_additional_variables.py +++ b/tests/io/mscw_ctm/test_additional_variables.py @@ -128,3 +128,14 @@ def test_update_EC_units(): assert (concCecpm25 == concCecpm25_from_func).all() assert concCecpm25.units == concCecpm25_from_func.units + + +def test_ratpmconfig(): + from pyaerocom.aeroval.config.pmratios.base_config import get_CFG + reportyear = year = 2019 + CFG = get_CFG( + reportyear=reportyear, + year=year, + model_dir="/lustre/storeB/project/fou/kl/CAMEO/u8_cams0201/", + ) + assert CFG['raise_exceptions'] == False From 3a00a070d83257e397b6d615e21b3d90a7882f14 Mon Sep 17 00:00:00 2001 From: Jan Griesfeller Date: Thu, 17 Oct 2024 10:00:34 +0200 Subject: [PATCH 16/27] add seperate example config test --- pyaerocom/aeroval/config/pmratios/base_config.py | 4 ++-- tests/io/mscw_ctm/test_additional_variables.py | 11 ----------- tests/io/mscw_ctm/test_aeroval_configs.py | 13 +++++++++++++ 3 files changed, 15 insertions(+), 13 deletions(-) create mode 100644 tests/io/mscw_ctm/test_aeroval_configs.py diff --git a/pyaerocom/aeroval/config/pmratios/base_config.py b/pyaerocom/aeroval/config/pmratios/base_config.py index 4f0526520..f90eed2ab 100644 --- a/pyaerocom/aeroval/config/pmratios/base_config.py +++ b/pyaerocom/aeroval/config/pmratios/base_config.py @@ -180,7 +180,7 @@ def get_CFG(reportyear, year, model_dir) -> dict: # units, since the colocated StationData objects (from which the # new UngriddedData is computed, will perform AeroCom unit check # and conversion) - "(EBASMC;concpm10/EBASMC;concpm25)" + "(EBASMC;concpm10/EBASMC;concpm25)" }, obs_aux_units={"ratpm10pm25": "1"}, ), @@ -211,7 +211,7 @@ def get_CFG(reportyear, year, model_dir) -> dict: # units, since the colocated StationData objects (from which the # new UngriddedData is computed, will perform AeroCom unit check # and conversion) - "(EBASMC;concpm25/EBASMC;concpm10)" + "(EBASMC;concpm25/EBASMC;concpm10)" }, obs_aux_units={"ratpm25pm10": "1"}, ), diff --git a/tests/io/mscw_ctm/test_additional_variables.py b/tests/io/mscw_ctm/test_additional_variables.py index 7af78c44c..bccbb057f 100644 --- a/tests/io/mscw_ctm/test_additional_variables.py +++ b/tests/io/mscw_ctm/test_additional_variables.py @@ -128,14 +128,3 @@ def test_update_EC_units(): assert (concCecpm25 == concCecpm25_from_func).all() assert concCecpm25.units == concCecpm25_from_func.units - - -def test_ratpmconfig(): - from pyaerocom.aeroval.config.pmratios.base_config import get_CFG - reportyear = year = 2019 - CFG = get_CFG( - reportyear=reportyear, - year=year, - model_dir="/lustre/storeB/project/fou/kl/CAMEO/u8_cams0201/", - ) - assert CFG['raise_exceptions'] == False diff --git a/tests/io/mscw_ctm/test_aeroval_configs.py b/tests/io/mscw_ctm/test_aeroval_configs.py new file mode 100644 index 000000000..c324036be --- /dev/null +++ b/tests/io/mscw_ctm/test_aeroval_configs.py @@ -0,0 +1,13 @@ +from pyaerocom.aeroval.config.pmratios.base_config import get_CFG + + +def test_ratpmconfig(): + """short test if the example configuration for pm ratios is still in the code""" + + reportyear = year = 2019 + CFG = get_CFG( + reportyear=reportyear, + year=year, + model_dir="/lustre/storeB/project/fou/kl/CAMEO/u8_cams0201/", + ) + assert CFG["raise_exceptions"] == False From 30a7fdb47afe2b1c83a36ed26a7089ad18b23d79 Mon Sep 17 00:00:00 2001 From: Jan Griesfeller Date: Thu, 17 Oct 2024 10:13:50 +0200 Subject: [PATCH 17/27] linting --- docs/_static/aeroval/sample_pm_ratios.py | 59 +++++++++++++---------- tests/io/mscw_ctm/test_aeroval_configs.py | 2 +- 2 files changed, 34 insertions(+), 27 deletions(-) diff --git a/docs/_static/aeroval/sample_pm_ratios.py b/docs/_static/aeroval/sample_pm_ratios.py index 044146a7b..5bcad8eca 100644 --- a/docs/_static/aeroval/sample_pm_ratios.py +++ b/docs/_static/aeroval/sample_pm_ratios.py @@ -8,34 +8,41 @@ from pyaerocom.aeroval.config.pmratios.base_config import get_CFG reportyear = year = 2019 -CFG = get_CFG(reportyear=reportyear, year=year, - # model directory - model_dir="/lustre/storeB/project/fou/kl/CAMEO/u8_cams0201/" - ) +CFG = get_CFG( + reportyear=reportyear, + year=year, + # model directory + model_dir="/lustre/storeB/project/fou/kl/CAMEO/u8_cams0201/", +) user = getuser() -CFG.update(dict( - json_basedir=os.path.abspath(f"/home/{user}/data/aeroval-local-web/data"), # always adjust to your environment - coldata_basedir=os.path.abspath(f"/home/{user}/data/aeroval-local-web/coldata"), - # always adjust to your environment - clear_existing_json=True, - add_model_maps=True, - # if True, the analysis will stop whenever an error occurs (else, errors that - # occurred will be written into the logfiles) - raise_exceptions=False, - modelmaps_opts=dict(maps_freq="monthly", maps_res_deg=5), - # Regional filter for analysis - periods=[f"{year}"], - proj_id="RATPM", - exp_id=f"ratpm testing {year}", - exp_name=f"Evaluation of EMEP runs for {year}", - exp_descr=( - f"Evaluation of EMEP runs for {year} for CAMEO. The EMEP model, is compared against observations from EEA and EBAS." - ), - exp_pi="jang@met.no", - public=True, - # directory where colocated data files are supposed to be stored - weighted_stats=True, )) +CFG.update( + dict( + json_basedir=os.path.abspath( + f"/home/{user}/data/aeroval-local-web/data" + ), # always adjust to your environment + coldata_basedir=os.path.abspath(f"/home/{user}/data/aeroval-local-web/coldata"), + # always adjust to your environment + clear_existing_json=True, + add_model_maps=True, + # if True, the analysis will stop whenever an error occurs (else, errors that + # occurred will be written into the logfiles) + raise_exceptions=False, + modelmaps_opts=dict(maps_freq="monthly", maps_res_deg=5), + # Regional filter for analysis + periods=[f"{year}"], + proj_id="RATPM", + exp_id=f"ratpm testing {year}", + exp_name=f"Evaluation of EMEP runs for {year}", + exp_descr=( + f"Evaluation of EMEP runs for {year} for CAMEO. The EMEP model, is compared against observations from EEA and EBAS." + ), + exp_pi="jang@met.no", + public=True, + # directory where colocated data files are supposed to be stored + weighted_stats=True, + ) +) stp = EvalSetup(**CFG) diff --git a/tests/io/mscw_ctm/test_aeroval_configs.py b/tests/io/mscw_ctm/test_aeroval_configs.py index c324036be..cff52fcbb 100644 --- a/tests/io/mscw_ctm/test_aeroval_configs.py +++ b/tests/io/mscw_ctm/test_aeroval_configs.py @@ -10,4 +10,4 @@ def test_ratpmconfig(): year=year, model_dir="/lustre/storeB/project/fou/kl/CAMEO/u8_cams0201/", ) - assert CFG["raise_exceptions"] == False + assert not CFG["raise_exceptions"] From 0d1ead8413b53b880a8c60aa67a31f7d293c02df Mon Sep 17 00:00:00 2001 From: Jan Griesfeller Date: Thu, 17 Oct 2024 10:22:28 +0200 Subject: [PATCH 18/27] typo --- docs/aeroval-examples.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/aeroval-examples.rst b/docs/aeroval-examples.rst index b86ac101e..2838933fa 100644 --- a/docs/aeroval-examples.rst +++ b/docs/aeroval-examples.rst @@ -21,7 +21,7 @@ Example IO aux file for model reading .. literalinclude:: _static/aeroval/sample_gridded_io_aux.py -Example for pm ratios compared to EMEP mode data +Example for pm ratios compared to EMEP model data --------------------- .. literalinclude:: _static/aeroval/sample_pm_ratios.py From 36a4f3350160af1d37b33482906b2c22ac56397e Mon Sep 17 00:00:00 2001 From: Jan Griesfeller Date: Thu, 17 Oct 2024 11:42:40 +0200 Subject: [PATCH 19/27] adjusted to code review --- pyaerocom/io/mscw_ctm/additional_variables.py | 16 +++++++++++++++- pyaerocom/io/mscw_ctm/reader.py | 4 ---- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/pyaerocom/io/mscw_ctm/additional_variables.py b/pyaerocom/io/mscw_ctm/additional_variables.py index 9208141b7..5795b9fb1 100644 --- a/pyaerocom/io/mscw_ctm/additional_variables.py +++ b/pyaerocom/io/mscw_ctm/additional_variables.py @@ -1,9 +1,11 @@ +import logging + import xarray as xr from geonum.atmosphere import T0_STD, p0 - from pyaerocom.aux_var_helpers import concx_to_vmrx from pyaerocom.molmasses import get_molmass +logger = logging.getLogger(__name__) def add_dataarrays(arr0: xr.DataArray, *arrs: xr.DataArray) -> xr.DataArray: """ @@ -401,6 +403,12 @@ def calc_ratpm10pm25(concpm10: xr.DataArray, concpm25: xr.DataArray) -> xr.DataA ratio of concpm10 / concpm25 in units of 1 """ + try: + if concpm10.attrs["units"] != concpm25.attrs["units"]: + logger.warning( + f"concpm10 unit {concpm10.attrs['units']} not equal to concpm25 unit {concpm25.attrs['units']}!") + except KeyError: + pass ratpm10pm25 = concpm10 / concpm25 ratpm10pm25.attrs["units"] = "1" return ratpm10pm25 @@ -423,6 +431,12 @@ def calc_ratpm25pm10(concpm25: xr.DataArray, concpm10: xr.DataArray) -> xr.DataA ratio of concpm25 / concpm10 in units of 1 """ + try: + if concpm10.attrs["units"] != concpm25.attrs["units"]: + logger.warning( + f"concpm10 unit {concpm10.attrs['units']} not equal to concpm25 unit {concpm25.attrs['units']}!") + except KeyError: + pass ratpm25pm10 = concpm25 / concpm10 ratpm25pm10.attrs["units"] = "1" return ratpm25pm10 diff --git a/pyaerocom/io/mscw_ctm/reader.py b/pyaerocom/io/mscw_ctm/reader.py index ed4790081..c717fde3c 100755 --- a/pyaerocom/io/mscw_ctm/reader.py +++ b/pyaerocom/io/mscw_ctm/reader.py @@ -707,10 +707,6 @@ def read_var(self, var_name, ts_type=None, **kwargs): proj_info=proj_info, ) - # !obsolete - # if var.is_deposition: - # implicit_to_explicit_rates(gridded, ts_type) - # At this point a GriddedData object with name gridded should exist gridded.metadata["data_id"] = self._data_id From 5e1822dde255db7ca2202813b0a459a599e70f58 Mon Sep 17 00:00:00 2001 From: Jan Griesfeller Date: Thu, 17 Oct 2024 11:56:06 +0200 Subject: [PATCH 20/27] linting --- pyaerocom/io/mscw_ctm/additional_variables.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pyaerocom/io/mscw_ctm/additional_variables.py b/pyaerocom/io/mscw_ctm/additional_variables.py index 5795b9fb1..abdf51e0c 100644 --- a/pyaerocom/io/mscw_ctm/additional_variables.py +++ b/pyaerocom/io/mscw_ctm/additional_variables.py @@ -7,6 +7,7 @@ logger = logging.getLogger(__name__) + def add_dataarrays(arr0: xr.DataArray, *arrs: xr.DataArray) -> xr.DataArray: """ Add a bunch of :class:`xarray.DataArray` instances @@ -406,7 +407,8 @@ def calc_ratpm10pm25(concpm10: xr.DataArray, concpm25: xr.DataArray) -> xr.DataA try: if concpm10.attrs["units"] != concpm25.attrs["units"]: logger.warning( - f"concpm10 unit {concpm10.attrs['units']} not equal to concpm25 unit {concpm25.attrs['units']}!") + f"concpm10 unit {concpm10.attrs['units']} not equal to concpm25 unit {concpm25.attrs['units']}!" + ) except KeyError: pass ratpm10pm25 = concpm10 / concpm25 @@ -434,7 +436,8 @@ def calc_ratpm25pm10(concpm25: xr.DataArray, concpm10: xr.DataArray) -> xr.DataA try: if concpm10.attrs["units"] != concpm25.attrs["units"]: logger.warning( - f"concpm10 unit {concpm10.attrs['units']} not equal to concpm25 unit {concpm25.attrs['units']}!") + f"concpm10 unit {concpm10.attrs['units']} not equal to concpm25 unit {concpm25.attrs['units']}!" + ) except KeyError: pass ratpm25pm10 = concpm25 / concpm10 From dc384f8781d6922786b876ebc32dcf0629de9cfb Mon Sep 17 00:00:00 2001 From: lewisblake Date: Fri, 18 Oct 2024 12:52:22 +0000 Subject: [PATCH 21/27] a fix to test wtih charlie --- pyaerocom/aeroval/modelmaps_engine.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyaerocom/aeroval/modelmaps_engine.py b/pyaerocom/aeroval/modelmaps_engine.py index aea53c4e2..cb2bc4ac7 100644 --- a/pyaerocom/aeroval/modelmaps_engine.py +++ b/pyaerocom/aeroval/modelmaps_engine.py @@ -112,7 +112,6 @@ def _run_model(self, model_name: str, var_list): make_overlay = OVERLAY in self.cfg.modelmaps_opts.plot_types.get( model_name, False ) - if self.cfg.modelmaps_opts.plot_types == {CONTOUR} or make_contour: _files = self._process_contour_map_var( model_name, var, self.reanalyse_existing @@ -401,6 +400,7 @@ def _read_model_data(self, model_name: str, var: str) -> GriddedData: kwargs.update(**self.cfg.colocation_opts.model_kwargs) if var in self.cfg.colocation_opts.model_read_opts: kwargs.update(self.cfg.colocation_opts.model_read_opts[var]) + kwargs.update(self.cfg.get_model_entry(model_name)["model_kwargs"]) if model_reader is not None and model_reader in MODELREADERS_USE_MAP_FREQ: ts_types = reader.ts_types From c5272474a005e05df56a547375d562f458a48a67 Mon Sep 17 00:00:00 2001 From: lewisblake Date: Fri, 18 Oct 2024 13:02:27 +0000 Subject: [PATCH 22/27] safter version until model_kwargs fixed in pydantic model_entry --- pyaerocom/aeroval/modelmaps_engine.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyaerocom/aeroval/modelmaps_engine.py b/pyaerocom/aeroval/modelmaps_engine.py index cb2bc4ac7..229c0b87d 100644 --- a/pyaerocom/aeroval/modelmaps_engine.py +++ b/pyaerocom/aeroval/modelmaps_engine.py @@ -400,7 +400,7 @@ def _read_model_data(self, model_name: str, var: str) -> GriddedData: kwargs.update(**self.cfg.colocation_opts.model_kwargs) if var in self.cfg.colocation_opts.model_read_opts: kwargs.update(self.cfg.colocation_opts.model_read_opts[var]) - kwargs.update(self.cfg.get_model_entry(model_name)["model_kwargs"]) + kwargs.update(self.cfg.get_model_entry(model_name).get("model_kwargs", {})) if model_reader is not None and model_reader in MODELREADERS_USE_MAP_FREQ: ts_types = reader.ts_types From c0314a1186d56f7d80f6081a84df96cd37ea3955 Mon Sep 17 00:00:00 2001 From: lewisblake Date: Tue, 5 Nov 2024 14:00:10 +0000 Subject: [PATCH 23/27] add model_data_dir to model_entry --- pyaerocom/aeroval/modelentry.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pyaerocom/aeroval/modelentry.py b/pyaerocom/aeroval/modelentry.py index 0d8e46db3..ef5699fe3 100644 --- a/pyaerocom/aeroval/modelentry.py +++ b/pyaerocom/aeroval/modelentry.py @@ -200,6 +200,7 @@ class ModelEntry(BaseModel): model_read_aux: dict = {} model_rename_vars: dict = {} flex_ts_type: bool = False + model_data_dir: str | None = None @property def aux_funs_required(self): From 234f859090b9cdba7912edcf8d6c612bd9062d3b Mon Sep 17 00:00:00 2001 From: Magnus Ulimoen Date: Wed, 6 Nov 2024 13:43:21 +0100 Subject: [PATCH 24/27] Use documented attribute --- pyaerocom/io/pyaro/read_pyaro.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyaerocom/io/pyaro/read_pyaro.py b/pyaerocom/io/pyaro/read_pyaro.py index 8c9fdc20e..95ef4a09f 100644 --- a/pyaerocom/io/pyaro/read_pyaro.py +++ b/pyaerocom/io/pyaro/read_pyaro.py @@ -148,7 +148,7 @@ def _convert_to_ungriddeddata(self, pyaro_data: dict[str, Data]) -> UngriddedDat var_size = {var: len(pyaro_data[var]) for var in pyaro_data} vars = list(pyaro_data.keys()) total_size = sum(list(var_size.values())) - units = {var: {"units": pyaro_data[var]._units} for var in pyaro_data} + units = {var: {"units": pyaro_data[var].units} for var in pyaro_data} # Object necessary for ungriddeddata var_idx = {var: i for i, var in enumerate(vars)} From 7ed6026ee334ee003153cbfc80d896507cc4c6c7 Mon Sep 17 00:00:00 2001 From: lewisblake Date: Sun, 10 Nov 2024 10:40:14 +0000 Subject: [PATCH 25/27] add attributes which previously were given as kwargs --- pyaerocom/aeroval/modelentry.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pyaerocom/aeroval/modelentry.py b/pyaerocom/aeroval/modelentry.py index ef5699fe3..a8d1bed0a 100644 --- a/pyaerocom/aeroval/modelentry.py +++ b/pyaerocom/aeroval/modelentry.py @@ -199,8 +199,11 @@ class ModelEntry(BaseModel): model_add_vars: dict[str, tuple[str, ...]] = {} model_read_aux: dict = {} model_rename_vars: dict = {} - flex_ts_type: bool = False + flex_ts_type: bool = True model_data_dir: str | None = None + # attributes previously given as kwargs used in CAMS2_83 + gridded_reader_id: dict[str, str] = {"model": "ReadGridded", "obs": "ReadGridded"} + model_kwargs: dict = {} @property def aux_funs_required(self): From 2c704b5d8878b4084b11b13c5982e779ccf33850 Mon Sep 17 00:00:00 2001 From: lewisblake Date: Sun, 10 Nov 2024 10:43:40 +0000 Subject: [PATCH 26/27] clean up dead code --- pyaerocom/aeroval/modelentry.py | 150 -------------------------------- 1 file changed, 150 deletions(-) diff --git a/pyaerocom/aeroval/modelentry.py b/pyaerocom/aeroval/modelentry.py index a8d1bed0a..50f9d77ae 100644 --- a/pyaerocom/aeroval/modelentry.py +++ b/pyaerocom/aeroval/modelentry.py @@ -2,151 +2,9 @@ from copy import deepcopy from pydantic import BaseModel, ConfigDict - -# from pyaerocom._lowlevel_helpers import ( -# BrowseDict, -# # DictStrKeysListVals, -# # DictType, -# StrType, -# ) from pyaerocom.aeroval.aux_io_helpers import check_aux_info -# class ModelEntry_old(BrowseDict): -# """Modeln configuration for evaluation (dictionary) - -# Note -# ----model_read_aux -# Only :attr:`model_id` is mandatory, the rest is optional. - -# Attributes -# ---------- -# model_id : str -# ID of model run in AeroCom database (e.g. 'ECMWF_CAMS_REAN') -# model_ts_type_read : str or dict, optional -# may be specified to explicitly define the reading frequency of the -# model data. Not to be confused with :attr:`ts_type`, which specifies -# the frequency used for colocation. Can be specified variable specific -# by providing a dictionary. -# model_use_vars : dict -# dictionary that specifies mapping of model variables. Keys are -# observation variables, values are strings specifying the corresponding -# model variable to be used -# (e.g. model_use_vars=dict(od550aer='od550csaer')) -# model_add_vars : dict -# dictionary that specifies additional model variables. Keys are -# observation variables, values are lists of strings specifying the -# corresponding model variables to be used -# (e.g. model_use_vars=dict(od550aer=['od550csaer', 'od550so4'])) -# model_rename_vars : dict -# key / value pairs specifying new variable names for model variables -# in the output json files (is applied after co-location). -# model_read_aux : dict -# may be used to specify additional computation methods of variables from -# models. Keys are obs variables, values are dictionaries with keys -# `vars_required` (list of required variables for computation of var -# and `fun` (method that takes list of read data objects and computes -# and returns var) -# """ - -# model_id = StrType() -# model_use_vars = DictType() -# model_add_vars = DictStrKeysListVals() -# model_read_aux = DictType() -# model_rename_vars = DictType() - -# def __init__(self, model_id, **kwargs): -# self.model_id = model_id -# self.model_ts_type_read = "" -# self.model_use_vars = {} -# self.model_add_vars = {} -# self.model_rename_vars = {} -# self.model_read_aux = {} - -# self.kwargs = kwargs - -# self.update(**kwargs) - -# @property -# def aux_funs_required(self): -# """ -# Boolean specifying whether this entry requires auxiliary variables -# """ -# return True if bool(self.model_read_aux) else False - -# def json_repr(self) -> dict: -# sup_rep = super().json_repr() - -# # a little hacky, but makes the cams2-82 configs work -# try: -# for key in sup_rep["model_read_aux"]: -# sup_rep["model_read_aux"][key]["fun"] = inspect.getsource( -# deepcopy(sup_rep["model_read_aux"][key]["fun"]) -# ) -# except TypeError: -# pass - -# return sup_rep - -# def get_vars_to_process(self, obs_vars: list) -> tuple: -# """ -# Get lists of obs / mod variables to be processed - -# Parameters -# ---------- -# obs_vars : list -# list of observation variables - -# Returns -# ------- -# list -# list of observation variables (potentially extended from input -# list) -# list -# corresponding model variables which are mapped based on content -# of :attr:`model_add_vars` and :attr:`model_use_vars`. - -# """ -# obsout, modout = [], [] -# for obsvar in obs_vars: -# obsout.append(obsvar) -# if obsvar in self.model_use_vars: -# modout.append(self.model_use_vars[obsvar]) -# else: -# modout.append(obsvar) - -# for ovar, mvars in self.model_add_vars.items(): -# if not isinstance(mvars, list): -# raise AttributeError( -# f"values of model_add_vars need to be lists, even if " -# f"only single variables are to be added: " -# f"{self.model_add_vars}" -# ) -# for mvar in mvars: -# obsout.append(ovar) -# modout.append(mvar) -# return (obsout, modout) - -# def get_varname_web(self, mod_var, obs_var): -# if obs_var in self.model_add_vars and mod_var in self.model_add_vars[obs_var]: -# return mod_var -# return obs_var - -# def _get_aux_funcs_setup(self, funs): -# mra = {} -# for var, aux_info in self.model_read_aux.items(): -# mra[var] = check_aux_info(funcs=funs, **aux_info) -# return mra - -# def prep_dict_analysis(self, funs=None) -> dict: -# if funs is None: -# funs = {} -# output = deepcopy(self.to_dict()) -# if self.aux_funs_required: -# output["model_read_aux"].update(self._get_aux_funcs_setup(funs)) -# return output - - class ModelEntry(BaseModel): """Model configuration for evaluation (BaseModel) @@ -186,8 +44,6 @@ class ModelEntry(BaseModel): ## Pydantic ConfigDict model_config = ConfigDict( - # arbitrary_types_allowed=True, - # extra="allow", validate_assignment=True, protected_namespaces=(), ) @@ -243,12 +99,6 @@ def get_vars_to_process(self, obs_vars: tuple) -> tuple: modout.append(obsvar) for ovar, mvars in self.model_add_vars.items(): - # if not isinstance(mvars, tuple): - # raise AttributeError( - # f"values of model_add_vars need to be lists, even if " - # f"only single variables are to be added: " - # f"{self.model_add_vars}" - # ) for mvar in mvars: obsout.append(ovar) modout.append(mvar) From c322808aab4efe2cb265b4451cdd15dc1bde267b Mon Sep 17 00:00:00 2001 From: lewisblake Date: Sun, 10 Nov 2024 11:03:58 +0000 Subject: [PATCH 27/27] add _lowlevel_helpers.py without old classes --- pyaerocom/_lowlevel_helpers.py | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/pyaerocom/_lowlevel_helpers.py b/pyaerocom/_lowlevel_helpers.py index 60a89b3a9..ace6258ee 100644 --- a/pyaerocom/_lowlevel_helpers.py +++ b/pyaerocom/_lowlevel_helpers.py @@ -126,13 +126,6 @@ def validate(self, val): return val -# class DictType(Validator): -# def validate(self, val): -# if not isinstance(val, dict): -# raise ValueError(f"need dict, got {val}") -# return val - - class FlexList(Validator): """list that can be instantated via input str, tuple or list or None""" @@ -170,17 +163,6 @@ def validate(self, val): return val -# class DictStrKeysListVals(Validator): -# def validate(self, val: dict): -# if not isinstance(val, dict): -# raise ValueError(f"need dict, got {val}") -# if any(not isinstance(x, str) for x in val): -# raise ValueError(f"all keys need to be str type in {val}") -# if any(not isinstance(x, list) for x in val.values()): -# raise ValueError(f"all values need to be list type in {val}") -# return val - - class Loc(abc.ABC): """Abstract descriptor representing a path location