Merge pull request #758 from metno/split-map-files-by-year

Split map files by year

pyaerocom/aeroval/coldatatojson_engine.py

@@ -225,27 +225,32 @@ def process_coldata(self, coldata: ColocatedData):
 
             _write_site_data(ts_objs, out_dirs["ts"])
 
-            map_data, scat_data = _process_map_and_scat(
-                data,
-                map_meta,
-                site_indices,
-                periods,
-                main_freq,
-                stats_min_num,
-                seasons,
-                add_trends,
-                trends_min_yrs,
-                use_fairmode,
-                obs_var,
-            )
-
-            map_name = get_json_mapname(obs_name, var_name_web, model_name, model_var, vert_code)
+            logger.info("Processing map and scat data by period")
+            for period in periods:
+                # compute map_data and scat_data just for this period
+                map_data, scat_data = _process_map_and_scat(
+                    data,
+                    map_meta,
+                    site_indices,
+                    [period],
+                    main_freq,
+                    stats_min_num,
+                    seasons,
+                    add_trends,
+                    trends_min_yrs,
+                    use_fairmode,
+                    obs_var,
+                )
 
-            outfile_map = os.path.join(out_dirs["map"], map_name)
-            write_json(map_data, outfile_map, ignore_nan=True)
+                # the files in /map and /scat will be split up according to their time period as well
+                map_name = get_json_mapname(
+                    obs_name, var_name_web, model_name, model_var, vert_code, period
+                )
+                outfile_map = os.path.join(out_dirs["map"], map_name)
+                write_json(map_data, outfile_map, ignore_nan=True)
 
-            outfile_scat = os.path.join(out_dirs["scat"], map_name)
-            write_json(scat_data, outfile_scat, ignore_nan=True)
+                outfile_scat = os.path.join(out_dirs["scat"], map_name)
+                write_json(scat_data, outfile_scat, ignore_nan=True)
 
         if coldata.ts_type == "hourly" and use_diurnal:
             logger.info("Processing diurnal profiles")

pyaerocom/aeroval/coldatatojson_helpers.py

@@ -46,9 +46,9 @@ def get_stationfile_name(station_name, obs_name, var_name_web, vert_code):
     return f"{station_name}_{obs_name}-{var_name_web}_{vert_code}.json"
 
 
-def get_json_mapname(obs_name, var_name_web, model_name, model_var, vert_code):
+def get_json_mapname(obs_name, var_name_web, model_name, model_var, vert_code, period):
     """Get name base name of json file"""
-    return f"{obs_name}-{var_name_web}_{vert_code}_{model_name}-{model_var}.json"
+    return f"{obs_name}-{var_name_web}_{vert_code}_{model_name}-{model_var}_{period}.json"
 
 
 def _write_stationdata_json(ts_data, out_dir):

pyaerocom/aeroval/experiment_output.py

@@ -307,15 +307,16 @@ def _info_from_map_file(filename):
             name of model variable
         """
         spl = os.path.basename(filename).split(".json")[0].split("_")
-        if len(spl) != 3:
+        if len(spl) != 4:
             raise ValueError(
                 f"invalid map filename: {filename}. Must "
-                f"contain exactly 2 underscores _ to separate "
-                f"obsinfo, vertical and model info"
+                f"contain exactly 3 underscores _ to separate "
+                f"obsinfo, vertical, model info, and periods"
             )
         obsinfo = spl[0]
         vert_code = spl[1]
         modinfo = spl[2]
+        per = spl[3]
 
         mspl = modinfo.split("-")
         mvar = mspl[-1]
@@ -324,18 +325,19 @@ def _info_from_map_file(filename):
         ospl = obsinfo.split("-")
         ovar = ospl[-1]
         oname = "-".join(ospl[:-1])
-        return (oname, ovar, vert_code, mname, mvar)
+
+        return (oname, ovar, vert_code, mname, mvar, per)
 
     def _results_summary(self):
-        res = [[], [], [], [], []]
+        res = [[], [], [], [], [], []]
         files = self._get_json_output_files("map")
         tab = []
         for file in files:
             item = self._info_from_map_file(file)
             for i, entry in enumerate(item):
                 res[i].append(entry)
         output = {}
-        for i, name in enumerate(["obs", "ovar", "vc", "mod", "mvar"]):
+        for i, name in enumerate(["obs", "ovar", "vc", "mod", "mvar", "per"]):
             output[name] = list(set(res[i]))
         return output
 
@@ -355,7 +357,14 @@ def clean_json_files(self):
         vert_codes = self.cfg.obs_cfg.all_vert_types
         for file in mapfiles:
             try:
-                (obs_name, obs_var, vert_code, mod_name, mod_var) = self._info_from_map_file(file)
+                (
+                    obs_name,
+                    obs_var,
+                    vert_code,
+                    mod_name,
+                    mod_var,
+                    period,
+                ) = self._info_from_map_file(file)
             except Exception as e:
                 logger.warning(
                     f"FATAL: invalid file convention for map json file:"
@@ -731,7 +740,9 @@ def _create_menu_dict(self):
         new = {}
         files = self._get_json_output_files("map")
         for file in files:
-            (obs_name, obs_var, vert_code, mod_name, mod_var) = self._info_from_map_file(file)
+            (obs_name, obs_var, oper, vert_code, mod_name, mod_var) = self._info_from_map_file(
+                file
+            )
 
             if self._is_part_of_experiment(obs_name, obs_var, mod_name, mod_var):
 

pyaerocom/aeroval/glob_defaults.py

@@ -54,6 +54,7 @@
     "wetoxs": {"scale": [0, 1.25, 2.5, 3.75, 5, 6.25, 7.5, 8.75, 10], "colmap": "coolwarm"},
     "wetoxn": {"scale": [0, 1.25, 2.5, 3.75, 5, 6.25, 7.5, 8.75, 10], "colmap": "coolwarm"},
     "wetrdn": {"scale": [0, 1.25, 2.5, 3.75, 5, 6.25, 7.5, 8.75, 10], "colmap": "coolwarm"},
+    "prmm": {"scale": [0, 5, 10, 15, 20, 25, 30, 35, 40], "colmap": "coolwarm"},
     "dryoxs": {"scale": [0, 1.25, 2.5, 3.75, 5, 6.25, 7.5, 8.75, 10], "colmap": "coolwarm"},
     "dryoxn": {"scale": [0, 1.25, 2.5, 3.75, 5, 6.25, 7.5, 8.75, 10], "colmap": "coolwarm"},
     "dryrdn": {"scale": [0, 1.25, 2.5, 3.75, 5, 6.25, 7.5, 8.75, 10], "colmap": "coolwarm"},

tests/aeroval/test_aeroval_HIGHLEV.py

@@ -10,37 +10,37 @@
 from tests.conftest import geojson_unavail
 
 CHK_CFG1 = {
-    "map": ["AERONET-Sun-od550aer_Column_TM5-AP3-CTRL-od550aer.json"],
+    "map": ["AERONET-Sun-od550aer_Column_TM5-AP3-CTRL-od550aer_2010.json"],
     "contour": ["od550aer_TM5-AP3-CTRL.geojson", "od550aer_TM5-AP3-CTRL.json"],
     "hm": ["glob_stats_daily.json", "glob_stats_monthly.json", "glob_stats_yearly.json"],
     "hm/ts": 10,  # number of .json files in sub dir
-    "scat": ["AERONET-Sun-od550aer_Column_TM5-AP3-CTRL-od550aer.json"],
+    "scat": ["AERONET-Sun-od550aer_Column_TM5-AP3-CTRL-od550aer_2010.json"],
     "ts": 11,  # number of .json files in subdir
     "ts/diurnal": 0,  # number of .json files in subdir
 }
 
 CHK_CFG2 = {
     "map": [
-        "AERONET-Sun-od550aer_Column_TM5-AP3-CTRL-od550aer.json",
-        "AERONET-SDA-od550aer_Column_TM5-AP3-CTRL-od550aer.json",
+        "AERONET-Sun-od550aer_Column_TM5-AP3-CTRL-od550aer_2010.json",
+        "AERONET-SDA-od550aer_Column_TM5-AP3-CTRL-od550aer_2010.json",
     ],
     "contour": 0,
     "hm": ["glob_stats_monthly.json"],
     "hm/ts": 21,  # number of .json files in subdir
     "scat": [
-        "AERONET-Sun-od550aer_Column_TM5-AP3-CTRL-od550aer.json",
-        "AERONET-SDA-od550aer_Column_TM5-AP3-CTRL-od550aer.json",
+        "AERONET-Sun-od550aer_Column_TM5-AP3-CTRL-od550aer_2010.json",
+        "AERONET-SDA-od550aer_Column_TM5-AP3-CTRL-od550aer_2010.json",
     ],
     "ts": 40,  # number of .json files in subdir
     "ts/diurnal": 0,  # number of .json files in subdir
 }
 
 CHK_CFG4 = {
-    "map": ["SDA-and-Sun-od550aer_Column_TM5-AP3-CTRL-od550aer.json"],
+    "map": ["SDA-and-Sun-od550aer_Column_TM5-AP3-CTRL-od550aer_2010.json"],
     "contour": 0,
     "hm": ["glob_stats_monthly.json"],
     "hm/ts": 10,  # number of .json files in subdir
-    "scat": ["SDA-and-Sun-od550aer_Column_TM5-AP3-CTRL-od550aer.json"],
+    "scat": ["SDA-and-Sun-od550aer_Column_TM5-AP3-CTRL-od550aer_2010.json"],
     "ts": 21,  # number of .json files in subdir
     "ts/diurnal": 0,  # number of .json files in subdir
 }

tests/aeroval/test_coldatatojson_helpers2.py

@@ -47,8 +47,8 @@ def test_get_stationfile_name():
 
 
 def test_get_json_mapname():
-    json = get_json_mapname("obs1", "var1", "mod1", "var1", "Column")
-    assert json == "obs1-var1_Column_mod1-var1.json"
+    json = get_json_mapname("obs1", "var1", "mod1", "var1", "Column", "period")
+    assert json == "obs1-var1_Column_mod1-var1_period.json"
 
 
 @pytest.mark.parametrize(

tests/aeroval/test_experiment_output.py

@@ -167,29 +167,30 @@ def test_ExperimentOutput_update_heatmap_json_EMPTY(dummy_expout: ExperimentOutp
 
 def test_ExperimentOutput__info_from_map_file():
     output = ExperimentOutput._info_from_map_file(
-        "EBAS-2010-ac550aer_Surface_ECHAM-HAM-ac550dryaer.json"
+        "EBAS-2010-ac550aer_Surface_ECHAM-HAM-ac550dryaer_2010.json"
     )
-    assert output == ("EBAS-2010", "ac550aer", "Surface", "ECHAM-HAM", "ac550dryaer")
+
+    assert output == ("EBAS-2010", "ac550aer", "Surface", "ECHAM-HAM", "ac550dryaer", "2010")
 
 
 @pytest.mark.parametrize(
     "filename",
     [
         "blaaaa",
-        "EBAS-2010-ac550aer_Surface_ECHAM-HAM_ac550dryaer.json",
+        "EBAS-2010-ac550aer_Surface_ECHAM-HAM_ac550dryaer_2010.json",  # has four underscores
     ],
 )
 def test_ExperimentOutput__info_from_map_file_error(filename: str):
     with pytest.raises(ValueError) as e:
         ExperimentOutput._info_from_map_file(filename)
     assert str(e.value) == (
         f"invalid map filename: {filename}. "
-        "Must contain exactly 2 underscores _ to separate obsinfo, vertical and model info"
+        "Must contain exactly 3 underscores _ to separate obsinfo, vertical, model info, and periods"
     )
 
 
 def test_ExperimentOutput__results_summary_EMPTY(dummy_expout: ExperimentOutput):
-    assert dummy_expout._results_summary() == dict(obs=[], ovar=[], vc=[], mod=[], mvar=[])
+    assert dummy_expout._results_summary() == dict(obs=[], ovar=[], vc=[], mod=[], mvar=[], per=[])
 
 
 def test_ExperimentOutput_clean_json_files_EMPTY(dummy_expout: ExperimentOutput):