Merge branch 'main-dev' into log-file

pyaerocom/aeroval/coldatatojson_helpers.py

@@ -1075,13 +1075,15 @@ def _calc_temporal_corr(coldata):
         return np.nan, np.nan
     elif coldata.has_latlon_dims:
         coldata = coldata.flatten_latlondim_station_name()
-    arr = coldata.data
+
     # Use only sites that contain at least 3 valid data points (otherwise
     # correlation will be 1).
-    obs_ok = arr[0].count(dim="time") > 2
-    arr[0] = arr[0].where(obs_ok, drop=True)
-    arr[1] = arr[1].where(obs_ok, drop=True)
-    if np.prod(arr.shape) == 0:
+    obs_ok = coldata.data[0].count(dim="time") > 2
+    arr = []
+    arr.append(coldata.data[0].where(obs_ok, drop=True))
+    arr.append(coldata.data[1].where(obs_ok, drop=True))
+
+    if np.prod(arr[0].shape) == 0 or np.prod(arr[1].shape) == 0:
         return np.nan, np.nan
     corr_time = xr.corr(arr[1], arr[0], dim="time")
     with ignore_warnings(RuntimeWarning, "Mean of empty slice", "All-NaN slice encountered"):

pyaerocom/colocation.py

@@ -723,6 +723,9 @@ def colocate_gridded_ungridded(
     if regrid_res_deg is not None:
         data = _regrid_gridded(data, regrid_scheme, regrid_res_deg)
 
+    # Special ts_typs for which all stations with ts_type< are removed
+    reduce_station_data_ts_type = ts_type
+
     ts_type_src_data = data.ts_type
     ts_type, ts_type_data = _check_ts_type(data, ts_type)
     if not colocate_time and ts_type < ts_type_data:
@@ -751,7 +754,12 @@ def colocate_gridded_ungridded(
     # get timeseries from all stations in provided time resolution
     # (time resampling is done below in main loop)
     all_stats = data_ref.to_station_data_all(
-        vars_to_convert=var_ref, start=obs_start, stop=obs_stop, by_station_name=True, **kwargs
+        vars_to_convert=var_ref,
+        start=obs_start,
+        stop=obs_stop,
+        by_station_name=True,
+        ts_type_preferred=reduce_station_data_ts_type,
+        **kwargs,
     )
 
     obs_stat_data = all_stats["stats"]

pyaerocom/ungriddeddata.py

@@ -35,6 +35,8 @@
 from pyaerocom.stationdata import StationData
 from pyaerocom.units_helpers import get_unit_conversion_fac
 
+from .tstype import TsType
+
 logger = logging.getLogger(__name__)
 
 
@@ -807,6 +809,7 @@ def to_station_data(
         start=None,
         stop=None,
         freq=None,
+        ts_type_preferred=None,
         merge_if_multi=True,
         merge_pref_attr=None,
         merge_sort_by_largest=True,
@@ -901,6 +904,17 @@ def to_station_data(
                 stat = self._metablock_to_stationdata(
                     idx, vars_to_convert, start, stop, add_meta_keys
                 )
+                if ts_type_preferred is not None:
+                    if "ts_type" in stat["var_info"][vars_to_convert[0]].keys():
+                        if TsType(stat["var_info"][vars_to_convert[0]]["ts_type"]) < TsType(
+                            ts_type_preferred
+                        ):
+                            continue
+                    elif "ts_type" in stat.keys():
+                        if TsType(stat["ts_type"]) < TsType(ts_type_preferred):
+                            continue
+                    else:
+                        raise KeyError("Could not find ts_type in stat")
                 stats.append(stat)
             except (VarNotAvailableError, DataCoverageError) as e:
                 logger.info(f"Skipping meta index {idx}. Reason: {repr(e)}")
@@ -1167,6 +1181,7 @@ def to_station_data_all(
         start=None,
         stop=None,
         freq=None,
+        ts_type_preferred=None,
         by_station_name=True,
         ignore_index=None,
         **kwargs,
@@ -1223,6 +1238,7 @@ def to_station_data_all(
                     freq,
                     merge_if_multi=True,
                     allow_wildcards_station_name=False,
+                    ts_type_preferred=ts_type_preferred,
                     **kwargs,
                 )