From 9ae08dac6e168a8ba85f9aa26589e2dfaac84887 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Tue, 3 Dec 2024 19:40:33 +0000 Subject: [PATCH] Per #3006, committing changes since the code is compiling. Added IODADataConfig_default file to store default settings for reading IODA data. --- data/config/IODADataConfig_default | 38 +++++ data/config/Makefile.am | 1 + data/config/Makefile.in | 1 + data/config/PairStatConfig_default | 134 +++++------------- data/table_files/Makefile.in | 1 + src/tools/core/pair_stat/pair_stat.cc | 129 ++++++++--------- src/tools/core/pair_stat/pair_stat.h | 49 ++----- .../core/pair_stat/pair_stat_conf_info.cc | 117 +++++++-------- .../core/pair_stat/pair_stat_conf_info.h | 43 +++--- 9 files changed, 225 insertions(+), 288 deletions(-) create mode 100644 data/config/IODADataConfig_default diff --git a/data/config/IODADataConfig_default b/data/config/IODADataConfig_default new file mode 100644 index 000000000..ec1bb571f --- /dev/null +++ b/data/config/IODADataConfig_default @@ -0,0 +1,38 @@ +//////////////////////////////////////////////////////////////////////////////// +// +// IODA Data configuration file. +// +// For additional information, please see the MET User's Guide. +// +//////////////////////////////////////////////////////////////////////////////// + +// +// Mapping of input IODA variable names to output variables names. +// The default IODA map, obs_var_map, is appended to this map. +// +obs_name_map = []; + +// +// Default mapping for Metadata. +// +metadata_map = [ + { key = "message_type"; val = "msg_type,station_ob"; }, + { key = "station_id"; val = "station_id,report_identifier"; }, + { key = "pressure"; val = "air_pressure,pressure"; }, + { key = "height"; val = "height,height_above_mean_sea_level"; }, + { key = "datetime"; val = "datetime,dateTime"; }, + { key = "elevation"; val = "elevation,station_elevation"; }, + { key = "nlocs"; val = "Location"; } +]; + +// +// Default mapping for obs to qc. +// +obs_to_qc_map = [ + { key = "wind_from_direction"; val = "eastward_wind,northward_wind"; }, + { key = "wind_speed"; val = "eastward_wind,northward_wind"; } +]; + +missing_thresh = [ <=-1e9, >=1e9, ==-9999 ]; + +//////////////////////////////////////////////////////////////////////////////// diff --git a/data/config/Makefile.am b/data/config/Makefile.am index 52f05a173..02678017c 100644 --- a/data/config/Makefile.am +++ b/data/config/Makefile.am @@ -27,6 +27,7 @@ config_DATA = \ GridStatConfig_default \ GridDiagConfig_default \ IODA2NCConfig_default \ + IODADataConfig_default \ Madis2NcConfig_default \ MODEAnalysisConfig_default \ MODEConfig_default \ diff --git a/data/config/Makefile.in b/data/config/Makefile.in index 76cdeca69..5268ac67b 100644 --- a/data/config/Makefile.in +++ b/data/config/Makefile.in @@ -318,6 +318,7 @@ config_DATA = \ GridStatConfig_default \ GridDiagConfig_default \ IODA2NCConfig_default \ + IODADataConfig_default \ Madis2NcConfig_default \ MODEAnalysisConfig_default \ MODEConfig_default \ diff --git a/data/config/PairStatConfig_default b/data/config/PairStatConfig_default index 41cb1ce77..f7cb77cd8 100644 --- a/data/config/PairStatConfig_default +++ b/data/config/PairStatConfig_default @@ -8,33 +8,20 @@ // // Output model name to be written +// Non-empty string overrides input data values // -model = "FCST"; +model = ""; // // Output description to be written -// May be set separately in each "obs.field" entry +// May be set separately in each "pairs" entry // desc = "NA"; //////////////////////////////////////////////////////////////////////////////// // -// Verification grid -// May be set separately in each "field" entry -// -regrid = { - to_grid = NONE; - method = NEAREST; - width = 1; - vld_thresh = 0.5; - shape = SQUARE; -} - -//////////////////////////////////////////////////////////////////////////////// - -// -// May be set separately in each "field" entry +// May be set separately in each "pairs" entry // censor_thresh = []; censor_val = []; @@ -50,43 +37,42 @@ hss_ec_value = NA; rank_corr_flag = FALSE; // -// Forecast and observation fields to be verified +// Forecast and observation pairs to be verified +// Parse "pairs" like we'd parse "field", supporting things like set_attr_name // + +pairs = [ + { fcst_name = , obs_name = , filters }, + { }, + +] + + fcst = { - field = [ - { - name = "SPFH"; - level = [ "P500" ]; - cat_thresh = [ >80.0 ]; - }, - - { - name = "TMP"; - level = [ "P500" ]; - cat_thresh = [ >273.0 ]; - }, - - { - name = "HGT"; - level = [ "P500" ]; - cat_thresh = [ >0.0 ]; - }, - - { - name = "UGRD"; - level = [ "P500" ]; - cat_thresh = [ >5.0 ]; - }, - - { - name = "VGRD"; - level = [ "P500" ]; - cat_thresh = [ >5.0 ]; - } - ]; + pairs = [ + { + name = "TMP"; level = "P100-1000"... P100, P200, XXX P1100 // IODA forecast variable name or comma-separated list of FCST_VAR names + filter_str_inc = [ { name = "FCST_LEVEL"; value = "P100,P200,P300"; } ]; + filter_str_exc = [ { name = ""; value = ""; } ]; + filter_thresh = [ { name = ""; thresh = NA; } ]; + cat_thresh = [ >273, >283 ]; + cnt_thresh = [ NA, >273 ]; + } + ]; +} +obs = { + pairs = [ + { + name = "TMP"; // IODA observation variable name or comma-separated list of OBS_VAR names + filter_str_inc = [ { name = "OBS_LEVEL"; value = "Z2"; } ]; + filter_str_exc = [ { name = ""; value = ""; } ]; + filter_thresh = [ { name = ""; thresh = NA; } ]; + cat_thresh = [ >273, >283 ]; + cnt_thresh = [ NA, >273 ]; + } + ]; } -obs = fcst; //////////////////////////////////////////////////////////////////////////////// @@ -103,24 +89,9 @@ duplicate_flag = NONE; obs_summary = NONE; obs_perc_value = 50; -// -// Mapping of message type group name to comma-separated list of values. -// -message_type_group_map = [ - { key = "SURFACE"; val = "ADPSFC,SFCSHP,MSONET"; }, - { key = "ANYAIR"; val = "AIRCAR,AIRCFT"; }, - { key = "ANYSFC"; val = "ADPSFC,SFCSHP,ADPUPA,PROFLR,MSONET"; }, - { key = "ONLYSF"; val = "ADPSFC,SFCSHP"; }, - { key = "LANDSF"; val = "ADPSFC,MSONET"; }, - { key = "WATERSF"; val = "SFCSHP"; } -]; - -obtype_as_group_val_flag = FALSE; - //////////////////////////////////////////////////////////////////////////////// // -// Climatology mean data // May be set separately in the "fcst" and "obs" dictionaries // climo_mean = { @@ -236,39 +207,6 @@ boot = { //////////////////////////////////////////////////////////////////////////////// -// -// Interpolation methods -// May be set separately in each "obs.field" entry -// -interp = { - vld_thresh = 1.0; - shape = SQUARE; - - type = [ - { - method = NEAREST; - width = 1; - } - ]; -} - -//////////////////////////////////////////////////////////////////////////////// - -// -// HiRA verification method -// May be set separately in each "obs.field" entry -// -hira = { - flag = FALSE; - width = [ 2, 3, 4, 5 ]; - vld_thresh = 1.0; - cov_thresh = [ ==0.25 ]; - shape = SQUARE; - prob_cat_thresh = []; -} - -//////////////////////////////////////////////////////////////////////////////// - // // Threshold for SEEPS p1 (Probability of being dry) // diff --git a/data/table_files/Makefile.in b/data/table_files/Makefile.in index 0fe32fe8e..75aeaa2b4 100644 --- a/data/table_files/Makefile.in +++ b/data/table_files/Makefile.in @@ -310,6 +310,7 @@ top_builddir = @top_builddir@ top_srcdir = @top_srcdir@ tablefilesdir = $(pkgdatadir)/table_files tablefiles_DATA = \ + met_header_columns_V12.1.txt \ met_header_columns_V12.0.txt \ met_header_columns_V11.1.txt \ met_header_columns_V11.0.txt \ diff --git a/src/tools/core/pair_stat/pair_stat.cc b/src/tools/core/pair_stat/pair_stat.cc index 815a58e23..51b426fdd 100644 --- a/src/tools/core/pair_stat/pair_stat.cc +++ b/src/tools/core/pair_stat/pair_stat.cc @@ -47,10 +47,6 @@ #include "nc_obs_util.h" #include "nc_point_obs_in.h" -#ifdef WITH_UGRID -#include "vx_data2d_ugrid.h" -#endif - #ifdef WITH_PYTHON #include "data2d_nc_met.h" #include "pointdata_python.h" @@ -72,8 +68,8 @@ static void setup_table (AsciiTable &); static void build_outfile_name(unixtime, int, const char *, ConcatString &); -static void process_fcst_climo_files(); -static void process_obs_file(int); +static void process_mpr_pairs(const ConcatString &, PairsFormat); +static void process_ioda_pairs(const ConcatString &); static void process_scores(); static void do_cts (CTSInfo *&, int, const PairDataPoint *); @@ -93,29 +89,35 @@ static void usage(); static void set_pairs(const StringArray &); static void set_format(const StringArray &); static void set_config(const StringArray &); -#ifdef WITH_UGRID -static void set_ugrid_config(const StringArray &); -#endif static void set_outdir(const StringArray &); //////////////////////////////////////////////////////////////////////// int met_main(int argc, char *argv[]) { - int i; // Process the command line arguments process_command_line(argc, argv); - // Process the forecast and climo files - process_fcst_climo_files(); + // Process each pairs file + for(int i=0; i " - << "The configuration file \"" << ugrid_config_names[i]<< "\" does not exist.\n\n"; + // Read each specified config file in order + for(int i=0; iset_dict(fdict); diff --git a/src/tools/core/pair_stat/pair_stat_conf_info.h b/src/tools/core/pair_stat/pair_stat_conf_info.h index 73446bb91..c24434b03 100644 --- a/src/tools/core/pair_stat/pair_stat_conf_info.h +++ b/src/tools/core/pair_stat/pair_stat_conf_info.h @@ -87,11 +87,29 @@ static const STATLineType txt_file_type[n_txt] = { STATLineType::seeps // 21 }; -//////////////////////////////////////////////////////////////////////// +/////////////////////////////////////////////////////////////////////////////// + +// +// Supported input pairs formats +// + +enum class PairsFormat { + None, // Default + MPR, // ASCII files containing MET MPR lines + Python, // Stat MPR data via Python embedding + IODA, // IODA pairs file +}; + +/////////////////////////////////////////////////////////////////////////////// + +extern ConcatString pairsformat_to_string(const PairsFormat); +extern PairsFormat string_to_pairsformat(const std::string &); + +/////////////////////////////////////////////////////////////////////////////// class PairStatConfInfo; // forward reference -//////////////////////////////////////////////////////////////////////// +/////////////////////////////////////////////////////////////////////////////// class PairStatVxOpt { @@ -163,7 +181,7 @@ class PairStatVxOpt { void clear(); - void process_config(GrdFileType, Dictionary &, Dictionary &); + void process_config(PairsFormat, Dictionary &, Dictionary &); void set_vx_pd(PairStatConfInfo *); bool is_uv_match(const PairStatVxOpt &) const; @@ -252,14 +270,6 @@ class PairStatConfInfo { ConcatString seeps_climo_name; // SEESP climo filename SingleThresh seeps_p1_thresh; // SEESP p1 threshold -#ifdef WITH_UGRID - bool ignore_ugrid_dataset; - ConcatString ugrid_nc; // NetCDF for coordinate variables of unstructured grid - ConcatString ugrid_dataset; // UGRid dataset name (mpas, lfric etc) - ConcatString ugrid_map_config; // User's configuration file which contains ugrid metadata mapping - double ugrid_max_distance_km; // max distance to be the closest neighbor to unstructured grid -#endif - // Summary of output file options across all verification tasks STATOutputType output_flag[n_txt]; // Flag for each output line type @@ -267,16 +277,13 @@ class PairStatConfInfo { void clear(); - void read_config(const char *, const char *); -#ifdef WITH_UGRID - void read_ugrid_configs(StringArray ugrid_config_names, const char * user_config); -#endif + void read_config(const StringArray &); - void process_config(GrdFileType); + void process_config(PairsFormat); void process_grib_codes(); void process_flags(); - void process_masks(const Grid &); - void process_geog(const Grid &, const char *); + void process_masks(); + void process_geog(); void set_vx_pd(); // Dump out the counts