From 4d9cdc2741cb1aa9f8ea4b469c99fe137b4b19a6 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Tue, 22 Feb 2022 17:35:54 -0700 Subject: [PATCH 01/16] Per #1259, this is a work in progress. Added hooks for PCT, PSTD, PRC, PRJ, and ECLV outputs from ensemble-stat. Also added fcat_ta and ocat_ta ThreshArray objects to store the needed fcst and obs thresholds. Still need to actually update ensemble_stat.cc to use them and also need to make config file, unit test, and documentation updates. --- met/data/config/EnsembleStatConfig_default | 5 ++ .../tools/core/ensemble_stat/ensemble_stat.cc | 15 ++-- .../tools/core/ensemble_stat/ensemble_stat.h | 12 +-- .../ensemble_stat/ensemble_stat_conf_info.cc | 88 +++++++++++++++---- .../ensemble_stat/ensemble_stat_conf_info.h | 68 ++++++++------ 5 files changed, 130 insertions(+), 58 deletions(-) diff --git a/met/data/config/EnsembleStatConfig_default b/met/data/config/EnsembleStatConfig_default index 339ab939be..d105a032d0 100644 --- a/met/data/config/EnsembleStatConfig_default +++ b/met/data/config/EnsembleStatConfig_default @@ -261,6 +261,11 @@ output_flag = { orank = NONE; ssvar = NONE; relp = NONE; + pct = NONE; + pstd = NONE; + pjc = NONE; + prc = NONE; + eclv = NONE; } //////////////////////////////////////////////////////////////////////////////// diff --git a/met/src/tools/core/ensemble_stat/ensemble_stat.cc b/met/src/tools/core/ensemble_stat/ensemble_stat.cc index ce0784e84e..b8ee623b14 100644 --- a/met/src/tools/core/ensemble_stat/ensemble_stat.cc +++ b/met/src/tools/core/ensemble_stat/ensemble_stat.cc @@ -65,6 +65,7 @@ // 033 11/15/21 Halley Gotway MET #1968 Ensemble -ctrl error check. // 034 01/14/21 McCabe MET #1695 All members in one file. // 035 02/15/22 Halley Gotway MET #1583 Add HiRA option. +// 036 02/20/22 Halley Gotway MET #1259 Write probabilistic statistics. // //////////////////////////////////////////////////////////////////////// @@ -403,10 +404,10 @@ void process_command_line(int argc, char **argv) { shc.set_model(conf_info.model.c_str()); // Allocate arrays to store threshold counts - thresh_cnt_na = new NumArray [conf_info.get_max_n_thresh()]; - thresh_nbrhd_cnt_na = new NumArray * [conf_info.get_max_n_thresh()]; + thresh_cnt_na = new NumArray [conf_info.get_max_n_ens_thresh()]; + thresh_nbrhd_cnt_na = new NumArray * [conf_info.get_max_n_ens_thresh()]; - for(i=0; icat_ta.n() > max_n_thresh) { - max_n_thresh = ens_info->cat_ta.n(); + if(ens_info->cat_ta.n() > max_n_ens_thresh) { + max_n_ens_thresh = ens_info->cat_ta.n(); } } @@ -430,7 +431,7 @@ void EnsembleStatConfInfo::process_config(GrdFileType etype, } // Parse settings for each verification task - for(i=0; i " + << "The number of thresholds for each field in \"fcst." + << conf_key_cat_thresh + << "\" must match the number of thresholds for each " + << "field in \"obs." << conf_key_cat_thresh << "\".\n\n"; + exit(1); + } + // Conf: prob_cat_thresh - prob_cat_ta = fdict.lookup_thresh_array(conf_key_prob_cat_thresh); + pcat_ta = fdict.lookup_thresh_array(conf_key_prob_cat_thresh); // Conf: duplicate_flag duplicate_flag = parse_conf_duplicate_flag(&odict); @@ -1107,7 +1127,12 @@ void EnsembleStatVxOpt::set_vx_pd(EnsembleStatConfInfo *conf_info, int ctrl_inde void EnsembleStatVxOpt::set_perc_thresh(const PairDataEnsemble *pd_ptr) { - if(!othr_ta.need_perc()) return; + // + // Check if percentile thresholds were requested + // + if(!othr_ta.need_perc() && + !fcat_ta.need_perc() && + !ocat_ta.need_perc()) return; // // Sort the input arrays @@ -1125,6 +1150,8 @@ void EnsembleStatVxOpt::set_perc_thresh(const PairDataEnsemble *pd_ptr) { // the fcst and obs slots. // othr_ta.set_perc(&fsort, &osort, &csort, &othr_ta, &othr_ta); + fcat_ta.set_perc(&fsort, &osort, &csort, &fcat_ta, &ocat_ta); + ocat_ta.set_perc(&fsort, &osort, &csort, &fcat_ta, &ocat_ta); return; } @@ -1154,8 +1181,7 @@ int EnsembleStatVxOpt::n_txt_row(int i_txt_row) const { // Point Vx: Message Types * Masks * Interpolations * Obs Thresholds * Alphas // Grid Vx: Masks * Interpolations * Obs Thresholds * Alphas n = (get_n_msg_typ() + 1) * get_n_mask() * get_n_interp() * - get_n_o_thresh() * get_n_ci_alpha(); - + get_n_obs_thresh() * get_n_ci_alpha(); break; case(i_rhist): @@ -1166,8 +1192,7 @@ int EnsembleStatVxOpt::n_txt_row(int i_txt_row) const { // Point Vx: Message Types * Masks * Interpolations * Obs Thresholds // Grid Vx: Masks * Interpolations * Obs Thresholds n = (get_n_msg_typ() + 1) * get_n_mask() * get_n_interp() * - get_n_o_thresh(); - + get_n_obs_thresh(); break; case(i_orank): @@ -1177,8 +1202,7 @@ int EnsembleStatVxOpt::n_txt_row(int i_txt_row) const { // Number of ORANK lines possible = // Number of pairs * Obs Thresholds - n = vx_pd.get_n_pair() * get_n_o_thresh(); - + n = vx_pd.get_n_pair() * get_n_obs_thresh(); break; case(i_ssvar): @@ -1188,6 +1212,34 @@ int EnsembleStatVxOpt::n_txt_row(int i_txt_row) const { n = 0; break; + case(i_pct): + case(i_pjc): + case(i_prc): + + // Maximum number of PCT, PJC, and PRC lines possible = + // Point Vx: Message Types * Masks * Interpolations * Categorical Thresholds + // Grid Vx: Masks * Interpolations * Categorical Thresholds + n = (get_n_msg_typ() + 1) * get_n_mask() * get_n_interp() * + get_n_cat_thresh(); + break; + + case(i_pstd): + + // Maximum number of PSTD lines possible = + // Point Vx: Message Types * Masks * Interpolations * Categorical Thresholds * Alphas + // Grid Vx: Masks * Interpolations * Categorical Thresholds * Alphas + n = (get_n_msg_typ() + 1) * get_n_mask() * get_n_interp() * + get_n_cat_thresh() * get_n_ci_alpha(); + break; + + case(i_eclv): + + // Maximum number of ECLV lines possible = + // Point Vx: Message Types * Masks * Interpolations * Probability Thresholds + // Grid Vx: Masks * Interpolations * Probability Thresholds + n = (get_n_msg_typ() + 1) * get_n_mask() * get_n_interp() * + get_n_prob_thresh() * get_n_prob_thresh(); + default: mlog << Error << "\nEnsembleStatVxOpt::n_txt_row(int) -> " << "unexpected output type index value: " << i_txt_row diff --git a/met/src/tools/core/ensemble_stat/ensemble_stat_conf_info.h b/met/src/tools/core/ensemble_stat/ensemble_stat_conf_info.h index ceaedf50d0..1b98a04dc4 100644 --- a/met/src/tools/core/ensemble_stat/ensemble_stat_conf_info.h +++ b/met/src/tools/core/ensemble_stat/ensemble_stat_conf_info.h @@ -34,12 +34,18 @@ static const int i_phist = 3; static const int i_orank = 4; static const int i_ssvar = 5; static const int i_relp = 6; -static const int n_txt = 7; +static const int i_pct = 7; +static const int i_pstd = 8; +static const int i_pjc = 9; +static const int i_prc = 10; +static const int i_eclv = 11; +static const int n_txt = 12; // Text file type static const STATLineType txt_file_type[n_txt] = { stat_ecnt, stat_rps, stat_rhist, stat_phist, - stat_orank, stat_ssvar, stat_relp + stat_orank, stat_ssvar, stat_relp, stat_pct, + stat_pstd, stat_pjc, stat_prc, stat_eclv }; //////////////////////////////////////////////////////////////////////// @@ -111,9 +117,7 @@ class EnsembleStatVxOpt { StringArray mask_name_area; // Masking area (grid + poly) region names StringArray msg_typ; // Verifying message types - ThreshArray othr_ta; // Observation filtering thresholds - ClimoCDFInfo cdf_info; // Climo CDF info NumArray ci_alpha; // Alpha value for confidence intervals @@ -122,7 +126,10 @@ class EnsembleStatVxOpt { double ssvar_bin_size; // SSVAR bin size double phist_bin_size; // PHIST bin size - ThreshArray prob_cat_ta; // Categorical thresholds for probabilities + + ThreshArray fcat_ta; // Forecast categorical thresholds to derive probabilities + ThreshArray ocat_ta; // Observation categorical thresholds for verifying events + ThreshArray pcat_ta; // Categorical thresholds to evaluate derived probabilities DuplicateType duplicate_flag; // Duplicate observations ObsSummary obs_summary; // Summarize observations @@ -154,20 +161,24 @@ class EnsembleStatVxOpt { int get_n_mask() const; int get_n_mask_area() const; - int get_n_o_thresh() const; - int get_n_cdf_bin() const; - int get_n_ci_alpha() const; + int get_n_obs_thresh() const; + int get_n_cat_thresh() const; + int get_n_prob_thresh() const; + int get_n_cdf_bin() const; + int get_n_ci_alpha() const; }; //////////////////////////////////////////////////////////////////////// -inline int EnsembleStatVxOpt::get_n_msg_typ() const { return(msg_typ.n()); } -inline int EnsembleStatVxOpt::get_n_interp() const { return(interp_info.n_interp); } -inline int EnsembleStatVxOpt::get_n_mask() const { return(mask_name.n()); } -inline int EnsembleStatVxOpt::get_n_mask_area() const { return(mask_name_area.n()); } -inline int EnsembleStatVxOpt::get_n_o_thresh() const { return(othr_ta.n()); } -inline int EnsembleStatVxOpt::get_n_cdf_bin() const { return(cdf_info.n_bin); } -inline int EnsembleStatVxOpt::get_n_ci_alpha() const { return(ci_alpha.n()); } +inline int EnsembleStatVxOpt::get_n_msg_typ() const { return(msg_typ.n()); } +inline int EnsembleStatVxOpt::get_n_interp() const { return(interp_info.n_interp); } +inline int EnsembleStatVxOpt::get_n_mask() const { return(mask_name.n()); } +inline int EnsembleStatVxOpt::get_n_mask_area() const { return(mask_name_area.n()); } +inline int EnsembleStatVxOpt::get_n_obs_thresh() const { return(othr_ta.n()); } +inline int EnsembleStatVxOpt::get_n_cat_thresh() const { return(ocat_ta.n()); } +inline int EnsembleStatVxOpt::get_n_prob_thresh() const { return(pcat_ta.n()); } +inline int EnsembleStatVxOpt::get_n_cdf_bin() const { return(cdf_info.n_bin); } +inline int EnsembleStatVxOpt::get_n_ci_alpha() const { return(ci_alpha.n()); } //////////////////////////////////////////////////////////////////////// @@ -178,12 +189,13 @@ class EnsembleStatConfInfo { void init_from_scratch(); // Ensemble processing - int n_ens_var; // Number of ensemble fields to be processed - int max_n_thresh; // Maximum number of ensemble thresholds - int max_hira_size; // Maximum size of a HiRA neighborhoods + int n_ens_var; // Number of ensemble fields to be processed + int max_n_ens_thresh; // Maximum number of ensemble thresholds // Ensemble verification - int n_vx; // Number of ensemble fields to be verified + int n_vx; // Number of ensemble fields to be verified + int max_hira_size; // Maximum size of a HiRA neighborhoods + int max_n_cat_thresh; // Maximum number of categorical thresholds public: @@ -256,20 +268,22 @@ class EnsembleStatConfInfo { int n_stat_row() const; // Maximum across all verification tasks - int get_max_n_thresh() const; - int get_max_hira_size() const; + int get_max_n_ens_thresh() const; + int get_max_hira_size() const; + int get_max_n_cat_thresh() const; int get_compression_level(); }; //////////////////////////////////////////////////////////////////////// -inline int EnsembleStatConfInfo::get_n_ens_var() const { return(n_ens_var); } -inline int EnsembleStatConfInfo::get_n_nbrhd() const { return(n_nbrhd); } -inline int EnsembleStatConfInfo::get_n_vx() const { return(n_vx); } -inline int EnsembleStatConfInfo::get_max_n_thresh() const { return(max_n_thresh); } -inline int EnsembleStatConfInfo::get_max_hira_size() const { return(max_hira_size); } -inline int EnsembleStatConfInfo::get_compression_level() { return(conf.nc_compression()); } +inline int EnsembleStatConfInfo::get_n_ens_var() const { return(n_ens_var); } +inline int EnsembleStatConfInfo::get_n_nbrhd() const { return(n_nbrhd); } +inline int EnsembleStatConfInfo::get_n_vx() const { return(n_vx); } +inline int EnsembleStatConfInfo::get_max_n_ens_thresh() const { return(max_n_ens_thresh); } +inline int EnsembleStatConfInfo::get_max_hira_size() const { return(max_hira_size); } +inline int EnsembleStatConfInfo::get_max_n_cat_thresh() const { return(max_n_cat_thresh); } +inline int EnsembleStatConfInfo::get_compression_level() { return(conf.nc_compression()); } //////////////////////////////////////////////////////////////////////// From 522c58df39f0080967ee872dbf49499c9c92579d Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Tue, 22 Feb 2022 18:12:04 -0700 Subject: [PATCH 02/16] Per #1259, add the eclv_points config option to Ensemble-Stat. --- met/data/config/EnsembleStatConfig_default | 5 ++ .../tools/core/ensemble_stat/ensemble_stat.cc | 47 ++++++++++++++++++- .../ensemble_stat/ensemble_stat_conf_info.cc | 46 +++++++++++++++--- .../ensemble_stat/ensemble_stat_conf_info.h | 23 ++++++--- 4 files changed, 107 insertions(+), 14 deletions(-) diff --git a/met/data/config/EnsembleStatConfig_default b/met/data/config/EnsembleStatConfig_default index d105a032d0..bef7d953c2 100644 --- a/met/data/config/EnsembleStatConfig_default +++ b/met/data/config/EnsembleStatConfig_default @@ -98,6 +98,11 @@ nmep_smooth = { //////////////////////////////////////////////////////////////////////////////// +// +// May be set separately in each "field" entry +// +eclv_points = 0.05; + // // Forecast and observation fields to be verified // diff --git a/met/src/tools/core/ensemble_stat/ensemble_stat.cc b/met/src/tools/core/ensemble_stat/ensemble_stat.cc index b8ee623b14..90deb3828a 100644 --- a/met/src/tools/core/ensemble_stat/ensemble_stat.cc +++ b/met/src/tools/core/ensemble_stat/ensemble_stat.cc @@ -2315,7 +2315,7 @@ void setup_nc_file(unixtime valid_ut, const char *suffix) { //////////////////////////////////////////////////////////////////////// void setup_txt_files() { - int i, n, n_phist_bin, max_n_ens, max_col; + int i, n, n_phist_bin, n_prob, n_eclv, max_n_ens, max_col; ConcatString tmp_str; // Check to see if the text files have already been set up @@ -2330,6 +2330,10 @@ void setup_txt_files() { // ///////////////////////////////////////////////////////////////////// + // Maximum number of probability thresholds + n_prob = conf_info.get_max_n_prob_thresh(); + n_eclv = conf_info.get_max_n_eclv_points(); + // Compute the number of PHIST bins for(i=n_phist_bin=0; i Date: Wed, 23 Feb 2022 10:06:48 -0700 Subject: [PATCH 03/16] Per #1259, this actually fixes a bug in Ensemble-Stat. Consolidated the writing of STAT output lines for both point and gridded verification into a common write_txt_files() function. In the process, I discovered that we were checking the top-level conf_info.output_flag option instead of the vx_opt specific ones. These changes switch to using the vx_opt ones so that we can control the output written task by task. --- .../tools/core/ensemble_stat/ensemble_stat.cc | 205 ++++++++++++++++-- .../tools/core/ensemble_stat/ensemble_stat.h | 6 +- 2 files changed, 184 insertions(+), 27 deletions(-) diff --git a/met/src/tools/core/ensemble_stat/ensemble_stat.cc b/met/src/tools/core/ensemble_stat/ensemble_stat.cc index 90deb3828a..394ec2be4c 100644 --- a/met/src/tools/core/ensemble_stat/ensemble_stat.cc +++ b/met/src/tools/core/ensemble_stat/ensemble_stat.cc @@ -143,9 +143,11 @@ static ConcatString get_ens_mn_var_name(int); static void setup_nc_file (unixtime, const char *); static void setup_txt_files (); static void setup_table (AsciiTable &); - static void build_outfile_name(unixtime, const char *, ConcatString &); + +static void write_txt_files(const EnsembleStatVxOpt &, + const PairDataEnsemble &, bool); static void write_ens_nc(EnsVarInfo *, int, DataPlane &); static void write_ens_var_float(EnsVarInfo *, float *, const DataPlane &, const char *, const char *); @@ -478,8 +480,13 @@ void process_command_line(int argc, char **argv) { conf_info.output_flag[i_phist] != STATOutputType_None || conf_info.output_flag[i_ssvar] != STATOutputType_None || conf_info.output_flag[i_relp] != STATOutputType_None || - conf_info.output_flag[i_orank] != STATOutputType_None)) vx_flag = 1; - else vx_flag = 0; + conf_info.output_flag[i_orank] != STATOutputType_None || + conf_info.output_flag[i_pct] != STATOutputType_None || + conf_info.output_flag[i_pstd] != STATOutputType_None || + conf_info.output_flag[i_pjc] != STATOutputType_None || + conf_info.output_flag[i_prc] != STATOutputType_None || + conf_info.output_flag[i_eclv] != STATOutputType_None)) vx_flag = true; + else vx_flag = false; // Process ensemble mean information ens_mean_flag = false; @@ -1324,7 +1331,7 @@ void process_point_scores() { PairDataEnsemble *pd_ptr = (PairDataEnsemble *) 0; PairDataEnsemble pd; ConcatString cs; - int i, j, k, l, m, n; + int i, j, k, l; mlog << Debug(2) << "\n" << sep_str << "\n\n"; @@ -1417,6 +1424,11 @@ void process_point_scores() { // Compute observation ranks and pair values pd_ptr->compute_pair_vals(conf_info.rng_ptr); + // Write the stat output + write_txt_files(conf_info.vx_opt[i], *pd_ptr, true); + +/* JHG delete me for point scores + // Process each filtering threshold for(m=0; m 0) { + write_rhist_row(shc, &pd, + vx_opt.output_flag[i_rhist], + stat_at, i_stat_row, + txt_at[i_rhist], i_txt_row[i_rhist]); + } + } + + // Write PHIST counts if greater than 0 + if(vx_opt.output_flag[i_phist] != STATOutputType_None) { + + pd.phist_bin_size = vx_opt.vx_pd.pd[0][0][0].phist_bin_size; + pd.compute_phist(); + + if(pd.phist_na.sum() > 0) { + write_phist_row(shc, &pd, + vx_opt.output_flag[i_phist], + stat_at, i_stat_row, + txt_at[i_phist], i_txt_row[i_phist]); + } + } + + // Write RELP counts + if(vx_opt.output_flag[i_relp] != STATOutputType_None) { + + pd.compute_relp(); + + if(pd.relp_na.sum() > 0) { + write_relp_row(shc, &pd, + vx_opt.output_flag[i_relp], + stat_at, i_stat_row, + txt_at[i_relp], i_txt_row[i_relp]); + } + } + + // Write SSVAR scores + if(vx_opt.output_flag[i_ssvar] != STATOutputType_None) { + + pd.ssvar_bin_size = vx_opt.vx_pd.pd[0][0][0].ssvar_bin_size; + pd.compute_ssvar(); + + // Make sure there are bins to process + if(pd.ssvar_bins) { + + // Add rows to the output AsciiTables for SSVAR + stat_at.add_rows(pd.ssvar_bins[0].n_bin * + vx_opt.ci_alpha.n()); + + if(vx_opt.output_flag[i_ssvar] == STATOutputType_Both) { + txt_at[i_ssvar].add_rows(pd.ssvar_bins[0].n_bin * + vx_opt.ci_alpha.n()); + } + + // Write the SSVAR data for each alpha value + for(j=0; j Date: Wed, 23 Feb 2022 10:17:14 -0700 Subject: [PATCH 04/16] Per #1259, ci-run-unit strip out commented-out old code for writing stat output lines. --- .../tools/core/ensemble_stat/ensemble_stat.cc | 193 ------------------ 1 file changed, 193 deletions(-) diff --git a/met/src/tools/core/ensemble_stat/ensemble_stat.cc b/met/src/tools/core/ensemble_stat/ensemble_stat.cc index 394ec2be4c..9ddf970b40 100644 --- a/met/src/tools/core/ensemble_stat/ensemble_stat.cc +++ b/met/src/tools/core/ensemble_stat/ensemble_stat.cc @@ -1427,100 +1427,6 @@ void process_point_scores() { // Write the stat output write_txt_files(conf_info.vx_opt[i], *pd_ptr, true); -/* JHG delete me for point scores - - // Process each filtering threshold - for(m=0; msubset_pairs_obs_thresh(conf_info.vx_opt[i].othr_ta[m]); - - // Continue if there are no points - if(pd.n_obs == 0) continue; - - // Compute ECNT scores - if(conf_info.output_flag[i_ecnt] != STATOutputType_None) { - do_ecnt(conf_info.vx_opt[i], - conf_info.vx_opt[i].othr_ta[m], &pd); - } - - // Compute RPS scores - if(conf_info.output_flag[i_rps] != STATOutputType_None) { - do_rps(conf_info.vx_opt[i], - conf_info.vx_opt[i].othr_ta[m], &pd); - } - - // Write RHIST counts - if(conf_info.output_flag[i_rhist] != STATOutputType_None) { - - pd.compute_rhist(); - - if(pd.rhist_na.sum() > 0) { - write_rhist_row(shc, &pd, - conf_info.output_flag[i_rhist], - stat_at, i_stat_row, - txt_at[i_rhist], i_txt_row[i_rhist]); - } - } - - // Write PHIST counts if greater than 0 - if(conf_info.output_flag[i_phist] != STATOutputType_None) { - - pd.compute_phist(); - - if(pd.phist_na.sum() > 0) { - write_phist_row(shc, &pd, - conf_info.output_flag[i_phist], - stat_at, i_stat_row, - txt_at[i_phist], i_txt_row[i_phist]); - } - } - - // Write RELP counts - if(conf_info.output_flag[i_relp] != STATOutputType_None) { - - pd.compute_relp(); - - if(pd.relp_na.sum() > 0) { - write_relp_row(shc, &pd, - conf_info.output_flag[i_relp], - stat_at, i_stat_row, - txt_at[i_relp], i_txt_row[i_relp]); - } - } - - // Write SSVAR scores - if(conf_info.output_flag[i_ssvar] != STATOutputType_None) { - - pd.compute_ssvar(); - - // Make sure there are bins to process - if(pd.ssvar_bins) { - - // Add rows to the output AsciiTables for SSVAR - stat_at.add_rows(pd.ssvar_bins[0].n_bin * - conf_info.vx_opt[i].ci_alpha.n()); - - if(conf_info.output_flag[i_ssvar] == STATOutputType_Both) { - txt_at[i_ssvar].add_rows(pd.ssvar_bins[0].n_bin * - conf_info.vx_opt[i].ci_alpha.n()); - } - - // Write the SSVAR data for each alpha value - for(n=0; n 0) { - write_rhist_row(shc, &pd, - conf_info.output_flag[i_rhist], - stat_at, i_stat_row, - txt_at[i_rhist], i_txt_row[i_rhist]); - } - } - - // Write PHIST counts if greater than 0 - if(conf_info.output_flag[i_phist] != STATOutputType_None) { - - pd.phist_bin_size = conf_info.vx_opt[i].vx_pd.pd[0][0][0].phist_bin_size; - pd.compute_phist(); - - if(pd.phist_na.sum() > 0) { - write_phist_row(shc, &pd, - conf_info.output_flag[i_phist], - stat_at, i_stat_row, - txt_at[i_phist], i_txt_row[i_phist]); - } - } - - // Write RELP counts - if(conf_info.output_flag[i_relp] != STATOutputType_None) { - - pd.compute_relp(); - - if(pd.relp_na.sum() > 0) { - write_relp_row(shc, &pd, - conf_info.output_flag[i_relp], - stat_at, i_stat_row, - txt_at[i_relp], i_txt_row[i_relp]); - } - } - - // Write SSVAR scores - if(conf_info.output_flag[i_ssvar] != STATOutputType_None) { - - pd.ssvar_bin_size = conf_info.vx_opt[i].vx_pd.pd[0][0][0].ssvar_bin_size; - pd.compute_ssvar(); - - // Make sure there are bins to process - if(pd.ssvar_bins) { - - // Add rows to the output AsciiTables for SSVAR - stat_at.add_rows(pd.ssvar_bins[0].n_bin * - conf_info.vx_opt[i].ci_alpha.n()); - - if(conf_info.output_flag[i_ssvar] == STATOutputType_Both) { - txt_at[i_ssvar].add_rows(pd.ssvar_bins[0].n_bin * - conf_info.vx_opt[i].ci_alpha.n()); - } - - // Write the SSVAR data for each alpha value - for(m=0; m Date: Wed, 23 Feb 2022 11:44:35 -0700 Subject: [PATCH 05/16] Per #1259, no real change to point_stat. Just removing a blank line. --- met/src/tools/core/point_stat/point_stat.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/met/src/tools/core/point_stat/point_stat.cc b/met/src/tools/core/point_stat/point_stat.cc index 73dfa2af24..5f40c314be 100644 --- a/met/src/tools/core/point_stat/point_stat.cc +++ b/met/src/tools/core/point_stat/point_stat.cc @@ -158,7 +158,6 @@ static void do_pct (const PointStatVxOpt &, const PairDataPoint *); static void do_hira_ens ( int, const PairDataPoint *); static void do_hira_prob ( int, const PairDataPoint *); - static void finish_txt_files(); static void clean_up(); From eebe6499d394b793b0dfade0fcfb3649b7d5a540 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Wed, 23 Feb 2022 15:51:46 -0700 Subject: [PATCH 06/16] Per #1259, making progress. Added prob_pct_thresh option to ensemble-stat to define how PCT thresholds. Still need to update documentation, tests, and config files. --- met/data/config/EnsembleStatConfig_default | 18 +- met/scripts/config/EnsembleStatConfig | 16 ++ met/src/basic/vx_config/config_constants.h | 1 + .../tools/core/ensemble_stat/ensemble_stat.cc | 215 +++++++++++++++--- .../ensemble_stat/ensemble_stat_conf_info.cc | 53 ++--- .../ensemble_stat/ensemble_stat_conf_info.h | 29 ++- 6 files changed, 240 insertions(+), 92 deletions(-) diff --git a/met/data/config/EnsembleStatConfig_default b/met/data/config/EnsembleStatConfig_default index bef7d953c2..90fd944268 100644 --- a/met/data/config/EnsembleStatConfig_default +++ b/met/data/config/EnsembleStatConfig_default @@ -39,7 +39,7 @@ regrid = { //////////////////////////////////////////////////////////////////////////////// // -// May be set separately in each "field" entry +// May be set separately in each "ens.field" entry // censor_thresh = []; censor_val = []; @@ -99,7 +99,13 @@ nmep_smooth = { //////////////////////////////////////////////////////////////////////////////// // -// May be set separately in each "field" entry +// May be set separately in each "fcst.field" entry +// +prob_cat_thresh = []; +prob_pct_thresh = [ ==0.25 ]; + +// +// May be set separately in each "obs.field" entry // eclv_points = 0.05; @@ -165,12 +171,6 @@ message_type_group_map = [ ens_ssvar_bin_size = 1.0; ens_phist_bin_size = 0.05; -// -// Categorical thresholds to define ensemble probabilities -// May be set separately in each "fcst.field" entry -// -prob_cat_thresh = []; - //////////////////////////////////////////////////////////////////////////////// // @@ -204,6 +204,8 @@ climo_stdev = { climo_cdf = { cdf_bins = 10; center_bins = FALSE; + write_bins = TRUE; + direct_prob = FALSE; } //////////////////////////////////////////////////////////////////////////////// diff --git a/met/scripts/config/EnsembleStatConfig b/met/scripts/config/EnsembleStatConfig index 64367110ea..e2bc7d71b3 100644 --- a/met/scripts/config/EnsembleStatConfig +++ b/met/scripts/config/EnsembleStatConfig @@ -109,6 +109,17 @@ nmep_smooth = { //////////////////////////////////////////////////////////////////////////////// +// +// May be set separately in each "fcst.field" entry +// +prob_cat_thresh = []; +prob_pct_thresh = [ ==0.25 ]; + +// +// May be set separately in each "obs.field" entry +// +eclv_points = 0.05; + // // Forecast and observation fields to be verified // @@ -270,6 +281,11 @@ output_flag = { orank = BOTH; ssvar = BOTH; relp = BOTH; + pct = BOTH; + pstd = BOTH; + pjc = BOTH; + prc = BOTH; + eclv = BOTH; } //////////////////////////////////////////////////////////////////////////////// diff --git a/met/src/basic/vx_config/config_constants.h b/met/src/basic/vx_config/config_constants.h index f3c5550f02..7a87358da2 100644 --- a/met/src/basic/vx_config/config_constants.h +++ b/met/src/basic/vx_config/config_constants.h @@ -746,6 +746,7 @@ static const char conf_key_rank_flag[] = "rank"; static const char conf_key_ssvar_bin[] = "ens_ssvar_bin_size"; static const char conf_key_phist_bin[] = "ens_phist_bin_size"; static const char conf_key_prob_cat_thresh[] = "prob_cat_thresh"; +static const char conf_key_prob_pct_thresh[] = "prob_pct_thresh"; static const char conf_key_obs_error[] = "obs_error"; static const char conf_key_dist_type[] = "dist_type"; static const char conf_key_dist_parm[] = "dist_parm"; diff --git a/met/src/tools/core/ensemble_stat/ensemble_stat.cc b/met/src/tools/core/ensemble_stat/ensemble_stat.cc index 9ddf970b40..7442017699 100644 --- a/met/src/tools/core/ensemble_stat/ensemble_stat.cc +++ b/met/src/tools/core/ensemble_stat/ensemble_stat.cc @@ -148,6 +148,8 @@ static void build_outfile_name(unixtime, const char *, static void write_txt_files(const EnsembleStatVxOpt &, const PairDataEnsemble &, bool); +static void do_pct(const EnsembleStatVxOpt &, const PairDataEnsemble &); + static void write_ens_nc(EnsVarInfo *, int, DataPlane &); static void write_ens_var_float(EnsVarInfo *, float *, const DataPlane &, const char *, const char *); @@ -1936,7 +1938,7 @@ void do_rps(const EnsembleStatVxOpt &vx_opt, // Store observation filering threshold rps_info.othresh = othresh; - rps_info.set_prob_cat_thresh(vx_opt.pcat_ta); + rps_info.set_prob_cat_thresh(vx_opt.prob_cat_ta); // If prob_cat_thresh is empty and climo data is available, // use climo_cdf thresholds instead @@ -2140,7 +2142,7 @@ void setup_txt_files() { ///////////////////////////////////////////////////////////////////// // Maximum number of probability thresholds - n_prob = conf_info.get_max_n_prob_thresh(); + n_prob = conf_info.get_max_n_prob_pct_thresh(); n_eclv = conf_info.get_max_n_eclv_points(); // Compute the number of PHIST bins @@ -2281,15 +2283,15 @@ void setup_txt_files() { break; case(i_pstd): - write_pct_header_row(1, n_prob, txt_at[i], 0, 0); + write_pstd_header_row(1, n_prob, txt_at[i], 0, 0); break; case(i_pjc): - write_pct_header_row(1, n_prob, txt_at[i], 0, 0); + write_pjc_header_row(1, n_prob, txt_at[i], 0, 0); break; case(i_prc): - write_pct_header_row(1, n_prob, txt_at[i], 0, 0); + write_prc_header_row(1, n_prob, txt_at[i], 0, 0); break; case(i_eclv): @@ -2464,37 +2466,14 @@ void write_txt_files(const EnsembleStatVxOpt &vx_opt, } // end for i - // Derive probabilities for each threshold - for(i=0; i 0 && pd_pnt.csd_na.n_valid() > 0 ? + vx_opt.get_n_cdf_bin() : 1); + + if(n_bin > 1) { + mlog << Debug(2) + << "Applying " << n_bin << " climatology bins.\n"; + } + + // Allocate memory + pct_info = new PCTInfo [n_bin]; + + // Store the current fcst_var value + fcst_var_cs = shc.get_fcst_var(); + + // Process each probability threshold + for(i=0; i 0 || (double) (n_vld/pd_ens.n_ens) >= conf_info.vld_data_thresh) { + pd_pnt.add_grid_pair((double) n_thr / n_vld, pd_ens.o_na[i_obs], + pd_ens.cmn_na[i_obs], pd_ens.csd_na[i_obs], + pd_ens.wgt_na[i_obs]); + } + + } // end for i_obs + + // Process the climo CDF bins + for(j=0; j 1) pd = subset_climo_cdf_bin(pd_pnt, + vx_opt.cdf_info.cdf_ta, j); + else pd = pd_pnt; + + // Store thresholds + pct_info[j].fthresh = vx_opt.prob_pct_ta; + pct_info[j].othresh = vx_opt.prob_cat_ta[i]; + pct_info[j].allocate_n_alpha(vx_opt.get_n_ci_alpha()); + + for(k=0; k 1) { + + PCTInfo pct_mean; + compute_pct_mean(pct_info, n_bin, pct_mean); + + // Write out PSTD + if(vx_opt.output_flag[i_pstd] != STATOutputType_None) { + write_pstd_row(shc, pct_mean, + vx_opt.output_flag[i_pstd], + -1, n_bin, stat_at, i_stat_row, + txt_at[i_pstd], i_txt_row[i_pstd]); + } + } // end if n_bin > 1 + + } // end for i (ocnt_ta) + + // Reset the forecast variable name + shc.set_fcst_var(fcst_var_cs); + + // Dealloate memory + if(pct_info) { delete [] pct_info; pct_info = (PCTInfo *) 0; } + + return; +} + +//////////////////////////////////////////////////////////////////////// + void write_ens_nc(EnsVarInfo * ens_info, int i_var, DataPlane &ens_dp) { int i, j, k, l; double t, v; diff --git a/met/src/tools/core/ensemble_stat/ensemble_stat_conf_info.cc b/met/src/tools/core/ensemble_stat/ensemble_stat_conf_info.cc index d54821f609..87d2dab82a 100644 --- a/met/src/tools/core/ensemble_stat/ensemble_stat_conf_info.cc +++ b/met/src/tools/core/ensemble_stat/ensemble_stat_conf_info.cc @@ -697,20 +697,20 @@ int EnsembleStatConfInfo::n_stat_row() const { //////////////////////////////////////////////////////////////////////// -int EnsembleStatConfInfo::get_max_n_cat_thresh() const { +int EnsembleStatConfInfo::get_max_n_prob_cat_thresh() const { int i, n; - for(i=0,n=0; i " - << "The number of thresholds for each field in \"fcst." - << conf_key_cat_thresh - << "\" must match the number of thresholds for each " - << "field in \"obs." << conf_key_cat_thresh << "\".\n\n"; - exit(1); - } - // Conf: prob_cat_thresh - pcat_ta = fdict.lookup_thresh_array(conf_key_prob_cat_thresh); + prob_cat_ta = fdict.lookup_thresh_array(conf_key_prob_cat_thresh); + + // Conf: prob_pct_thresh + prob_pct_ta = fdict.lookup_thresh_array(conf_key_prob_pct_thresh); + prob_pct_ta = string_to_prob_thresh(prob_pct_ta.get_str().c_str()); // Conf: duplicate_flag duplicate_flag = parse_conf_duplicate_flag(&odict); @@ -1164,9 +1151,7 @@ void EnsembleStatVxOpt::set_perc_thresh(const PairDataEnsemble *pd_ptr) { // // Check if percentile thresholds were requested // - if(!othr_ta.need_perc() && - !fcat_ta.need_perc() && - !ocat_ta.need_perc()) return; + if(!othr_ta.need_perc() && !prob_cat_ta.need_perc()) return; // // Sort the input arrays @@ -1183,9 +1168,10 @@ void EnsembleStatVxOpt::set_perc_thresh(const PairDataEnsemble *pd_ptr) { // Compute percentiles, passing the observation thresholds in for // the fcst and obs slots. // - othr_ta.set_perc(&fsort, &osort, &csort, &othr_ta, &othr_ta); - fcat_ta.set_perc(&fsort, &osort, &csort, &fcat_ta, &ocat_ta); - ocat_ta.set_perc(&fsort, &osort, &csort, &fcat_ta, &ocat_ta); + othr_ta.set_perc(&fsort, &osort, &csort, + &othr_ta, &othr_ta); + prob_cat_ta.set_perc(&fsort, &osort, &csort, + &prob_cat_ta, &prob_cat_ta); return; } @@ -1254,7 +1240,7 @@ int EnsembleStatVxOpt::n_txt_row(int i_txt_row) const { // Point Vx: Message Types * Masks * Interpolations * Categorical Thresholds // Grid Vx: Masks * Interpolations * Categorical Thresholds n = (get_n_msg_typ() + 1) * get_n_mask() * get_n_interp() * - get_n_cat_thresh(); + get_n_prob_cat_thresh(); break; case(i_pstd): @@ -1263,7 +1249,7 @@ int EnsembleStatVxOpt::n_txt_row(int i_txt_row) const { // Point Vx: Message Types * Masks * Interpolations * Categorical Thresholds * Alphas // Grid Vx: Masks * Interpolations * Categorical Thresholds * Alphas n = (get_n_msg_typ() + 1) * get_n_mask() * get_n_interp() * - get_n_cat_thresh() * get_n_ci_alpha(); + get_n_prob_cat_thresh() * get_n_ci_alpha(); break; case(i_eclv): @@ -1272,7 +1258,8 @@ int EnsembleStatVxOpt::n_txt_row(int i_txt_row) const { // Point Vx: Message Types * Masks * Interpolations * Probability Thresholds // Grid Vx: Masks * Interpolations * Probability Thresholds n = (get_n_msg_typ() + 1) * get_n_mask() * get_n_interp() * - get_n_prob_thresh() * get_n_prob_thresh(); + get_n_prob_cat_thresh() * get_n_prob_cat_thresh(); + break; default: mlog << Error << "\nEnsembleStatVxOpt::n_txt_row(int) -> " diff --git a/met/src/tools/core/ensemble_stat/ensemble_stat_conf_info.h b/met/src/tools/core/ensemble_stat/ensemble_stat_conf_info.h index 5e96e14034..846176d95b 100644 --- a/met/src/tools/core/ensemble_stat/ensemble_stat_conf_info.h +++ b/met/src/tools/core/ensemble_stat/ensemble_stat_conf_info.h @@ -131,9 +131,8 @@ class EnsembleStatVxOpt { double ssvar_bin_size; // SSVAR bin size double phist_bin_size; // PHIST bin size - ThreshArray fcat_ta; // Forecast categorical thresholds to derive probabilities - ThreshArray ocat_ta; // Observation categorical thresholds for verifying events - ThreshArray pcat_ta; // Categorical thresholds to evaluate derived probabilities + ThreshArray prob_cat_ta; // Categorical thresholds for probabilities, including RPS + ThreshArray prob_pct_ta; // PCT thresholds DuplicateType duplicate_flag; // Duplicate observations ObsSummary obs_summary; // Summarize observations @@ -165,11 +164,11 @@ class EnsembleStatVxOpt { int get_n_mask() const; int get_n_mask_area() const; - int get_n_obs_thresh() const; - int get_n_cat_thresh() const; - int get_n_prob_thresh() const; + int get_n_obs_thresh() const; + int get_n_prob_cat_thresh() const; + int get_n_prob_pct_thresh() const; - int get_n_eclv_points() const; + int get_n_eclv_points() const; int get_n_cdf_bin() const; int get_n_ci_alpha() const; }; @@ -181,9 +180,9 @@ inline int EnsembleStatVxOpt::get_n_interp() const { return(interp_info.n_i inline int EnsembleStatVxOpt::get_n_mask() const { return(mask_name.n()); } inline int EnsembleStatVxOpt::get_n_mask_area() const { return(mask_name_area.n()); } -inline int EnsembleStatVxOpt::get_n_obs_thresh() const { return(othr_ta.n()); } -inline int EnsembleStatVxOpt::get_n_cat_thresh() const { return(ocat_ta.n()); } -inline int EnsembleStatVxOpt::get_n_prob_thresh() const { return(pcat_ta.n()); } +inline int EnsembleStatVxOpt::get_n_obs_thresh() const { return(othr_ta.n()); } +inline int EnsembleStatVxOpt::get_n_prob_cat_thresh() const { return(prob_cat_ta.n()); } +inline int EnsembleStatVxOpt::get_n_prob_pct_thresh() const { return(prob_pct_ta.n()); } inline int EnsembleStatVxOpt::get_n_eclv_points() const { return(eclv_points.n()); } inline int EnsembleStatVxOpt::get_n_cdf_bin() const { return(cdf_info.n_bin); } @@ -276,11 +275,11 @@ class EnsembleStatConfInfo { int n_stat_row() const; // Maximum across all verification tasks - int get_max_n_ens_thresh() const; - int get_max_hira_size() const; - int get_max_n_cat_thresh() const; - int get_max_n_prob_thresh() const; - int get_max_n_eclv_points() const; + int get_max_n_ens_thresh() const; + int get_max_hira_size() const; + int get_max_n_prob_cat_thresh() const; + int get_max_n_prob_pct_thresh() const; + int get_max_n_eclv_points() const; int get_compression_level(); }; From 17d81fcd470cc36acf1105fb4a2156acbf06bc5d Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Thu, 24 Feb 2022 16:04:47 -0700 Subject: [PATCH 07/16] Per #1259, update the Ensemble-Stat chapter of the user's guide. --- met/docs/Users_Guide/ensemble-stat.rst | 69 ++++++++++++++++++-------- 1 file changed, 48 insertions(+), 21 deletions(-) diff --git a/met/docs/Users_Guide/ensemble-stat.rst b/met/docs/Users_Guide/ensemble-stat.rst index f063e23f52..2047ca2a0c 100644 --- a/met/docs/Users_Guide/ensemble-stat.rst +++ b/met/docs/Users_Guide/ensemble-stat.rst @@ -19,7 +19,7 @@ Ensemble forecasts are often created as a set of deterministic forecasts. The en Typically an ensemble is constructed by selecting a single forecast value from each member for each observation. When the High Resolution Assessment (HiRA) interpolation method is chosen, all of the nearby neighborhood points surrounding each observation from each member are used. Therefore, processing an N-member ensemble using a HiRA neighborhood of size M produces ensemble output with size N*M. This approach fully leverages information from all nearby grid points to evaluate the ensemble quality. -The ensemble relative frequency is the simplest method for turning a set of deterministic forecasts into something resembling a probability forecast. MET will create the ensemble relative frequency as the proportion of ensemble members forecasting some event. For example, if 5 out of 10 ensemble members predict measurable precipitation at a grid location, then the ensemble relative frequency of precipitation will be :math:`5/10=0.5`. If the ensemble relative frequency is calibrated (unlikely) then this could be thought of as a probability of precipitation. +The ensemble relative frequency is the simplest method for turning a set of deterministic forecasts into something resembling a probability forecast. For each categorical threshold (cat_thresh) listed for each field array entry of the ensemble dictionary (ens.field), MET will create the ensemble relative frequency as the proportion of ensemble members forecasting that event. For example, if 5 out of 10 ensemble members predict measurable precipitation at a grid location, then the ensemble relative frequency of precipitation will be :math:`5/10=0.5`. If the ensemble relative frequency is calibrated (unlikely) then this could be thought of as a probability of precipitation. The neighborhood ensemble probability (NEP) and neighborhood maximum ensemble probability (NMEP) methods are described in :ref:`Schwartz and Sobash (2017) `. They are an extension of the ensemble relative frequencies described above. The NEP value is computed by averaging the relative frequency of the event within the neighborhood over all ensemble members. The NMEP value is computed as the fraction of ensemble members for which the event is occurring somewhere within the surrounding neighborhood. The NMEP output is typically smoothed using a Gaussian kernel filter. The neighborhood sizes and smoothing options can be customized in the configuration file. @@ -36,6 +36,8 @@ The relative position (RELP) is a count of the number of times each ensemble mem The ranked probability score (RPS) is included in the Ranked Probability Score (RPS) line type. It is the mean of the Brier scores computed from ensemble probabilities derived for each probability category threshold (prob_cat_thresh) specified in the configuration file. The continuous ranked probability score (CRPS) is the average the distance between the forecast (ensemble) cumulative distribution function and the observation cumulative distribution function. It is an analog of the Brier score, but for continuous forecast and observation fields. The CRPS statistic is computed using two methods: assuming a normal distribution defined by the ensemble mean and spread (:ref:`Gneiting et al., 2004 `) and using the empirical ensemble distribution (:ref:`Hersbach, 2000 `). The CRPS statistic is included in the Ensemble Continuous Statistics (ECNT) line type, along with other statistics quantifying the ensemble spread and ensemble mean skill. +The Ensemble-Stat tool can derive ensemble relative frequencies and verify them as probability forecasts all in the same run. Note however that these simple ensemble relative frequencies are not actually calibrated probability forecasts. If probabilistic line types are requested (output_flag), this logic is applied to each pair of fields listed in the forecast (fcst) and observation (obs) dictionaries of the configuration file. Each probability category threshold (prob_cat_thresh) listed for the forecast field is applied to the input ensemble members to derive a relative frequency forecast. That same probability category threshold is also applied to the (gridded or point) observations to determine whether or not the event actually occurred. The paired ensemble relative freqencies and observation events are used to populate an Nx2 probabilistic contingency table. The dimension of that table is determined by the probability PCT threshold (prob_pct_thresh) configuration file option. All probabilistic output types requested are derived from the this Nx2 table and written to the ascii output files. Note that the FCST_VAR name header column is automatically reset as "PROB({FCST_VAR}{THRESH})" where {FCST_VAR} is the current field being evaluated and {THRESH} is the threshold that was applied. + Climatology data ---------------- @@ -150,6 +152,7 @@ ____________________ ci_alpha = [ 0.05 ]; interp = { field = BOTH; vld_thresh = 1.0; shape = SQUARE; type = [ { method = NEAREST; width = 1; } ]; } + eclv_points = []; sid_inc = []; sid_exc = []; duplicate_flag = NONE; @@ -301,33 +304,42 @@ ____________________ ens_ssvar_bin_size = 1.0; ens_phist_bin_size = 0.05; - prob_cat_thresh = []; Setting up the **fcst** and **obs** dictionaries of the configuration file is described in :numref:`config_options`. The following are some special considerations for the Ensemble-Stat tool. - The **ens** and **fcst** dictionaries do not need to include the same fields. Users may specify any number of ensemble fields to be summarized, but generally there are many fewer fields with verifying observations available. The **ens** dictionary specifies the fields to be summarized while the **fcst** dictionary specifies the fields to be verified. - The **obs** dictionary looks very similar to the **fcst** dictionary. If verifying against point observations which are assigned GRIB1 codes, the observation section must be defined following GRIB1 conventions. When verifying GRIB1 forecast data, one can easily copy over the forecast settings to the observation dictionary using **obs = fcst;**. However, when verifying non-GRIB1 forecast data, users will need to specify the **fcst** and **obs** sections separately. - The **ens_ssvar_bin_size** and **ens_phist_bin_size** specify the width of the categorical bins used to accumulate frequencies for spread-skill-variance or probability integral transform statistics, respectively. +____________________ + +.. code-block:: none + + prob_cat_thresh = []; + prob_pct_thresh = []; + + +The **prob_cat_thresh** entry is an array of thresholds. It is applied both to the computation of the RPS line type as well as the when generating probabilistic output line types. Since these thresholds can change for each variable, they can be specified separately for each **fcst.field** entry. If left empty but climatological mean and standard deviation data is provided, the **climo_cdf** thresholds will be used instead. If no climatology data is provided, and the RPS output line type is requested, then the **prob_cat_thresh** array must be defined. When probabilistic output line types are requested, for each **prob_cat_thresh** threshold listed, ensemble relative frequencies are derived and verified against the point and/or gridded observations. -The **prob_cat_thresh** entry is an array of thresholds to be applied in the computation of the RPS line type. Since these thresholds can change for each variable, they can be specified separately for each **fcst.field** entry. If left empty but climatological mean and standard deviation data is provided, the **climo_cdf** thresholds will be used instead. If no climatology data is provided, and the RPS output line type is requested, then the **prob_cat_thresh** array must be defined. +The **prob_pct_thresh** entry is an array of thresholds which define the Nx2 probabilistic contingency table used to evaluate probability forecasts. It can be specified separately for each **fcst.field** entry. These thresholds must span the range [0, 1]. A shorthand notation to create equal bin widths is provided. For example, the following setting creates 4 probability bins of width 0.25 from 0 to 1. + +.. code-block:: none + + prob_cat_thresh = [ ==0.25 ]; __________________ .. code-block:: none obs_error = { - flag = FALSE; - dist_type = NONE; - dist_parm = []; - inst_bias_scale = 1.0; - inst_bias_offset = 0.0; + flag = FALSE; + dist_type = NONE; + dist_parm = []; + inst_bias_scale = 1.0; + inst_bias_offset = 0.0; } @@ -353,13 +365,18 @@ _________________ .. code-block:: none output_flag = { - ecnt = NONE; - rps = NONE; - rhist = NONE; - phist = NONE; - orank = NONE; - ssvar = NONE; - relp = NONE; + ecnt = NONE; + rps = NONE; + rhist = NONE; + phist = NONE; + orank = NONE; + ssvar = NONE; + relp = NONE; + pct = NONE; + pstd = NONE; + pjc = NONE; + prc = NONE; + eclv = NONE; } @@ -380,12 +397,22 @@ The **output_flag** array controls the type of output that is generated. Each fl 7. **RELP** for Relative Position Counts +8. **PCT** for Contingency Table counts for derived ensemble relative frequencies + +9. **PSTD** for Probabilistic statistics for dichotomous outcomes for derived ensemble relative frequencies + +10. **PJC** for Joint and Conditional factorization for derived ensemble relative frequencies + +11. **PRC** for Receiver Operating Characteristic for derived ensemble relative frequencies + +12. **ECLV** for Economic Cost/Loss Relative Value for derived ensemble relative frequencies + _____________________ .. code-block:: none - ensemble_flag = { - latlon = TRUE; + ensemble_flag = { + latlon = TRUE; mean = TRUE; stdev = TRUE; minus = TRUE; @@ -399,7 +426,7 @@ _____________________ nmep = FALSE; rank = TRUE; weight = FALSE; - } + } The **ensemble_flag** specifies which derived ensemble fields should be calculated and output. Setting the flag to TRUE produces output of the specified field, while FALSE produces no output for that field type. The flags correspond to the following output line types: From 422efa595290fe55de81c5fb31674b61f38a56d1 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Fri, 25 Feb 2022 17:49:36 -0700 Subject: [PATCH 08/16] Per #1259, move the derivation of cdp thresholds into the thresh_array.cc. --- met/src/basic/vx_util/thresh_array.cc | 21 +++++++++++++++++++++ met/src/basic/vx_util/thresh_array.h | 1 + met/src/libcode/vx_statistics/ens_stats.cc | 17 +---------------- 3 files changed, 23 insertions(+), 16 deletions(-) diff --git a/met/src/basic/vx_util/thresh_array.cc b/met/src/basic/vx_util/thresh_array.cc index 1575ef1f04..dd68d0578c 100644 --- a/met/src/basic/vx_util/thresh_array.cc +++ b/met/src/basic/vx_util/thresh_array.cc @@ -771,6 +771,27 @@ ThreshArray process_rps_cdp_thresh(const ThreshArray &ta) { //////////////////////////////////////////////////////////////////////// +ThreshArray derive_cdp_thresh(const ThreshArray &ta) { + SingleThresh st; + ThreshArray ta_out; + + for(int i=0; i Date: Fri, 25 Feb 2022 17:56:45 -0700 Subject: [PATCH 09/16] Per #1259, ci-run-unit add logic to ensemble-stat to compute probabilistic outputs in 2 ways. --- met/data/config/EnsembleStatConfig_default | 6 +- met/docs/Users_Guide/ensemble-stat.rst | 4 +- .../tools/core/ensemble_stat/ensemble_stat.cc | 300 +++++++++++++----- .../ensemble_stat/ensemble_stat_conf_info.cc | 37 ++- .../ensemble_stat/ensemble_stat_conf_info.h | 9 +- 5 files changed, 252 insertions(+), 104 deletions(-) diff --git a/met/data/config/EnsembleStatConfig_default b/met/data/config/EnsembleStatConfig_default index 90fd944268..0413cae12e 100644 --- a/met/data/config/EnsembleStatConfig_default +++ b/met/data/config/EnsembleStatConfig_default @@ -99,9 +99,13 @@ nmep_smooth = { //////////////////////////////////////////////////////////////////////////////// // -// May be set separately in each "fcst.field" entry +// May be set separately in each "fcst.field" and "obs.field" entry // prob_cat_thresh = []; + +// +// May be set separately in each "fcst.field" entry +// prob_pct_thresh = [ ==0.25 ]; // diff --git a/met/docs/Users_Guide/ensemble-stat.rst b/met/docs/Users_Guide/ensemble-stat.rst index 2047ca2a0c..3ca0deb588 100644 --- a/met/docs/Users_Guide/ensemble-stat.rst +++ b/met/docs/Users_Guide/ensemble-stat.rst @@ -36,7 +36,9 @@ The relative position (RELP) is a count of the number of times each ensemble mem The ranked probability score (RPS) is included in the Ranked Probability Score (RPS) line type. It is the mean of the Brier scores computed from ensemble probabilities derived for each probability category threshold (prob_cat_thresh) specified in the configuration file. The continuous ranked probability score (CRPS) is the average the distance between the forecast (ensemble) cumulative distribution function and the observation cumulative distribution function. It is an analog of the Brier score, but for continuous forecast and observation fields. The CRPS statistic is computed using two methods: assuming a normal distribution defined by the ensemble mean and spread (:ref:`Gneiting et al., 2004 `) and using the empirical ensemble distribution (:ref:`Hersbach, 2000 `). The CRPS statistic is included in the Ensemble Continuous Statistics (ECNT) line type, along with other statistics quantifying the ensemble spread and ensemble mean skill. -The Ensemble-Stat tool can derive ensemble relative frequencies and verify them as probability forecasts all in the same run. Note however that these simple ensemble relative frequencies are not actually calibrated probability forecasts. If probabilistic line types are requested (output_flag), this logic is applied to each pair of fields listed in the forecast (fcst) and observation (obs) dictionaries of the configuration file. Each probability category threshold (prob_cat_thresh) listed for the forecast field is applied to the input ensemble members to derive a relative frequency forecast. That same probability category threshold is also applied to the (gridded or point) observations to determine whether or not the event actually occurred. The paired ensemble relative freqencies and observation events are used to populate an Nx2 probabilistic contingency table. The dimension of that table is determined by the probability PCT threshold (prob_pct_thresh) configuration file option. All probabilistic output types requested are derived from the this Nx2 table and written to the ascii output files. Note that the FCST_VAR name header column is automatically reset as "PROB({FCST_VAR}{THRESH})" where {FCST_VAR} is the current field being evaluated and {THRESH} is the threshold that was applied. +The Ensemble-Stat tool can derive ensemble relative frequencies and verify them as probability forecasts all in the same run. Note however that these simple ensemble relative frequencies are not actually calibrated probability forecasts. If probabilistic line types are requested (output_flag), this logic is applied to each pair of fields listed in the forecast (fcst) and observation (obs) dictionaries of the configuration file. Each probability category threshold (prob_cat_thresh) listed for the forecast field is applied to the input ensemble members to derive a relative frequency forecast. The probability category threshold (prob_cat_thresh) parsed from the corresponding observation entry is applied to the (gridded or point) observations to determine whether or not the event actually occurred. The paired ensemble relative freqencies and observation events are used to populate an Nx2 probabilistic contingency table. The dimension of that table is determined by the probability PCT threshold (prob_pct_thresh) configuration file option parsed from the forecast dictionary. All probabilistic output types requested are derived from the this Nx2 table and written to the ascii output files. Note that the FCST_VAR name header column is automatically reset as "PROB({FCST_VAR}{THRESH})" where {FCST_VAR} is the current field being evaluated and {THRESH} is the threshold that was applied. + +Note that if no probability category thresholds (prob_cat_thresh) are defined, but climatological mean and standard deviation data is provided along with climatological bins, climatological distribution percentile thresholds are automatically derived and used to compute probabilistic outputs. Climatology data ---------------- diff --git a/met/src/tools/core/ensemble_stat/ensemble_stat.cc b/met/src/tools/core/ensemble_stat/ensemble_stat.cc index 7442017699..2174ea7796 100644 --- a/met/src/tools/core/ensemble_stat/ensemble_stat.cc +++ b/met/src/tools/core/ensemble_stat/ensemble_stat.cc @@ -143,12 +143,15 @@ static ConcatString get_ens_mn_var_name(int); static void setup_nc_file (unixtime, const char *); static void setup_txt_files (); static void setup_table (AsciiTable &); -static void build_outfile_name(unixtime, const char *, - ConcatString &); +static void build_outfile_name(unixtime, const char *, ConcatString &); static void write_txt_files(const EnsembleStatVxOpt &, const PairDataEnsemble &, bool); + static void do_pct(const EnsembleStatVxOpt &, const PairDataEnsemble &); +static void do_pct_cat_thresh(const EnsembleStatVxOpt &, const PairDataEnsemble &); +static void do_pct_cdp_thresh(const EnsembleStatVxOpt &, const PairDataEnsemble &); +static void write_pct_info(const EnsembleStatVxOpt &, const PCTInfo *, int); static void write_ens_nc(EnsVarInfo *, int, DataPlane &); static void write_ens_var_float(EnsVarInfo *, float *, const DataPlane &, @@ -1938,7 +1941,7 @@ void do_rps(const EnsembleStatVxOpt &vx_opt, // Store observation filering threshold rps_info.othresh = othresh; - rps_info.set_prob_cat_thresh(vx_opt.prob_cat_ta); + rps_info.set_prob_cat_thresh(vx_opt.fcat_ta); // If prob_cat_thresh is empty and climo data is available, // use climo_cdf thresholds instead @@ -2497,26 +2500,55 @@ void write_txt_files(const EnsembleStatVxOpt &vx_opt, //////////////////////////////////////////////////////////////////////// -void do_pct(const EnsembleStatVxOpt &vx_opt, const PairDataEnsemble &pd_ens) { - int i, j, k; - int i_obs, i_ens, n_vld, n_thr, n_bin; +void do_pct(const EnsembleStatVxOpt &vx_opt, + const PairDataEnsemble &pd_ens) { + + // Flag to indicate the presence of valid climo data + bool have_climo = (pd_ens.cmn_na.n_valid() > 0 && + pd_ens.csd_na.n_valid() > 0); + + // If forecast probability thresholds were specified, use them. + if(vx_opt.fcat_ta.n() > 0) { + mlog << Debug(2) + << "Computing Probabilistic Statistics for " + << vx_opt.fcat_ta.n() + << " categorical thresholds.\n"; + do_pct_cat_thresh(vx_opt, pd_ens); + } + // Otherwise, if climo data is available and bins were requested, + // use climo_cdf thresholds instead. + else if(have_climo && vx_opt.cdf_info.cdf_ta.n() > 0) { + mlog << Debug(2) + << "Computing Probabilistic Statistics for " + << vx_opt.cdf_info.cdf_ta.n() + << " climatological distribution thresholds.\n"; + do_pct_cdp_thresh(vx_opt, pd_ens); + } + + // Otherwise, no work to be done. + return; +} + +//////////////////////////////////////////////////////////////////////// + +void do_pct_cat_thresh(const EnsembleStatVxOpt &vx_opt, + const PairDataEnsemble &pd_ens) { + int i, i_thr, i_bin, i_obs, i_ens; + int n_vld, n_evt, n_bin; PCTInfo *pct_info = (PCTInfo *) 0; PairDataPoint pd_pnt, pd; ConcatString fcst_var_cs, cs; - mlog << Debug(2) - << "Computing Probabilistic Statistics.\n"; - // Derive a PairDataPoint object from the PairDataEnsemble input pd_pnt.extend(pd_ens.n_obs); // Determine the number of climo CDF bins - n_bin = (pd_pnt.cmn_na.n_valid() > 0 && pd_pnt.csd_na.n_valid() > 0 ? + n_bin = (pd_ens.cmn_na.n_valid() > 0 && pd_ens.csd_na.n_valid() > 0 ? vx_opt.get_n_cdf_bin() : 1); if(n_bin > 1) { mlog << Debug(2) - << "Applying " << n_bin << " climatology bins.\n"; + << "Subsetting pairs into " << n_bin << " climatology bins.\n"; } // Allocate memory @@ -2526,14 +2558,14 @@ void do_pct(const EnsembleStatVxOpt &vx_opt, const PairDataEnsemble &pd_ens) { fcst_var_cs = shc.get_fcst_var(); // Process each probability threshold - for(i=0; i 0 || (double) (n_vld/pd_ens.n_ens) >= conf_info.vld_data_thresh) { - pd_pnt.add_grid_pair((double) n_thr / n_vld, pd_ens.o_na[i_obs], + pd_pnt.add_grid_pair((double) n_evt/n_vld, pd_ens.o_na[i_obs], pd_ens.cmn_na[i_obs], pd_ens.csd_na[i_obs], pd_ens.wgt_na[i_obs]); } @@ -2562,96 +2594,119 @@ void do_pct(const EnsembleStatVxOpt &vx_opt, const PairDataEnsemble &pd_ens) { } // end for i_obs // Process the climo CDF bins - for(j=0; j 1) pd = subset_climo_cdf_bin(pd_pnt, - vx_opt.cdf_info.cdf_ta, j); + vx_opt.cdf_info.cdf_ta, i_bin); else pd = pd_pnt; // Store thresholds - pct_info[j].fthresh = vx_opt.prob_pct_ta; - pct_info[j].othresh = vx_opt.prob_cat_ta[i]; - pct_info[j].allocate_n_alpha(vx_opt.get_n_ci_alpha()); + pct_info[i_bin].fthresh = vx_opt.fpct_ta; + pct_info[i_bin].othresh = vx_opt.ocat_ta[i_thr]; + pct_info[i_bin].allocate_n_alpha(vx_opt.get_n_ci_alpha()); - for(k=0; k 1) { + // Dealloate memory + if(pct_info) { delete [] pct_info; pct_info = (PCTInfo *) 0; } - PCTInfo pct_mean; - compute_pct_mean(pct_info, n_bin, pct_mean); + return; +} + +//////////////////////////////////////////////////////////////////////// + +void do_pct_cdp_thresh(const EnsembleStatVxOpt &vx_opt, + const PairDataEnsemble &pd_ens) { + int i, i_thr, i_bin, i_obs, i_ens; + int n_vld, n_evt, n_bin; + PCTInfo *pct_info = (PCTInfo *) 0; + PairDataPoint pd_pnt, pd; + ThreshArray cdp_thresh; - // Write out PSTD - if(vx_opt.output_flag[i_pstd] != STATOutputType_None) { - write_pstd_row(shc, pct_mean, - vx_opt.output_flag[i_pstd], - -1, n_bin, stat_at, i_stat_row, - txt_at[i_pstd], i_txt_row[i_pstd]); + // Derive a PairDataPoint object from the PairDataEnsemble input + pd_pnt.extend(pd_ens.n_obs); + + // Derive the climo distribution percentile thresholds + cdp_thresh = derive_cdp_thresh(vx_opt.cdf_info.cdf_ta); + n_bin = cdp_thresh.n(); + + mlog << Debug(2) << "Applying " << n_bin + << " climatological distribution percentile thresholds.\n"; + + // Allocate memory + pct_info = new PCTInfo [n_bin]; + + // Process each probability threshold + for(i_bin=0; i_bin 0 || (double) (n_vld/pd_ens.n_ens) >= conf_info.vld_data_thresh) { + pd_pnt.add_grid_pair((double) n_evt/n_vld, pd_ens.o_na[i_obs], + pd_ens.cmn_na[i_obs], pd_ens.csd_na[i_obs], + pd_ens.wgt_na[i_obs]); } - } // end if n_bin > 1 - } // end for i (ocnt_ta) + } // end for i_obs + + // Initialize + pct_info[i_bin].clear(); - // Reset the forecast variable name - shc.set_fcst_var(fcst_var_cs); + // Store thresholds + pct_info[i_bin].fthresh = vx_opt.fpct_ta; + pct_info[i_bin].othresh = cdp_thresh[i_bin]; + pct_info[i_bin].allocate_n_alpha(vx_opt.get_n_ci_alpha()); + + for(i=0; i 1) { + + PCTInfo pct_mean; + compute_pct_mean(pct_info, n_bin, pct_mean); + + // Write out PSTD + if(vx_opt.output_flag[i_pstd] != STATOutputType_None) { + write_pstd_row(shc, pct_mean, + vx_opt.output_flag[i_pstd], + -1, n_bin, stat_at, i_stat_row, + txt_at[i_pstd], i_txt_row[i_pstd]); + } + } // end if n_bin > 1 + + return; +} + +//////////////////////////////////////////////////////////////////////// + void write_ens_nc(EnsVarInfo * ens_info, int i_var, DataPlane &ens_dp) { int i, j, k, l; double t, v; diff --git a/met/src/tools/core/ensemble_stat/ensemble_stat_conf_info.cc b/met/src/tools/core/ensemble_stat/ensemble_stat_conf_info.cc index 87d2dab82a..cfde7621a0 100644 --- a/met/src/tools/core/ensemble_stat/ensemble_stat_conf_info.cc +++ b/met/src/tools/core/ensemble_stat/ensemble_stat_conf_info.cc @@ -797,8 +797,9 @@ void EnsembleStatVxOpt::clear() { ssvar_bin_size = bad_data_double; phist_bin_size = bad_data_double; - prob_cat_ta.clear(); - prob_pct_ta.clear(); + fcat_ta.clear(); + ocat_ta.clear(); + fpct_ta.clear(); duplicate_flag = DuplicateType_None; obs_summary = ObsSummary_None; @@ -983,11 +984,22 @@ void EnsembleStatVxOpt::process_config(GrdFileType ftype, Dictionary &fdict, phist_bin_size = odict.lookup_double(conf_key_phist_bin); // Conf: prob_cat_thresh - prob_cat_ta = fdict.lookup_thresh_array(conf_key_prob_cat_thresh); + fcat_ta = fdict.lookup_thresh_array(conf_key_prob_cat_thresh); + ocat_ta = odict.lookup_thresh_array(conf_key_prob_cat_thresh); + + // The number of thresholds must match + if(fcat_ta.n() != ocat_ta.n()) { + mlog << Error << "\nEnsembleStatVxOpt::process_config() -> " + << "The number of forecast (" << fcat_ta.n() + << ") and observation (" << ocat_ta.n() + << ") probability category thresholds in \"" + << conf_key_prob_cat_thresh << "\" must match.\n\n"; + exit(1); + } // Conf: prob_pct_thresh - prob_pct_ta = fdict.lookup_thresh_array(conf_key_prob_pct_thresh); - prob_pct_ta = string_to_prob_thresh(prob_pct_ta.get_str().c_str()); + fpct_ta = fdict.lookup_thresh_array(conf_key_prob_pct_thresh); + fpct_ta = string_to_prob_thresh(fpct_ta.get_str().c_str()); // Conf: duplicate_flag duplicate_flag = parse_conf_duplicate_flag(&odict); @@ -1151,7 +1163,9 @@ void EnsembleStatVxOpt::set_perc_thresh(const PairDataEnsemble *pd_ptr) { // // Check if percentile thresholds were requested // - if(!othr_ta.need_perc() && !prob_cat_ta.need_perc()) return; + if(!othr_ta.need_perc() && + !fcat_ta.need_perc() && + !ocat_ta.need_perc()) return; // // Sort the input arrays @@ -1165,13 +1179,12 @@ void EnsembleStatVxOpt::set_perc_thresh(const PairDataEnsemble *pd_ptr) { csort.sort_array(); // - // Compute percentiles, passing the observation thresholds in for - // the fcst and obs slots. + // Compute percentiles, passing the observation filtering + // thresholds in for the fcst and obs slots. // - othr_ta.set_perc(&fsort, &osort, &csort, - &othr_ta, &othr_ta); - prob_cat_ta.set_perc(&fsort, &osort, &csort, - &prob_cat_ta, &prob_cat_ta); + othr_ta.set_perc(&fsort, &osort, &csort, &othr_ta, &othr_ta); + fcat_ta.set_perc(&fsort, &osort, &csort, &fcat_ta, &ocat_ta); + ocat_ta.set_perc(&fsort, &osort, &csort, &fcat_ta, &ocat_ta); return; } diff --git a/met/src/tools/core/ensemble_stat/ensemble_stat_conf_info.h b/met/src/tools/core/ensemble_stat/ensemble_stat_conf_info.h index 846176d95b..73c3a60d95 100644 --- a/met/src/tools/core/ensemble_stat/ensemble_stat_conf_info.h +++ b/met/src/tools/core/ensemble_stat/ensemble_stat_conf_info.h @@ -131,8 +131,9 @@ class EnsembleStatVxOpt { double ssvar_bin_size; // SSVAR bin size double phist_bin_size; // PHIST bin size - ThreshArray prob_cat_ta; // Categorical thresholds for probabilities, including RPS - ThreshArray prob_pct_ta; // PCT thresholds + ThreshArray fcat_ta; // Forecast categorical probability-definition thresholds, including RPS + ThreshArray ocat_ta; // Observation categorical event-definition thresholds + ThreshArray fpct_ta; // Forecast PCT thresholds DuplicateType duplicate_flag; // Duplicate observations ObsSummary obs_summary; // Summarize observations @@ -181,8 +182,8 @@ inline int EnsembleStatVxOpt::get_n_mask() const { return(mask_name.n()); inline int EnsembleStatVxOpt::get_n_mask_area() const { return(mask_name_area.n()); } inline int EnsembleStatVxOpt::get_n_obs_thresh() const { return(othr_ta.n()); } -inline int EnsembleStatVxOpt::get_n_prob_cat_thresh() const { return(prob_cat_ta.n()); } -inline int EnsembleStatVxOpt::get_n_prob_pct_thresh() const { return(prob_pct_ta.n()); } +inline int EnsembleStatVxOpt::get_n_prob_cat_thresh() const { return(fcat_ta.n()); } +inline int EnsembleStatVxOpt::get_n_prob_pct_thresh() const { return(fpct_ta.n()); } inline int EnsembleStatVxOpt::get_n_eclv_points() const { return(eclv_points.n()); } inline int EnsembleStatVxOpt::get_n_cdf_bin() const { return(cdf_info.n_bin); } From 1cc85943840a7e6e4388ef0257991d9779a03d6c Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Mon, 28 Feb 2022 17:40:01 -0700 Subject: [PATCH 10/16] Per #1259, make the MET test script Ensemble-Stat config file more consistent with the default one. --- met/scripts/config/EnsembleStatConfig | 44 +++++++++++++++++++++------ 1 file changed, 35 insertions(+), 9 deletions(-) diff --git a/met/scripts/config/EnsembleStatConfig b/met/scripts/config/EnsembleStatConfig index e2bc7d71b3..0b8fcd894c 100644 --- a/met/scripts/config/EnsembleStatConfig +++ b/met/scripts/config/EnsembleStatConfig @@ -26,22 +26,25 @@ obtype = "MC_PCP"; // // Verification grid +// May be set separately in each "field" entry // regrid = { to_grid = NONE; method = NEAREST; width = 1; vld_thresh = 0.5; + shape = SQUARE; } //////////////////////////////////////////////////////////////////////////////// // -// May be set separately in each "field" entry +// May be set separately in each "ens.field" entry // censor_thresh = []; censor_val = []; cat_thresh = []; +nc_var_str = ""; // // Ensemble product fields to be processed @@ -80,6 +83,13 @@ ens = { ]; } +// +// IDs for ensemble members +// Only set if processing a single file with all ensembles +// +ens_member_ids = []; +control_id = ""; + //////////////////////////////////////////////////////////////////////////////// // @@ -110,9 +120,13 @@ nmep_smooth = { //////////////////////////////////////////////////////////////////////////////// // -// May be set separately in each "fcst.field" entry +// May be set separately in each "fcst.field" and "obs.field" entry // prob_cat_thresh = []; + +// +// May be set separately in each "fcst.field" entry +// prob_pct_thresh = [ ==0.25 ]; // @@ -144,6 +158,7 @@ obs = fcst; // Point observation filtering options // May be set separately in each "obs.field" entry // +message_type = []; sid_inc = []; sid_exc = []; obs_thresh = [ NA ]; @@ -165,8 +180,20 @@ obs_error = { dist_parm = []; inst_bias_scale = 1.0; inst_bias_offset = 0.0; + min = NA; + max = NA; } +// +// Mapping of message type group name to comma-separated list of values +// +message_type_group_map = [ + { key = "SURFACE"; val = "ADPSFC,SFCSHP,MSONET"; }, + { key = "ANYAIR"; val = "AIRCAR,AIRCFT"; }, + { key = "ANYSFC"; val = "ADPSFC,SFCSHP,ADPUPA,PROFLR,MSONET"; }, + { key = "ONLYSF"; val = "ADPSFC,SFCSHP"; } +]; + // // Ensemble bin sizes // May be set separately in each "obs.field" entry @@ -174,16 +201,10 @@ obs_error = { ens_ssvar_bin_size = 1.0; ens_phist_bin_size = 0.05; -// -// Categorical thresholds to define ensemble probabilities -// May be set separately in each "fcst.field" entry -// -prob_cat_thresh = []; - //////////////////////////////////////////////////////////////////////////////// // -// Climatology mean data +// Climatology data // climo_mean = { @@ -194,6 +215,7 @@ climo_mean = { method = NEAREST; width = 1; vld_thresh = 0.5; + shape = SQUARE; } time_interp_method = DW_MEAN; @@ -212,6 +234,8 @@ climo_stdev = { climo_cdf = { cdf_bins = 10; center_bins = FALSE; + write_bins = TRUE; + direct_prob = FALSE; } //////////////////////////////////////////////////////////////////////////////// @@ -255,6 +279,7 @@ ci_alpha = [ 0.05 ]; interp = { field = BOTH; vld_thresh = 1.0; + shape = SQUARE; type = [ { @@ -294,6 +319,7 @@ output_flag = { // Ensemble product output types // ensemble_flag = { + latlon = TRUE; mean = TRUE; stdev = TRUE; minus = TRUE; From 77063ec79f360471d068f4d48886f5f61170d2bc Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Tue, 1 Mar 2022 09:47:52 -0700 Subject: [PATCH 11/16] Per #1259, wrangling Ensemble-Stat config files, trying to make them as consistent as possible with the default configuration. --- met/data/config/EnsembleStatConfig_default | 23 ++-- met/scripts/config/EnsembleStatConfig | 21 ++- test/config/EnsembleStatConfig | 70 +++++++--- test/config/EnsembleStatConfig_MASK_SID | 70 +++++++--- test/config/EnsembleStatConfig_climo | 127 +++++++++++------- test/config/EnsembleStatConfig_grid_weight | 54 ++++++-- test/config/EnsembleStatConfig_one_cdf_bin | 118 ++++++++++------ test/config/EnsembleStatConfig_python | 80 +++++------ test/config/EnsembleStatConfig_qty_inc_exc | 66 +++++++-- .../EnsembleStatConfig_single_file_grib | 72 ++++++---- test/config/EnsembleStatConfig_single_file_nc | 54 +++++--- 11 files changed, 490 insertions(+), 265 deletions(-) diff --git a/met/data/config/EnsembleStatConfig_default b/met/data/config/EnsembleStatConfig_default index 0413cae12e..1cb84c3714 100644 --- a/met/data/config/EnsembleStatConfig_default +++ b/met/data/config/EnsembleStatConfig_default @@ -26,7 +26,6 @@ obtype = "ANALYS"; // // Verification grid -// May be set separately in each "field" entry // regrid = { to_grid = NONE; @@ -132,16 +131,16 @@ obs = fcst; // Point observation filtering options // May be set separately in each "obs.field" entry // -message_type = [ "ADPUPA" ]; -sid_inc = []; -sid_exc = []; -obs_thresh = [ NA ]; -obs_quality_inc = []; -obs_quality_exc = []; -duplicate_flag = NONE; -obs_summary = NONE; -obs_perc_value = 50; -skip_const = FALSE; +message_type = [ "ADPUPA" ]; +sid_inc = []; +sid_exc = []; +obs_thresh = [ NA ]; +obs_quality_inc = []; +obs_quality_exc = []; +duplicate_flag = NONE; +obs_summary = NONE; +obs_perc_value = 50; +skip_const = FALSE; // // Observation error options @@ -249,7 +248,7 @@ ci_alpha = [ 0.05 ]; interp = { field = BOTH; vld_thresh = 1.0; - shape = SQUARE; + shape = SQUARE; type = [ { diff --git a/met/scripts/config/EnsembleStatConfig b/met/scripts/config/EnsembleStatConfig index 0b8fcd894c..9a1845555e 100644 --- a/met/scripts/config/EnsembleStatConfig +++ b/met/scripts/config/EnsembleStatConfig @@ -26,7 +26,6 @@ obtype = "MC_PCP"; // // Verification grid -// May be set separately in each "field" entry // regrid = { to_grid = NONE; @@ -158,16 +157,16 @@ obs = fcst; // Point observation filtering options // May be set separately in each "obs.field" entry // -message_type = []; -sid_inc = []; -sid_exc = []; -obs_thresh = [ NA ]; -obs_quality_inc = []; -obs_quality_exc = []; -duplicate_flag = NONE; -obs_summary = NONE; -obs_perc_value = 50; -skip_const = FALSE; +message_type = []; +sid_inc = []; +sid_exc = []; +obs_thresh = [ NA ]; +obs_quality_inc = []; +obs_quality_exc = []; +duplicate_flag = NONE; +obs_summary = NONE; +obs_perc_value = 50; +skip_const = FALSE; // // Observation error options diff --git a/test/config/EnsembleStatConfig b/test/config/EnsembleStatConfig index 296555748e..1b12b3e973 100644 --- a/test/config/EnsembleStatConfig +++ b/test/config/EnsembleStatConfig @@ -28,13 +28,23 @@ obtype = "ANALYS"; // Verification grid // regrid = { - to_grid = NONE; - method = NEAREST; - width = 1; + to_grid = NONE; + method = NEAREST; + width = 1; + vld_thresh = 0.5; + shape = SQUARE; } //////////////////////////////////////////////////////////////////////////////// +// +// May be set separately in each "ens.field" entry +// +censor_thresh = []; +censor_val = []; +cat_thresh = []; +nc_var_str = ""; + // // Ensemble product fields to be processed // @@ -92,13 +102,26 @@ nmep_smooth = { //////////////////////////////////////////////////////////////////////////////// +// +// May be set separately in each "fcst.field" and "obs.field" entry +// +prob_cat_thresh = []; + +// +// May be set separately in each "fcst.field" entry +// +prob_pct_thresh = [ ==0.25 ]; + +// +// May be set separately in each "obs.field" entry +// +eclv_points = 0.05; + // // Forecast and observation fields to be verified // fcst = { message_type = [ "ADPSFC" ]; - sid_inc = []; - sid_exc = []; obs_quality_inc = []; obs_quality_exc = []; @@ -111,7 +134,7 @@ fcst = { name = "APCP"; level = "A24"; obs_thresh = [ NA, >0, >=2.54 ]; - obs_error = { + obs_error = { flag = ${OBS_ERROR_FLAG}; dist_type = NONE; } @@ -126,11 +149,16 @@ obs = fcst; // Point observation filtering options // May be set separately in each "obs.field" entry // -obs_thresh = [ NA ]; -duplicate_flag = NONE; -obs_summary = NONE; -obs_perc_value = 50; -skip_const = ${SKIP_CONST}; +message_type = []; +sid_inc = []; +sid_exc = []; +obs_thresh = [ NA ]; +obs_quality_inc = []; +obs_quality_exc = []; +duplicate_flag = NONE; +obs_summary = NONE; +obs_perc_value = 50; +skip_const = ${SKIP_CONST}; // // Observation error options @@ -154,12 +182,6 @@ obs_error = { ens_ssvar_bin_size = 1.0; ens_phist_bin_size = 0.05; -// -// Categorical thresholds to define ensemble probabilities -// May be set separately in each "fcst.field" entry -// -prob_cat_thresh = []; - //////////////////////////////////////////////////////////////////////////////// // @@ -185,12 +207,20 @@ mask = { //////////////////////////////////////////////////////////////////////////////// +// +// Confidence interval settings +// +ci_alpha = [ 0.05 ]; + +//////////////////////////////////////////////////////////////////////////////// + // // Interpolation methods // interp = { field = BOTH; vld_thresh = 1.0; + shape = SQUARE; type = [ { @@ -213,6 +243,11 @@ output_flag = { orank = BOTH; ssvar = BOTH; relp = BOTH; + pct = NONE; + pstd = NONE; + pjc = NONE; + prc = NONE; + eclv = NONE; } //////////////////////////////////////////////////////////////////////////////// @@ -221,6 +256,7 @@ output_flag = { // Ensemble product output types // ensemble_flag = { + latlon = TRUE; mean = TRUE; stdev = TRUE; minus = TRUE; diff --git a/test/config/EnsembleStatConfig_MASK_SID b/test/config/EnsembleStatConfig_MASK_SID index 764ad8280c..76e4adb43f 100644 --- a/test/config/EnsembleStatConfig_MASK_SID +++ b/test/config/EnsembleStatConfig_MASK_SID @@ -28,13 +28,23 @@ obtype = "ANALYS"; // Verification grid // regrid = { - to_grid = NONE; - method = NEAREST; - width = 1; + to_grid = NONE; + method = NEAREST; + width = 1; + vld_thresh = 0.5; + shape = SQUARE; } //////////////////////////////////////////////////////////////////////////////// +// +// May be set separately in each "ens.field" entry +// +censor_thresh = []; +censor_val = []; +cat_thresh = []; +nc_var_str = ""; + // // Ensemble product fields to be processed // @@ -89,15 +99,26 @@ nmep_smooth = { //////////////////////////////////////////////////////////////////////////////// +// +// May be set separately in each "fcst.field" and "obs.field" entry +// +prob_cat_thresh = []; + +// +// May be set separately in each "fcst.field" entry +// +prob_pct_thresh = [ ==0.25 ]; + +// +// May be set separately in each "obs.field" entry +// +eclv_points = 0.05; + // // Forecast and observation fields to be verified // fcst = { message_type = [ "ADPSFC" ]; - sid_inc = []; - sid_exc = []; - obs_quality_inc = []; - obs_quality_exc = []; field = [ { @@ -116,11 +137,16 @@ obs = fcst; // Point observation filtering options // May be set separately in each "obs.field" entry // -obs_thresh = [ NA ]; -duplicate_flag = NONE; -obs_summary = NONE; -obs_perc_value = 50; -skip_const = FALSE; +message_type = []; +sid_inc = []; +sid_exc = []; +obs_thresh = [ NA ]; +obs_quality_inc = []; +obs_quality_exc = []; +duplicate_flag = NONE; +obs_summary = NONE; +obs_perc_value = 50; +skip_const = FALSE; // // Observation error options @@ -144,12 +170,6 @@ obs_error = { ens_ssvar_bin_size = 1.0; ens_phist_bin_size = 0.05; -// -// Categorical thresholds to define ensemble probabilities -// May be set separately in each "fcst.field" entry -// -prob_cat_thresh = []; - //////////////////////////////////////////////////////////////////////////////// // @@ -176,12 +196,20 @@ mask = { //////////////////////////////////////////////////////////////////////////////// +// +// Confidence interval settings +// +ci_alpha = [ 0.05 ]; + +//////////////////////////////////////////////////////////////////////////////// + // // Interpolation methods // interp = { field = BOTH; vld_thresh = 1.0; + shape = SQUARE; type = [ { @@ -204,6 +232,11 @@ output_flag = { orank = BOTH; ssvar = STAT; relp = STAT; + pct = NONE; + pstd = NONE; + pjc = NONE; + prc = NONE; + eclv = NONE; } //////////////////////////////////////////////////////////////////////////////// @@ -212,6 +245,7 @@ output_flag = { // Ensemble product output types // ensemble_flag = { + latlon = TRUE; mean = TRUE; stdev = TRUE; minus = FALSE; diff --git a/test/config/EnsembleStatConfig_climo b/test/config/EnsembleStatConfig_climo index 478ceb1552..936dde8175 100644 --- a/test/config/EnsembleStatConfig_climo +++ b/test/config/EnsembleStatConfig_climo @@ -28,13 +28,23 @@ obtype = "ANALYS"; // Verification grid // regrid = { - to_grid = NONE; - method = NEAREST; - width = 1; + to_grid = NONE; + method = NEAREST; + width = 1; + vld_thresh = 0.5; + shape = SQUARE; } //////////////////////////////////////////////////////////////////////////////// +// +// May be set separately in each "ens.field" entry +// +censor_thresh = []; +censor_val = []; +cat_thresh = []; +nc_var_str = ""; + // // Ensemble product fields to be processed // @@ -84,6 +94,21 @@ nmep_smooth = { //////////////////////////////////////////////////////////////////////////////// +// +// May be set separately in each "fcst.field" and "obs.field" entry +// +prob_cat_thresh = []; + +// +// May be set separately in each "fcst.field" entry +// +prob_pct_thresh = [ ==0.25 ]; + +// +// May be set separately in each "obs.field" entry +// +eclv_points = 0.05; + // // Forecast and observation fields to be verified // @@ -97,6 +122,45 @@ obs = fcst; //////////////////////////////////////////////////////////////////////////////// +// +// Point observation filtering options +// May be set separately in each "obs.field" entry +// +message_type = []; +sid_inc = []; +sid_exc = []; +obs_thresh = [ NA ]; +obs_quality_inc = []; +obs_quality_exc = []; +duplicate_flag = NONE; +obs_summary = NONE; +obs_perc_value = 50; +skip_const = FALSE; + +// +// Observation error options +// Set dist_type to NONE to use the observation error table instead +// May be set separately in each "obs.field" entry +// +obs_error = { + flag = FALSE; + dist_type = NONE; + dist_parm = []; + inst_bias_scale = 1.0; + inst_bias_offset = 0.0; + min = NA; + max = NA; +} + +// +// Ensemble bin sizes +// May be set separately in each "obs.field" entry +// +ens_ssvar_bin_size = 1.0; +ens_phist_bin_size = 0.05; + +//////////////////////////////////////////////////////////////////////////////// + // // Climatology data // @@ -126,52 +190,12 @@ climo_stdev = { climo_cdf = { cdf_bins = 10; center_bins = FALSE; + write_bins = TRUE; + direct_prob = FALSE; } //////////////////////////////////////////////////////////////////////////////// -// -// Point observation filtering options -// May be set separately in each "obs.field" entry -// -obs_thresh = [ NA ]; -obs_quality_inc = []; -obs_quality_exc = []; -duplicate_flag = NONE; -obs_summary = NONE; -obs_perc_value = 50; -skip_const = FALSE; - -// -// Observation error options -// Set dist_type to NONE to use the observation error table instead -// May be set separately in each "obs.field" entry -// -obs_error = { - flag = FALSE; - dist_type = NONE; - dist_parm = []; - inst_bias_scale = 1.0; - inst_bias_offset = 0.0; - min = NA; - max = NA; -} - -// -// Ensemble bin sizes -// May be set separately in each "obs.field" entry -// -ens_ssvar_bin_size = 1.0; -ens_phist_bin_size = 0.05; - -// -// Categorical thresholds to define ensemble probabilities -// May be set separately in each "fcst.field" entry -// -prob_cat_thresh = []; - -//////////////////////////////////////////////////////////////////////////////// - // // Point observation time window // @@ -195,12 +219,20 @@ mask = { //////////////////////////////////////////////////////////////////////////////// +// +// Confidence interval settings +// +ci_alpha = [ 0.05 ]; + +//////////////////////////////////////////////////////////////////////////////// + // // Interpolation methods // interp = { field = BOTH; vld_thresh = 1.0; + shape = SQUARE; type = [ { @@ -223,6 +255,11 @@ output_flag = { orank = BOTH; ssvar = STAT; relp = STAT; + pct = NONE; + pstd = NONE; + pjc = NONE; + prc = NONE; + eclv = NONE; } //////////////////////////////////////////////////////////////////////////////// diff --git a/test/config/EnsembleStatConfig_grid_weight b/test/config/EnsembleStatConfig_grid_weight index dbd72bc580..19325f310a 100644 --- a/test/config/EnsembleStatConfig_grid_weight +++ b/test/config/EnsembleStatConfig_grid_weight @@ -32,10 +32,19 @@ regrid = { method = NEAREST; width = 1; vld_thresh = 0.5; + shape = SQUARE; } //////////////////////////////////////////////////////////////////////////////// +// +// May be set separately in each "ens.field" entry +// +censor_thresh = []; +censor_val = []; +cat_thresh = []; +nc_var_str = ""; + tmp_field = [ { name = "TMP"; level = [ "Z2" ]; } ]; // @@ -83,6 +92,21 @@ nmep_smooth = { //////////////////////////////////////////////////////////////////////////////// +// +// May be set separately in each "fcst.field" and "obs.field" entry +// +prob_cat_thresh = []; + +// +// May be set separately in each "fcst.field" entry +// +prob_pct_thresh = [ ==0.25 ]; + +// +// May be set separately in each "obs.field" entry +// +eclv_points = 0.05; + // // Forecast and observation fields to be verified // @@ -102,11 +126,16 @@ obs = fcst; // Point observation filtering options // May be set separately in each "obs.field" entry // -obs_thresh = [ NA ]; -duplicate_flag = NONE; -obs_summary = NONE; -obs_perc_value = 50; -skip_const = FALSE; +message_type = []; +sid_inc = []; +sid_exc = []; +obs_thresh = [ NA ]; +obs_quality_inc = []; +obs_quality_exc = []; +duplicate_flag = NONE; +obs_summary = NONE; +obs_perc_value = 50; +skip_const = FALSE; // // Observation error options @@ -130,16 +159,10 @@ obs_error = { ens_ssvar_bin_size = 1.0; ens_phist_bin_size = 0.05; -// -// Categorical thresholds to define ensemble probabilities -// May be set separately in each "fcst.field" entry -// -prob_cat_thresh = []; - //////////////////////////////////////////////////////////////////////////////// // -// Climatology mean data +// Climatology data // climo_mean = { @@ -194,6 +217,7 @@ ci_alpha = [ 0.05 ]; interp = { field = BOTH; vld_thresh = 1.0; + shape = SQUARE; type = [ { @@ -216,6 +240,11 @@ output_flag = { orank = NONE; ssvar = STAT; relp = STAT; + pct = NONE; + pstd = NONE; + pjc = NONE; + prc = NONE; + eclv = NONE; } //////////////////////////////////////////////////////////////////////////////// @@ -224,6 +253,7 @@ output_flag = { // Ensemble product output types // ensemble_flag = { + latlon = TRUE; mean = TRUE; stdev = FALSE; minus = FALSE; diff --git a/test/config/EnsembleStatConfig_one_cdf_bin b/test/config/EnsembleStatConfig_one_cdf_bin index 2a02b9475e..1197e77517 100644 --- a/test/config/EnsembleStatConfig_one_cdf_bin +++ b/test/config/EnsembleStatConfig_one_cdf_bin @@ -28,13 +28,23 @@ obtype = "ANALYS"; // Verification grid // regrid = { - to_grid = NONE; - method = NEAREST; - width = 1; + to_grid = NONE; + method = NEAREST; + width = 1; + vld_thresh = 0.5; + shape = SQUARE; } //////////////////////////////////////////////////////////////////////////////// +// +// May be set separately in each "ens.field" entry +// +censor_thresh = []; +censor_val = []; +cat_thresh = []; +nc_var_str = ""; + // // Ensemble product fields to be processed // @@ -85,43 +95,30 @@ nmep_smooth = { //////////////////////////////////////////////////////////////////////////////// // -// Forecast and observation fields to be verified +// May be set separately in each "fcst.field" and "obs.field" entry // -fcst = { - field = [ - { name = "TMP"; level = "Z2"; message_type = [ "ADPSFC" ]; }, - { name = "TMP"; level = "P850"; message_type = [ "ADPUPA" ]; } - ]; -} -obs = fcst; - -//////////////////////////////////////////////////////////////////////////////// +prob_cat_thresh = []; // -// Climatology data +// May be set separately in each "fcst.field" entry // -climo_mean = fcst; -climo_mean = { - - file_name = [ ${CLIMO_MEAN_FILE_LIST} ]; - regrid = { - method = BILIN; - width = 2; - vld_thresh = 0.5; - } - - time_interp_method = DW_MEAN; - day_interval = 1; - hour_interval = 6; -} +prob_pct_thresh = [ ==0.25 ]; // // May be set separately in each "obs.field" entry // -climo_cdf = { - cdf_bins = 1; - center_bins = FALSE; +eclv_points = 0.05; + +// +// Forecast and observation fields to be verified +// +fcst = { + field = [ + { name = "TMP"; level = "Z2"; message_type = [ "ADPSFC" ]; }, + { name = "TMP"; level = "P850"; message_type = [ "ADPUPA" ]; } + ]; } +obs = fcst; //////////////////////////////////////////////////////////////////////////////// @@ -129,13 +126,16 @@ climo_cdf = { // Point observation filtering options // May be set separately in each "obs.field" entry // -obs_thresh = [ NA ]; -obs_quality_inc = []; -obs_quality_exc = []; -duplicate_flag = NONE; -obs_summary = NONE; -obs_perc_value = 50; -skip_const = FALSE; +message_type = []; +sid_inc = []; +sid_exc = []; +obs_thresh = [ NA ]; +obs_quality_inc = []; +obs_quality_exc = []; +duplicate_flag = NONE; +obs_summary = NONE; +obs_perc_value = 50; +skip_const = FALSE; // // Observation error options @@ -159,11 +159,34 @@ obs_error = { ens_ssvar_bin_size = 1.0; ens_phist_bin_size = 0.05; +//////////////////////////////////////////////////////////////////////////////// + // -// Categorical thresholds to define ensemble probabilities -// May be set separately in each "fcst.field" entry +// Climatology data // -prob_cat_thresh = []; +climo_mean = fcst; +climo_mean = { + + file_name = [ ${CLIMO_MEAN_FILE_LIST} ]; + regrid = { + method = BILIN; + width = 2; + vld_thresh = 0.5; + shape = SQUARE; + } + + time_interp_method = DW_MEAN; + day_interval = 1; + hour_interval = 6; +} + +// +// May be set separately in each "obs.field" entry +// +climo_cdf = { + cdf_bins = 1; + center_bins = FALSE; +} //////////////////////////////////////////////////////////////////////////////// @@ -189,12 +212,20 @@ mask = { //////////////////////////////////////////////////////////////////////////////// +// +// Confidence interval settings +// +ci_alpha = [ 0.05 ]; + +//////////////////////////////////////////////////////////////////////////////// + // // Interpolation methods // interp = { field = BOTH; vld_thresh = 1.0; + shape = SQUARE; type = [ { @@ -217,6 +248,11 @@ output_flag = { orank = NONE; ssvar = NONE; relp = NONE; + pct = NONE; + pstd = NONE; + pjc = NONE; + prc = NONE; + eclv = NONE; } //////////////////////////////////////////////////////////////////////////////// diff --git a/test/config/EnsembleStatConfig_python b/test/config/EnsembleStatConfig_python index 59ae439495..b8261dcde3 100644 --- a/test/config/EnsembleStatConfig_python +++ b/test/config/EnsembleStatConfig_python @@ -26,7 +26,6 @@ obtype = "ANALYS"; // // Verification grid -// May be set separately in each "field" entry // regrid = { to_grid = NONE; @@ -39,7 +38,7 @@ regrid = { //////////////////////////////////////////////////////////////////////////////// // -// May be set separately in each "field" entry +// May be set separately in each "ens.field" entry // censor_thresh = []; censor_val = []; @@ -92,6 +91,21 @@ nmep_smooth = { //////////////////////////////////////////////////////////////////////////////// +// +// May be set separately in each "fcst.field" and "obs.field" entry +// +prob_cat_thresh = []; + +// +// May be set separately in each "fcst.field" entry +// +prob_pct_thresh = [ ==0.25 ]; + +// +// May be set separately in each "obs.field" entry +// +eclv_points = 0.05; + // // Forecast and observation fields to be verified // @@ -107,16 +121,16 @@ obs = { // Point observation filtering options // May be set separately in each "obs.field" entry // -message_type = []; -sid_inc = []; -sid_exc = []; -obs_thresh = [ NA ]; -obs_quality_inc = []; -obs_quality_exc = []; -duplicate_flag = NONE; -obs_summary = NONE; -obs_perc_value = 50; -skip_const = FALSE; +message_type = []; +sid_inc = []; +sid_exc = []; +obs_thresh = [ NA ]; +obs_quality_inc = []; +obs_quality_exc = []; +duplicate_flag = NONE; +obs_summary = NONE; +obs_perc_value = 50; +skip_const = FALSE; // // Observation error options @@ -134,7 +148,7 @@ obs_error = { } // -// Mapping of message type group name to comma-separated list of values. +// Mapping of message type group name to comma-separated list of values // message_type_group_map = [ { key = "SURFACE"; val = "ADPSFC,SFCSHP,MSONET"; }, @@ -150,39 +164,6 @@ message_type_group_map = [ ens_ssvar_bin_size = 1.0; ens_phist_bin_size = 0.05; -// -// Categorical thresholds to define ensemble probabilities -// May be set separately in each "fcst.field" entry -// -prob_cat_thresh = []; - -//////////////////////////////////////////////////////////////////////////////// - -// -// Climatology data -// -climo_mean = { - - file_name = []; - field = []; - - regrid = { - method = NEAREST; - width = 1; - vld_thresh = 0.5; - shape = SQUARE; - } - - time_interp_method = DW_MEAN; - day_interval = 31; - hour_interval = 6; -} - -climo_stdev = climo_mean; -climo_stdev = { - file_name = []; -} - //////////////////////////////////////////////////////////////////////////////// // @@ -220,7 +201,7 @@ ci_alpha = [ 0.05 ]; interp = { field = BOTH; vld_thresh = 1.0; - shape = SQUARE; + shape = SQUARE; type = [ { @@ -243,6 +224,11 @@ output_flag = { orank = NONE; ssvar = NONE; relp = NONE; + pct = NONE; + pstd = NONE; + pjc = NONE; + prc = NONE; + eclv = NONE; } //////////////////////////////////////////////////////////////////////////////// diff --git a/test/config/EnsembleStatConfig_qty_inc_exc b/test/config/EnsembleStatConfig_qty_inc_exc index 620511dd4c..62b42d75c2 100644 --- a/test/config/EnsembleStatConfig_qty_inc_exc +++ b/test/config/EnsembleStatConfig_qty_inc_exc @@ -28,13 +28,23 @@ obtype = "ANALYS"; // Verification grid // regrid = { - to_grid = NONE; - method = NEAREST; - width = 1; + to_grid = NONE; + method = NEAREST; + width = 1; + vld_thresh = 0.5; + shape = SQUARE; } //////////////////////////////////////////////////////////////////////////////// +// +// May be set separately in each "ens.field" entry +// +censor_thresh = []; +censor_val = []; +cat_thresh = []; +nc_var_str = ""; + // // Ensemble product fields to be processed // @@ -87,6 +97,21 @@ nmep_smooth = { //////////////////////////////////////////////////////////////////////////////// +// +// May be set separately in each "fcst.field" and "obs.field" entry +// +prob_cat_thresh = []; + +// +// May be set separately in each "fcst.field" entry +// +prob_pct_thresh = [ ==0.25 ]; + +// +// May be set separately in each "obs.field" entry +// +eclv_points = 0.05; + // // Forecast and observation fields to be verified // @@ -129,11 +154,16 @@ obs = fcst; // Point observation filtering options // May be set separately in each "obs.field" entry // -obs_thresh = [ NA ]; -duplicate_flag = NONE; -obs_summary = NONE; -obs_perc_value = 50; -skip_const = FALSE; +message_type = []; +sid_inc = []; +sid_exc = []; +obs_thresh = [ NA ]; +obs_quality_inc = []; +obs_quality_exc = []; +duplicate_flag = NONE; +obs_summary = NONE; +obs_perc_value = 50; +skip_const = FALSE; // // Observation error options @@ -157,12 +187,6 @@ obs_error = { ens_ssvar_bin_size = 1.0; ens_phist_bin_size = 0.05; -// -// Categorical thresholds to define ensemble probabilities -// May be set separately in each "fcst.field" entry -// -prob_cat_thresh = []; - //////////////////////////////////////////////////////////////////////////////// // @@ -187,12 +211,20 @@ mask = { //////////////////////////////////////////////////////////////////////////////// +// +// Confidence interval settings +// +ci_alpha = [ 0.05 ]; + +//////////////////////////////////////////////////////////////////////////////// + // // Interpolation methods // interp = { field = BOTH; vld_thresh = 1.0; + shape = SQUARE; type = [ { @@ -215,6 +247,11 @@ output_flag = { orank = BOTH; ssvar = NONE; relp = NONE; + pct = NONE; + pstd = NONE; + pjc = NONE; + prc = NONE; + eclv = NONE; } //////////////////////////////////////////////////////////////////////////////// @@ -223,6 +260,7 @@ output_flag = { // Ensemble product output types // ensemble_flag = { + latlon = TRUE; mean = TRUE; stdev = TRUE; minus = TRUE; diff --git a/test/config/EnsembleStatConfig_single_file_grib b/test/config/EnsembleStatConfig_single_file_grib index 76bb0bf3b5..27c802e30a 100644 --- a/test/config/EnsembleStatConfig_single_file_grib +++ b/test/config/EnsembleStatConfig_single_file_grib @@ -26,7 +26,6 @@ obtype = "ANALYS"; // // Verification grid -// May be set separately in each "field" entry // regrid = { to_grid = NONE; @@ -39,7 +38,7 @@ regrid = { //////////////////////////////////////////////////////////////////////////////// // -// May be set separately in each "field" entry +// May be set separately in each "ens.field" entry // censor_thresh = []; censor_val = []; @@ -56,9 +55,9 @@ ens = { field = [ { name = "PRMSL"; - level = "L0"; - lead_time = "06"; - GRIB_ens = "MET_ENS_MEMBER_ID"; + level = "L0"; + lead_time = "06"; + GRIB_ens = "MET_ENS_MEMBER_ID"; } ]; } @@ -99,17 +98,31 @@ nmep_smooth = { //////////////////////////////////////////////////////////////////////////////// +// +// May be set separately in each "fcst.field" and "obs.field" entry +// +prob_cat_thresh = []; + +// +// May be set separately in each "fcst.field" entry +// +prob_pct_thresh = [ ==0.25 ]; + +// +// May be set separately in each "obs.field" entry +// +eclv_points = 0.05; + // // Forecast and observation fields to be verified // fcst = { - field = [ { name = "PRMSL"; - level = "L0"; - lead_time = "06"; - GRIB_ens = "MET_ENS_MEMBER_ID"; + level = "L0"; + lead_time = "06"; + GRIB_ens = "MET_ENS_MEMBER_ID"; } ]; } @@ -118,9 +131,9 @@ obs = { field = [ { name = "PRMSL"; - level = "L0"; - lead_time = "06"; - GRIB_ens = "hi_res_ctl"; + level = "L0"; + lead_time = "06"; + GRIB_ens = "hi_res_ctl"; } ]; } @@ -130,16 +143,16 @@ obs = { // Point observation filtering options // May be set separately in each "obs.field" entry // -message_type = [ "ADPUPA" ]; -sid_inc = []; -sid_exc = []; -obs_thresh = [ NA ]; -obs_quality_inc = []; -obs_quality_exc = []; -duplicate_flag = NONE; -obs_summary = NONE; -obs_perc_value = 50; -skip_const = FALSE; +message_type = [ "ADPUPA" ]; +sid_inc = []; +sid_exc = []; +obs_thresh = [ NA ]; +obs_quality_inc = []; +obs_quality_exc = []; +duplicate_flag = NONE; +obs_summary = NONE; +obs_perc_value = 50; +skip_const = FALSE; // // Observation error options @@ -173,12 +186,6 @@ message_type_group_map = [ ens_ssvar_bin_size = 1.0; ens_phist_bin_size = 0.05; -// -// Categorical thresholds to define ensemble probabilities -// May be set separately in each "fcst.field" entry -// -prob_cat_thresh = []; - //////////////////////////////////////////////////////////////////////////////// // @@ -212,6 +219,8 @@ climo_stdev = { climo_cdf = { cdf_bins = 10; center_bins = FALSE; + write_bins = TRUE; + direct_prob = FALSE; } //////////////////////////////////////////////////////////////////////////////// @@ -251,7 +260,7 @@ ci_alpha = [ 0.05 ]; interp = { field = BOTH; vld_thresh = 1.0; - shape = SQUARE; + shape = SQUARE; type = [ { @@ -274,6 +283,11 @@ output_flag = { orank = NONE; ssvar = NONE; relp = NONE; + pct = NONE; + pstd = NONE; + pjc = NONE; + prc = NONE; + eclv = NONE; } //////////////////////////////////////////////////////////////////////////////// diff --git a/test/config/EnsembleStatConfig_single_file_nc b/test/config/EnsembleStatConfig_single_file_nc index 67bd0e3399..5f29e6c671 100644 --- a/test/config/EnsembleStatConfig_single_file_nc +++ b/test/config/EnsembleStatConfig_single_file_nc @@ -26,7 +26,6 @@ obtype = "ANALYS"; // // Verification grid -// May be set separately in each "field" entry // regrid = { to_grid = NONE; @@ -39,7 +38,7 @@ regrid = { //////////////////////////////////////////////////////////////////////////////// // -// May be set separately in each "field" entry +// May be set separately in each "ens.field" entry // censor_thresh = []; censor_val = []; @@ -106,6 +105,21 @@ nmep_smooth = { //////////////////////////////////////////////////////////////////////////////// +// +// May be set separately in each "fcst.field" and "obs.field" entry +// +prob_cat_thresh = []; + +// +// May be set separately in each "fcst.field" entry +// +prob_pct_thresh = [ ==0.25 ]; + +// +// May be set separately in each "obs.field" entry +// +eclv_points = 0.05; + // // Forecast and observation fields to be verified // @@ -126,22 +140,23 @@ obs = { } ]; } + //////////////////////////////////////////////////////////////////////////////// // // Point observation filtering options // May be set separately in each "obs.field" entry // -message_type = [ "ADPUPA" ]; -sid_inc = []; -sid_exc = []; -obs_thresh = [ NA ]; -obs_quality_inc = []; -obs_quality_exc = []; -duplicate_flag = NONE; -obs_summary = NONE; -obs_perc_value = 50; -skip_const = FALSE; +message_type = [ "ADPUPA" ]; +sid_inc = []; +sid_exc = []; +obs_thresh = [ NA ]; +obs_quality_inc = []; +obs_quality_exc = []; +duplicate_flag = NONE; +obs_summary = NONE; +obs_perc_value = 50; +skip_const = FALSE; // // Observation error options @@ -175,12 +190,6 @@ message_type_group_map = [ ens_ssvar_bin_size = 1.0; ens_phist_bin_size = 0.05; -// -// Categorical thresholds to define ensemble probabilities -// May be set separately in each "fcst.field" entry -// -prob_cat_thresh = []; - //////////////////////////////////////////////////////////////////////////////// // @@ -214,6 +223,8 @@ climo_stdev = { climo_cdf = { cdf_bins = 10; center_bins = FALSE; + write_bins = TRUE; + direct_prob = FALSE; } //////////////////////////////////////////////////////////////////////////////// @@ -253,7 +264,7 @@ ci_alpha = [ 0.05 ]; interp = { field = BOTH; vld_thresh = 1.0; - shape = SQUARE; + shape = SQUARE; type = [ { @@ -276,6 +287,11 @@ output_flag = { orank = NONE; ssvar = NONE; relp = NONE; + pct = NONE; + pstd = NONE; + pjc = NONE; + prc = NONE; + eclv = NONE; } //////////////////////////////////////////////////////////////////////////////// From 210a24069fec6f152121bd25f5664381e6209c8c Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Tue, 1 Mar 2022 13:12:43 -0700 Subject: [PATCH 12/16] Per #1259, update unit_met_test_scripts.xml to accurately list the output files created by ensemble-stat. --- test/xml/unit_met_test_scripts.xml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/test/xml/unit_met_test_scripts.xml b/test/xml/unit_met_test_scripts.xml index 80d05f9877..9d0176c561 100644 --- a/test/xml/unit_met_test_scripts.xml +++ b/test/xml/unit_met_test_scripts.xml @@ -461,10 +461,18 @@ -v 2 - &OUTPUT_DIR;/ensemble_stat/ensemble_stat_20100101_120000V_orank.txt &OUTPUT_DIR;/ensemble_stat/ensemble_stat_20100101_120000V_ecnt.txt + &OUTPUT_DIR;/ensemble_stat/ensemble_stat_20100101_120000V_rps.txt &OUTPUT_DIR;/ensemble_stat/ensemble_stat_20100101_120000V_rhist.txt &OUTPUT_DIR;/ensemble_stat/ensemble_stat_20100101_120000V_phist.txt + &OUTPUT_DIR;/ensemble_stat/ensemble_stat_20100101_120000V_orank.txt + &OUTPUT_DIR;/ensemble_stat/ensemble_stat_20100101_120000V_ssvar.txt + &OUTPUT_DIR;/ensemble_stat/ensemble_stat_20100101_120000V_relp.txt + &OUTPUT_DIR;/ensemble_stat/ensemble_stat_20100101_120000V_pct.txt + &OUTPUT_DIR;/ensemble_stat/ensemble_stat_20100101_120000V_pstd.txt + &OUTPUT_DIR;/ensemble_stat/ensemble_stat_20100101_120000V_pjc.txt + &OUTPUT_DIR;/ensemble_stat/ensemble_stat_20100101_120000V_prc.txt + &OUTPUT_DIR;/ensemble_stat/ensemble_stat_20100101_120000V_eclv.txt &OUTPUT_DIR;/ensemble_stat/ensemble_stat_20100101_120000V.stat &OUTPUT_DIR;/ensemble_stat/ensemble_stat_20100101_120000V_ens.nc &OUTPUT_DIR;/ensemble_stat/ensemble_stat_20100101_120000V_orank.nc From 157a33a75b9a36f407ae5601ed22bb7e680d9b23 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Tue, 1 Mar 2022 13:13:41 -0700 Subject: [PATCH 13/16] Per #1259, update unit_climatology_1.0deg.xml to demonstrate the computation of binned percentile thresholds. --- test/config/EnsembleStatConfig_climo | 10 +++++----- test/xml/unit_climatology_1.0deg.xml | 5 +++++ 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/test/config/EnsembleStatConfig_climo b/test/config/EnsembleStatConfig_climo index 936dde8175..fea0dc9180 100644 --- a/test/config/EnsembleStatConfig_climo +++ b/test/config/EnsembleStatConfig_climo @@ -255,11 +255,11 @@ output_flag = { orank = BOTH; ssvar = STAT; relp = STAT; - pct = NONE; - pstd = NONE; - pjc = NONE; - prc = NONE; - eclv = NONE; + pct = BOTH; + pstd = BOTH; + pjc = BOTH; + prc = BOTH; + eclv = BOTH; } //////////////////////////////////////////////////////////////////////////////// diff --git a/test/xml/unit_climatology_1.0deg.xml b/test/xml/unit_climatology_1.0deg.xml index 6f458f13c8..a357b5c8b1 100644 --- a/test/xml/unit_climatology_1.0deg.xml +++ b/test/xml/unit_climatology_1.0deg.xml @@ -264,6 +264,11 @@ &OUTPUT_DIR;/climatology_1.0deg/ensemble_stat_NCEP_1.0DEG_20120410_120000V.stat &OUTPUT_DIR;/climatology_1.0deg/ensemble_stat_NCEP_1.0DEG_20120410_120000V_ecnt.txt &OUTPUT_DIR;/climatology_1.0deg/ensemble_stat_NCEP_1.0DEG_20120410_120000V_orank.txt + &OUTPUT_DIR;/climatology_1.0deg/ensemble_stat_NCEP_1.0DEG_20120410_120000V_pct.txt + &OUTPUT_DIR;/climatology_1.0deg/ensemble_stat_NCEP_1.0DEG_20120410_120000V_pstd.txt + &OUTPUT_DIR;/climatology_1.0deg/ensemble_stat_NCEP_1.0DEG_20120410_120000V_prc.txt + &OUTPUT_DIR;/climatology_1.0deg/ensemble_stat_NCEP_1.0DEG_20120410_120000V_pjc.txt + &OUTPUT_DIR;/climatology_1.0deg/ensemble_stat_NCEP_1.0DEG_20120410_120000V_eclv.txt &OUTPUT_DIR;/climatology_1.0deg/ensemble_stat_NCEP_1.0DEG_20120410_120000V_ens.nc &OUTPUT_DIR;/climatology_1.0deg/ensemble_stat_NCEP_1.0DEG_20120410_120000V_orank.nc From a1632653f1075ca71b35996fb419970dd43c6a28 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Tue, 1 Mar 2022 13:16:32 -0700 Subject: [PATCH 14/16] Per #1259, update compute_pct_mean() to support a flag to incidate whether the total column should be summed or averaged. Previously, they were always summed since the climo bins were used to SUBSET the matched pairs. In Ensemble-Stat, the full set of pairs can now thresholded multiple times based on the climo bins. As such, the TOTAL value for each input should remain constant. Rather then summing those totals, they should now be averaged (but this is the average of a constant value). --- met/src/libcode/vx_statistics/compute_stats.cc | 7 ++++--- met/src/libcode/vx_statistics/compute_stats.h | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/met/src/libcode/vx_statistics/compute_stats.cc b/met/src/libcode/vx_statistics/compute_stats.cc index a95c1c9335..35705314eb 100644 --- a/met/src/libcode/vx_statistics/compute_stats.cc +++ b/met/src/libcode/vx_statistics/compute_stats.cc @@ -1249,7 +1249,7 @@ void compute_cnt_mean(const CNTInfo *cnt_info, int n, //////////////////////////////////////////////////////////////////////// void compute_pct_mean(const PCTInfo *cnt_info, int n, - PCTInfo &pct_mean) { + PCTInfo &pct_mean, bool sum_total) { int i; NumArray na; @@ -1267,9 +1267,10 @@ void compute_pct_mean(const PCTInfo *cnt_info, int n, pct_mean.allocate_n_alpha(1); pct_mean.alpha[0] = bad_data_double; - // Compute the sum of the totals + // Compute the sum or mean of the totals for(i=0,na.erase(); i Date: Tue, 1 Mar 2022 13:17:03 -0700 Subject: [PATCH 15/16] Per #1259, fix the logic for how to size the probabilistic output line types. --- .../tools/core/ensemble_stat/ensemble_stat.cc | 35 ++++++++++--------- .../ensemble_stat/ensemble_stat_conf_info.cc | 13 +++++++ .../ensemble_stat/ensemble_stat_conf_info.h | 1 - 3 files changed, 32 insertions(+), 17 deletions(-) diff --git a/met/src/tools/core/ensemble_stat/ensemble_stat.cc b/met/src/tools/core/ensemble_stat/ensemble_stat.cc index 2174ea7796..2c9e87e356 100644 --- a/met/src/tools/core/ensemble_stat/ensemble_stat.cc +++ b/met/src/tools/core/ensemble_stat/ensemble_stat.cc @@ -151,7 +151,7 @@ static void write_txt_files(const EnsembleStatVxOpt &, static void do_pct(const EnsembleStatVxOpt &, const PairDataEnsemble &); static void do_pct_cat_thresh(const EnsembleStatVxOpt &, const PairDataEnsemble &); static void do_pct_cdp_thresh(const EnsembleStatVxOpt &, const PairDataEnsemble &); -static void write_pct_info(const EnsembleStatVxOpt &, const PCTInfo *, int); +static void write_pct_info(const EnsembleStatVxOpt &, const PCTInfo *, int, bool); static void write_ens_nc(EnsVarInfo *, int, DataPlane &); static void write_ens_var_float(EnsVarInfo *, float *, const DataPlane &, @@ -2168,6 +2168,7 @@ void setup_txt_files() { max_col = max(max_col, n_ssvar_columns); max_col = max(max_col, get_n_relp_columns(max_n_ens)); max_col = max(max_col, get_n_pjc_columns(n_prob)); + max_col = max(max_col, get_n_eclv_columns(n_eclv)); max_col += n_header_columns; // Initialize file stream @@ -2509,19 +2510,11 @@ void do_pct(const EnsembleStatVxOpt &vx_opt, // If forecast probability thresholds were specified, use them. if(vx_opt.fcat_ta.n() > 0) { - mlog << Debug(2) - << "Computing Probabilistic Statistics for " - << vx_opt.fcat_ta.n() - << " categorical thresholds.\n"; do_pct_cat_thresh(vx_opt, pd_ens); } // Otherwise, if climo data is available and bins were requested, // use climo_cdf thresholds instead. else if(have_climo && vx_opt.cdf_info.cdf_ta.n() > 0) { - mlog << Debug(2) - << "Computing Probabilistic Statistics for " - << vx_opt.cdf_info.cdf_ta.n() - << " climatological distribution thresholds.\n"; do_pct_cdp_thresh(vx_opt, pd_ens); } @@ -2539,6 +2532,10 @@ void do_pct_cat_thresh(const EnsembleStatVxOpt &vx_opt, PairDataPoint pd_pnt, pd; ConcatString fcst_var_cs, cs; + mlog << Debug(2) + << "Computing Probabilistic Statistics for " + << vx_opt.fcat_ta.n() << " categorical thresholds.\n"; + // Derive a PairDataPoint object from the PairDataEnsemble input pd_pnt.extend(pd_ens.n_obs); @@ -2580,7 +2577,9 @@ void do_pct_cat_thresh(const EnsembleStatVxOpt &vx_opt, for(i_ens=0; i_ens 1) { PCTInfo pct_mean; - compute_pct_mean(pct_info, n_bin, pct_mean); + compute_pct_mean(pct_info, n_bin, pct_mean, sum_total); // Write out PSTD if(vx_opt.output_flag[i_pstd] != STATOutputType_None) { diff --git a/met/src/tools/core/ensemble_stat/ensemble_stat_conf_info.cc b/met/src/tools/core/ensemble_stat/ensemble_stat_conf_info.cc index cfde7621a0..9503bd6dc3 100644 --- a/met/src/tools/core/ensemble_stat/ensemble_stat_conf_info.cc +++ b/met/src/tools/core/ensemble_stat/ensemble_stat_conf_info.cc @@ -1254,6 +1254,10 @@ int EnsembleStatVxOpt::n_txt_row(int i_txt_row) const { // Grid Vx: Masks * Interpolations * Categorical Thresholds n = (get_n_msg_typ() + 1) * get_n_mask() * get_n_interp() * get_n_prob_cat_thresh(); + if(i_txt_row==i_pct) { + cout << "JHG: for i_pct, n_row = " << n + << ", get_n_prob_cat_thresh() = " << get_n_prob_cat_thresh() << "\n"; + } break; case(i_pstd): @@ -1284,6 +1288,15 @@ int EnsembleStatVxOpt::n_txt_row(int i_txt_row) const { return(n); } +//////////////////////////////////////////////////////////////////////// + +int EnsembleStatVxOpt::get_n_prob_cat_thresh() const { + + // Probability categories can be defined by the prob_cat_thresh or + // climo_cdf.bins configuration file options. + return(max(fcat_ta.n(), cdf_info.cdf_ta.n())); +} + //////////////////////////////////////////////////////////////////////// // // Code for struct EnsembleStatNcOutInfo diff --git a/met/src/tools/core/ensemble_stat/ensemble_stat_conf_info.h b/met/src/tools/core/ensemble_stat/ensemble_stat_conf_info.h index 73c3a60d95..e2d823a504 100644 --- a/met/src/tools/core/ensemble_stat/ensemble_stat_conf_info.h +++ b/met/src/tools/core/ensemble_stat/ensemble_stat_conf_info.h @@ -182,7 +182,6 @@ inline int EnsembleStatVxOpt::get_n_mask() const { return(mask_name.n()); inline int EnsembleStatVxOpt::get_n_mask_area() const { return(mask_name_area.n()); } inline int EnsembleStatVxOpt::get_n_obs_thresh() const { return(othr_ta.n()); } -inline int EnsembleStatVxOpt::get_n_prob_cat_thresh() const { return(fcat_ta.n()); } inline int EnsembleStatVxOpt::get_n_prob_pct_thresh() const { return(fpct_ta.n()); } inline int EnsembleStatVxOpt::get_n_eclv_points() const { return(eclv_points.n()); } From 5f68b2336e81146f80c0efa19c4ff993dfdd4dfb Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Tue, 1 Mar 2022 13:17:47 -0700 Subject: [PATCH 16/16] Per #1259, remove debug print statement. --- met/src/tools/core/ensemble_stat/ensemble_stat_conf_info.cc | 4 ---- 1 file changed, 4 deletions(-) diff --git a/met/src/tools/core/ensemble_stat/ensemble_stat_conf_info.cc b/met/src/tools/core/ensemble_stat/ensemble_stat_conf_info.cc index 9503bd6dc3..5c66524da2 100644 --- a/met/src/tools/core/ensemble_stat/ensemble_stat_conf_info.cc +++ b/met/src/tools/core/ensemble_stat/ensemble_stat_conf_info.cc @@ -1254,10 +1254,6 @@ int EnsembleStatVxOpt::n_txt_row(int i_txt_row) const { // Grid Vx: Masks * Interpolations * Categorical Thresholds n = (get_n_msg_typ() + 1) * get_n_mask() * get_n_interp() * get_n_prob_cat_thresh(); - if(i_txt_row==i_pct) { - cout << "JHG: for i_pct, n_row = " << n - << ", get_n_prob_cat_thresh() = " << get_n_prob_cat_thresh() << "\n"; - } break; case(i_pstd):