diff --git a/met/data/config/EnsembleStatConfig_default b/met/data/config/EnsembleStatConfig_default index 339ab939be..1cb84c3714 100644 --- a/met/data/config/EnsembleStatConfig_default +++ b/met/data/config/EnsembleStatConfig_default @@ -26,7 +26,6 @@ obtype = "ANALYS"; // // Verification grid -// May be set separately in each "field" entry // regrid = { to_grid = NONE; @@ -39,7 +38,7 @@ regrid = { //////////////////////////////////////////////////////////////////////////////// // -// May be set separately in each "field" entry +// May be set separately in each "ens.field" entry // censor_thresh = []; censor_val = []; @@ -98,6 +97,21 @@ nmep_smooth = { //////////////////////////////////////////////////////////////////////////////// +// +// May be set separately in each "fcst.field" and "obs.field" entry +// +prob_cat_thresh = []; + +// +// May be set separately in each "fcst.field" entry +// +prob_pct_thresh = [ ==0.25 ]; + +// +// May be set separately in each "obs.field" entry +// +eclv_points = 0.05; + // // Forecast and observation fields to be verified // @@ -117,16 +131,16 @@ obs = fcst; // Point observation filtering options // May be set separately in each "obs.field" entry // -message_type = [ "ADPUPA" ]; -sid_inc = []; -sid_exc = []; -obs_thresh = [ NA ]; -obs_quality_inc = []; -obs_quality_exc = []; -duplicate_flag = NONE; -obs_summary = NONE; -obs_perc_value = 50; -skip_const = FALSE; +message_type = [ "ADPUPA" ]; +sid_inc = []; +sid_exc = []; +obs_thresh = [ NA ]; +obs_quality_inc = []; +obs_quality_exc = []; +duplicate_flag = NONE; +obs_summary = NONE; +obs_perc_value = 50; +skip_const = FALSE; // // Observation error options @@ -160,12 +174,6 @@ message_type_group_map = [ ens_ssvar_bin_size = 1.0; ens_phist_bin_size = 0.05; -// -// Categorical thresholds to define ensemble probabilities -// May be set separately in each "fcst.field" entry -// -prob_cat_thresh = []; - //////////////////////////////////////////////////////////////////////////////// // @@ -199,6 +207,8 @@ climo_stdev = { climo_cdf = { cdf_bins = 10; center_bins = FALSE; + write_bins = TRUE; + direct_prob = FALSE; } //////////////////////////////////////////////////////////////////////////////// @@ -238,7 +248,7 @@ ci_alpha = [ 0.05 ]; interp = { field = BOTH; vld_thresh = 1.0; - shape = SQUARE; + shape = SQUARE; type = [ { @@ -261,6 +271,11 @@ output_flag = { orank = NONE; ssvar = NONE; relp = NONE; + pct = NONE; + pstd = NONE; + pjc = NONE; + prc = NONE; + eclv = NONE; } //////////////////////////////////////////////////////////////////////////////// diff --git a/met/docs/Users_Guide/ensemble-stat.rst b/met/docs/Users_Guide/ensemble-stat.rst index f063e23f52..3ca0deb588 100644 --- a/met/docs/Users_Guide/ensemble-stat.rst +++ b/met/docs/Users_Guide/ensemble-stat.rst @@ -19,7 +19,7 @@ Ensemble forecasts are often created as a set of deterministic forecasts. The en Typically an ensemble is constructed by selecting a single forecast value from each member for each observation. When the High Resolution Assessment (HiRA) interpolation method is chosen, all of the nearby neighborhood points surrounding each observation from each member are used. Therefore, processing an N-member ensemble using a HiRA neighborhood of size M produces ensemble output with size N*M. This approach fully leverages information from all nearby grid points to evaluate the ensemble quality. -The ensemble relative frequency is the simplest method for turning a set of deterministic forecasts into something resembling a probability forecast. MET will create the ensemble relative frequency as the proportion of ensemble members forecasting some event. For example, if 5 out of 10 ensemble members predict measurable precipitation at a grid location, then the ensemble relative frequency of precipitation will be :math:`5/10=0.5`. If the ensemble relative frequency is calibrated (unlikely) then this could be thought of as a probability of precipitation. +The ensemble relative frequency is the simplest method for turning a set of deterministic forecasts into something resembling a probability forecast. For each categorical threshold (cat_thresh) listed for each field array entry of the ensemble dictionary (ens.field), MET will create the ensemble relative frequency as the proportion of ensemble members forecasting that event. For example, if 5 out of 10 ensemble members predict measurable precipitation at a grid location, then the ensemble relative frequency of precipitation will be :math:`5/10=0.5`. If the ensemble relative frequency is calibrated (unlikely) then this could be thought of as a probability of precipitation. The neighborhood ensemble probability (NEP) and neighborhood maximum ensemble probability (NMEP) methods are described in :ref:`Schwartz and Sobash (2017) `. They are an extension of the ensemble relative frequencies described above. The NEP value is computed by averaging the relative frequency of the event within the neighborhood over all ensemble members. The NMEP value is computed as the fraction of ensemble members for which the event is occurring somewhere within the surrounding neighborhood. The NMEP output is typically smoothed using a Gaussian kernel filter. The neighborhood sizes and smoothing options can be customized in the configuration file. @@ -36,6 +36,10 @@ The relative position (RELP) is a count of the number of times each ensemble mem The ranked probability score (RPS) is included in the Ranked Probability Score (RPS) line type. It is the mean of the Brier scores computed from ensemble probabilities derived for each probability category threshold (prob_cat_thresh) specified in the configuration file. The continuous ranked probability score (CRPS) is the average the distance between the forecast (ensemble) cumulative distribution function and the observation cumulative distribution function. It is an analog of the Brier score, but for continuous forecast and observation fields. The CRPS statistic is computed using two methods: assuming a normal distribution defined by the ensemble mean and spread (:ref:`Gneiting et al., 2004 `) and using the empirical ensemble distribution (:ref:`Hersbach, 2000 `). The CRPS statistic is included in the Ensemble Continuous Statistics (ECNT) line type, along with other statistics quantifying the ensemble spread and ensemble mean skill. +The Ensemble-Stat tool can derive ensemble relative frequencies and verify them as probability forecasts all in the same run. Note however that these simple ensemble relative frequencies are not actually calibrated probability forecasts. If probabilistic line types are requested (output_flag), this logic is applied to each pair of fields listed in the forecast (fcst) and observation (obs) dictionaries of the configuration file. Each probability category threshold (prob_cat_thresh) listed for the forecast field is applied to the input ensemble members to derive a relative frequency forecast. The probability category threshold (prob_cat_thresh) parsed from the corresponding observation entry is applied to the (gridded or point) observations to determine whether or not the event actually occurred. The paired ensemble relative freqencies and observation events are used to populate an Nx2 probabilistic contingency table. The dimension of that table is determined by the probability PCT threshold (prob_pct_thresh) configuration file option parsed from the forecast dictionary. All probabilistic output types requested are derived from the this Nx2 table and written to the ascii output files. Note that the FCST_VAR name header column is automatically reset as "PROB({FCST_VAR}{THRESH})" where {FCST_VAR} is the current field being evaluated and {THRESH} is the threshold that was applied. + +Note that if no probability category thresholds (prob_cat_thresh) are defined, but climatological mean and standard deviation data is provided along with climatological bins, climatological distribution percentile thresholds are automatically derived and used to compute probabilistic outputs. + Climatology data ---------------- @@ -150,6 +154,7 @@ ____________________ ci_alpha = [ 0.05 ]; interp = { field = BOTH; vld_thresh = 1.0; shape = SQUARE; type = [ { method = NEAREST; width = 1; } ]; } + eclv_points = []; sid_inc = []; sid_exc = []; duplicate_flag = NONE; @@ -301,33 +306,42 @@ ____________________ ens_ssvar_bin_size = 1.0; ens_phist_bin_size = 0.05; - prob_cat_thresh = []; Setting up the **fcst** and **obs** dictionaries of the configuration file is described in :numref:`config_options`. The following are some special considerations for the Ensemble-Stat tool. - The **ens** and **fcst** dictionaries do not need to include the same fields. Users may specify any number of ensemble fields to be summarized, but generally there are many fewer fields with verifying observations available. The **ens** dictionary specifies the fields to be summarized while the **fcst** dictionary specifies the fields to be verified. - The **obs** dictionary looks very similar to the **fcst** dictionary. If verifying against point observations which are assigned GRIB1 codes, the observation section must be defined following GRIB1 conventions. When verifying GRIB1 forecast data, one can easily copy over the forecast settings to the observation dictionary using **obs = fcst;**. However, when verifying non-GRIB1 forecast data, users will need to specify the **fcst** and **obs** sections separately. - The **ens_ssvar_bin_size** and **ens_phist_bin_size** specify the width of the categorical bins used to accumulate frequencies for spread-skill-variance or probability integral transform statistics, respectively. +____________________ + +.. code-block:: none + + prob_cat_thresh = []; + prob_pct_thresh = []; + + +The **prob_cat_thresh** entry is an array of thresholds. It is applied both to the computation of the RPS line type as well as the when generating probabilistic output line types. Since these thresholds can change for each variable, they can be specified separately for each **fcst.field** entry. If left empty but climatological mean and standard deviation data is provided, the **climo_cdf** thresholds will be used instead. If no climatology data is provided, and the RPS output line type is requested, then the **prob_cat_thresh** array must be defined. When probabilistic output line types are requested, for each **prob_cat_thresh** threshold listed, ensemble relative frequencies are derived and verified against the point and/or gridded observations. -The **prob_cat_thresh** entry is an array of thresholds to be applied in the computation of the RPS line type. Since these thresholds can change for each variable, they can be specified separately for each **fcst.field** entry. If left empty but climatological mean and standard deviation data is provided, the **climo_cdf** thresholds will be used instead. If no climatology data is provided, and the RPS output line type is requested, then the **prob_cat_thresh** array must be defined. +The **prob_pct_thresh** entry is an array of thresholds which define the Nx2 probabilistic contingency table used to evaluate probability forecasts. It can be specified separately for each **fcst.field** entry. These thresholds must span the range [0, 1]. A shorthand notation to create equal bin widths is provided. For example, the following setting creates 4 probability bins of width 0.25 from 0 to 1. + +.. code-block:: none + + prob_cat_thresh = [ ==0.25 ]; __________________ .. code-block:: none obs_error = { - flag = FALSE; - dist_type = NONE; - dist_parm = []; - inst_bias_scale = 1.0; - inst_bias_offset = 0.0; + flag = FALSE; + dist_type = NONE; + dist_parm = []; + inst_bias_scale = 1.0; + inst_bias_offset = 0.0; } @@ -353,13 +367,18 @@ _________________ .. code-block:: none output_flag = { - ecnt = NONE; - rps = NONE; - rhist = NONE; - phist = NONE; - orank = NONE; - ssvar = NONE; - relp = NONE; + ecnt = NONE; + rps = NONE; + rhist = NONE; + phist = NONE; + orank = NONE; + ssvar = NONE; + relp = NONE; + pct = NONE; + pstd = NONE; + pjc = NONE; + prc = NONE; + eclv = NONE; } @@ -380,12 +399,22 @@ The **output_flag** array controls the type of output that is generated. Each fl 7. **RELP** for Relative Position Counts +8. **PCT** for Contingency Table counts for derived ensemble relative frequencies + +9. **PSTD** for Probabilistic statistics for dichotomous outcomes for derived ensemble relative frequencies + +10. **PJC** for Joint and Conditional factorization for derived ensemble relative frequencies + +11. **PRC** for Receiver Operating Characteristic for derived ensemble relative frequencies + +12. **ECLV** for Economic Cost/Loss Relative Value for derived ensemble relative frequencies + _____________________ .. code-block:: none - ensemble_flag = { - latlon = TRUE; + ensemble_flag = { + latlon = TRUE; mean = TRUE; stdev = TRUE; minus = TRUE; @@ -399,7 +428,7 @@ _____________________ nmep = FALSE; rank = TRUE; weight = FALSE; - } + } The **ensemble_flag** specifies which derived ensemble fields should be calculated and output. Setting the flag to TRUE produces output of the specified field, while FALSE produces no output for that field type. The flags correspond to the following output line types: diff --git a/met/scripts/config/EnsembleStatConfig b/met/scripts/config/EnsembleStatConfig index 64367110ea..9a1845555e 100644 --- a/met/scripts/config/EnsembleStatConfig +++ b/met/scripts/config/EnsembleStatConfig @@ -32,16 +32,18 @@ regrid = { method = NEAREST; width = 1; vld_thresh = 0.5; + shape = SQUARE; } //////////////////////////////////////////////////////////////////////////////// // -// May be set separately in each "field" entry +// May be set separately in each "ens.field" entry // censor_thresh = []; censor_val = []; cat_thresh = []; +nc_var_str = ""; // // Ensemble product fields to be processed @@ -80,6 +82,13 @@ ens = { ]; } +// +// IDs for ensemble members +// Only set if processing a single file with all ensembles +// +ens_member_ids = []; +control_id = ""; + //////////////////////////////////////////////////////////////////////////////// // @@ -109,6 +118,21 @@ nmep_smooth = { //////////////////////////////////////////////////////////////////////////////// +// +// May be set separately in each "fcst.field" and "obs.field" entry +// +prob_cat_thresh = []; + +// +// May be set separately in each "fcst.field" entry +// +prob_pct_thresh = [ ==0.25 ]; + +// +// May be set separately in each "obs.field" entry +// +eclv_points = 0.05; + // // Forecast and observation fields to be verified // @@ -133,15 +157,16 @@ obs = fcst; // Point observation filtering options // May be set separately in each "obs.field" entry // -sid_inc = []; -sid_exc = []; -obs_thresh = [ NA ]; -obs_quality_inc = []; -obs_quality_exc = []; -duplicate_flag = NONE; -obs_summary = NONE; -obs_perc_value = 50; -skip_const = FALSE; +message_type = []; +sid_inc = []; +sid_exc = []; +obs_thresh = [ NA ]; +obs_quality_inc = []; +obs_quality_exc = []; +duplicate_flag = NONE; +obs_summary = NONE; +obs_perc_value = 50; +skip_const = FALSE; // // Observation error options @@ -154,8 +179,20 @@ obs_error = { dist_parm = []; inst_bias_scale = 1.0; inst_bias_offset = 0.0; + min = NA; + max = NA; } +// +// Mapping of message type group name to comma-separated list of values +// +message_type_group_map = [ + { key = "SURFACE"; val = "ADPSFC,SFCSHP,MSONET"; }, + { key = "ANYAIR"; val = "AIRCAR,AIRCFT"; }, + { key = "ANYSFC"; val = "ADPSFC,SFCSHP,ADPUPA,PROFLR,MSONET"; }, + { key = "ONLYSF"; val = "ADPSFC,SFCSHP"; } +]; + // // Ensemble bin sizes // May be set separately in each "obs.field" entry @@ -163,16 +200,10 @@ obs_error = { ens_ssvar_bin_size = 1.0; ens_phist_bin_size = 0.05; -// -// Categorical thresholds to define ensemble probabilities -// May be set separately in each "fcst.field" entry -// -prob_cat_thresh = []; - //////////////////////////////////////////////////////////////////////////////// // -// Climatology mean data +// Climatology data // climo_mean = { @@ -183,6 +214,7 @@ climo_mean = { method = NEAREST; width = 1; vld_thresh = 0.5; + shape = SQUARE; } time_interp_method = DW_MEAN; @@ -201,6 +233,8 @@ climo_stdev = { climo_cdf = { cdf_bins = 10; center_bins = FALSE; + write_bins = TRUE; + direct_prob = FALSE; } //////////////////////////////////////////////////////////////////////////////// @@ -244,6 +278,7 @@ ci_alpha = [ 0.05 ]; interp = { field = BOTH; vld_thresh = 1.0; + shape = SQUARE; type = [ { @@ -270,6 +305,11 @@ output_flag = { orank = BOTH; ssvar = BOTH; relp = BOTH; + pct = BOTH; + pstd = BOTH; + pjc = BOTH; + prc = BOTH; + eclv = BOTH; } //////////////////////////////////////////////////////////////////////////////// @@ -278,6 +318,7 @@ output_flag = { // Ensemble product output types // ensemble_flag = { + latlon = TRUE; mean = TRUE; stdev = TRUE; minus = TRUE; diff --git a/met/src/basic/vx_config/config_constants.h b/met/src/basic/vx_config/config_constants.h index ae88c78d3e..24d9517542 100644 --- a/met/src/basic/vx_config/config_constants.h +++ b/met/src/basic/vx_config/config_constants.h @@ -746,6 +746,7 @@ static const char conf_key_rank_flag[] = "rank"; static const char conf_key_ssvar_bin[] = "ens_ssvar_bin_size"; static const char conf_key_phist_bin[] = "ens_phist_bin_size"; static const char conf_key_prob_cat_thresh[] = "prob_cat_thresh"; +static const char conf_key_prob_pct_thresh[] = "prob_pct_thresh"; static const char conf_key_obs_error[] = "obs_error"; static const char conf_key_dist_type[] = "dist_type"; static const char conf_key_dist_parm[] = "dist_parm"; diff --git a/met/src/basic/vx_util/thresh_array.cc b/met/src/basic/vx_util/thresh_array.cc index 1575ef1f04..dd68d0578c 100644 --- a/met/src/basic/vx_util/thresh_array.cc +++ b/met/src/basic/vx_util/thresh_array.cc @@ -771,6 +771,27 @@ ThreshArray process_rps_cdp_thresh(const ThreshArray &ta) { //////////////////////////////////////////////////////////////////////// +ThreshArray derive_cdp_thresh(const ThreshArray &ta) { + SingleThresh st; + ThreshArray ta_out; + + for(int i=0; icompute_pair_vals(conf_info.rng_ptr); - // Process each filtering threshold - for(m=0; msubset_pairs_obs_thresh(conf_info.vx_opt[i].othr_ta[m]); - - // Continue if there are no points - if(pd.n_obs == 0) continue; - - // Compute ECNT scores - if(conf_info.output_flag[i_ecnt] != STATOutputType_None) { - do_ecnt(conf_info.vx_opt[i], - conf_info.vx_opt[i].othr_ta[m], &pd); - } - - // Compute RPS scores - if(conf_info.output_flag[i_rps] != STATOutputType_None) { - do_rps(conf_info.vx_opt[i], - conf_info.vx_opt[i].othr_ta[m], &pd); - } - - // Write RHIST counts - if(conf_info.output_flag[i_rhist] != STATOutputType_None) { - - pd.compute_rhist(); - - if(pd.rhist_na.sum() > 0) { - write_rhist_row(shc, &pd, - conf_info.output_flag[i_rhist], - stat_at, i_stat_row, - txt_at[i_rhist], i_txt_row[i_rhist]); - } - } - - // Write PHIST counts if greater than 0 - if(conf_info.output_flag[i_phist] != STATOutputType_None) { - - pd.compute_phist(); - - if(pd.phist_na.sum() > 0) { - write_phist_row(shc, &pd, - conf_info.output_flag[i_phist], - stat_at, i_stat_row, - txt_at[i_phist], i_txt_row[i_phist]); - } - } - - // Write RELP counts - if(conf_info.output_flag[i_relp] != STATOutputType_None) { - - pd.compute_relp(); - - if(pd.relp_na.sum() > 0) { - write_relp_row(shc, &pd, - conf_info.output_flag[i_relp], - stat_at, i_stat_row, - txt_at[i_relp], i_txt_row[i_relp]); - } - } - - // Write SSVAR scores - if(conf_info.output_flag[i_ssvar] != STATOutputType_None) { - - pd.compute_ssvar(); - - // Make sure there are bins to process - if(pd.ssvar_bins) { - - // Add rows to the output AsciiTables for SSVAR - stat_at.add_rows(pd.ssvar_bins[0].n_bin * - conf_info.vx_opt[i].ci_alpha.n()); - - if(conf_info.output_flag[i_ssvar] == STATOutputType_Both) { - txt_at[i_ssvar].add_rows(pd.ssvar_bins[0].n_bin * - conf_info.vx_opt[i].ci_alpha.n()); - } - - // Write the SSVAR data for each alpha value - for(n=0; n 0) { - write_rhist_row(shc, &pd, - conf_info.output_flag[i_rhist], - stat_at, i_stat_row, - txt_at[i_rhist], i_txt_row[i_rhist]); - } - } - - // Write PHIST counts if greater than 0 - if(conf_info.output_flag[i_phist] != STATOutputType_None) { - - pd.phist_bin_size = conf_info.vx_opt[i].vx_pd.pd[0][0][0].phist_bin_size; - pd.compute_phist(); - - if(pd.phist_na.sum() > 0) { - write_phist_row(shc, &pd, - conf_info.output_flag[i_phist], - stat_at, i_stat_row, - txt_at[i_phist], i_txt_row[i_phist]); - } - } - - // Write RELP counts - if(conf_info.output_flag[i_relp] != STATOutputType_None) { - - pd.compute_relp(); - - if(pd.relp_na.sum() > 0) { - write_relp_row(shc, &pd, - conf_info.output_flag[i_relp], - stat_at, i_stat_row, - txt_at[i_relp], i_txt_row[i_relp]); - } - } - - // Write SSVAR scores - if(conf_info.output_flag[i_ssvar] != STATOutputType_None) { - - pd.ssvar_bin_size = conf_info.vx_opt[i].vx_pd.pd[0][0][0].ssvar_bin_size; - pd.compute_ssvar(); - - // Make sure there are bins to process - if(pd.ssvar_bins) { - - // Add rows to the output AsciiTables for SSVAR - stat_at.add_rows(pd.ssvar_bins[0].n_bin * - conf_info.vx_opt[i].ci_alpha.n()); - - if(conf_info.output_flag[i_ssvar] == STATOutputType_Both) { - txt_at[i_ssvar].add_rows(pd.ssvar_bins[0].n_bin * - conf_info.vx_opt[i].ci_alpha.n()); - } - - // Write the SSVAR data for each alpha value - for(m=0; m 0) { + write_rhist_row(shc, &pd, + vx_opt.output_flag[i_rhist], + stat_at, i_stat_row, + txt_at[i_rhist], i_txt_row[i_rhist]); + } + } + + // Write PHIST counts if greater than 0 + if(vx_opt.output_flag[i_phist] != STATOutputType_None) { + + pd.phist_bin_size = vx_opt.vx_pd.pd[0][0][0].phist_bin_size; + pd.compute_phist(); + + if(pd.phist_na.sum() > 0) { + write_phist_row(shc, &pd, + vx_opt.output_flag[i_phist], + stat_at, i_stat_row, + txt_at[i_phist], i_txt_row[i_phist]); + } + } + + // Write RELP counts + if(vx_opt.output_flag[i_relp] != STATOutputType_None) { + + pd.compute_relp(); + + if(pd.relp_na.sum() > 0) { + write_relp_row(shc, &pd, + vx_opt.output_flag[i_relp], + stat_at, i_stat_row, + txt_at[i_relp], i_txt_row[i_relp]); + } + } + + // Write SSVAR scores + if(vx_opt.output_flag[i_ssvar] != STATOutputType_None) { + + pd.ssvar_bin_size = vx_opt.vx_pd.pd[0][0][0].ssvar_bin_size; + pd.compute_ssvar(); + + // Make sure there are bins to process + if(pd.ssvar_bins) { + + // Add rows to the output AsciiTables for SSVAR + stat_at.add_rows(pd.ssvar_bins[0].n_bin * + vx_opt.ci_alpha.n()); + + if(vx_opt.output_flag[i_ssvar] == STATOutputType_Both) { + txt_at[i_ssvar].add_rows(pd.ssvar_bins[0].n_bin * + vx_opt.ci_alpha.n()); + } + + // Write the SSVAR data for each alpha value + for(j=0; j 0 && + pd_ens.csd_na.n_valid() > 0); + + // If forecast probability thresholds were specified, use them. + if(vx_opt.fcat_ta.n() > 0) { + do_pct_cat_thresh(vx_opt, pd_ens); + } + // Otherwise, if climo data is available and bins were requested, + // use climo_cdf thresholds instead. + else if(have_climo && vx_opt.cdf_info.cdf_ta.n() > 0) { + do_pct_cdp_thresh(vx_opt, pd_ens); + } + + // Otherwise, no work to be done. + return; +} + +//////////////////////////////////////////////////////////////////////// + +void do_pct_cat_thresh(const EnsembleStatVxOpt &vx_opt, + const PairDataEnsemble &pd_ens) { + int i, i_thr, i_bin, i_obs, i_ens; + int n_vld, n_evt, n_bin; + PCTInfo *pct_info = (PCTInfo *) 0; + PairDataPoint pd_pnt, pd; + ConcatString fcst_var_cs, cs; + + mlog << Debug(2) + << "Computing Probabilistic Statistics for " + << vx_opt.fcat_ta.n() << " categorical thresholds.\n"; + + // Derive a PairDataPoint object from the PairDataEnsemble input + pd_pnt.extend(pd_ens.n_obs); + + // Determine the number of climo CDF bins + n_bin = (pd_ens.cmn_na.n_valid() > 0 && pd_ens.csd_na.n_valid() > 0 ? + vx_opt.get_n_cdf_bin() : 1); + + if(n_bin > 1) { + mlog << Debug(2) + << "Subsetting pairs into " << n_bin << " climatology bins.\n"; + } + + // Allocate memory + pct_info = new PCTInfo [n_bin]; + + // Store the current fcst_var value + fcst_var_cs = shc.get_fcst_var(); + + // Process each probability threshold + for(i_thr=0; i_thr 0 || (double) (n_vld/pd_ens.n_ens) >= conf_info.vld_data_thresh) { + pd_pnt.add_grid_pair((double) n_evt/n_vld, pd_ens.o_na[i_obs], + pd_ens.cmn_na[i_obs], pd_ens.csd_na[i_obs], + pd_ens.wgt_na[i_obs]); + } + + } // end for i_obs + + // Process the climo CDF bins + for(i_bin=0; i_bin 1) pd = subset_climo_cdf_bin(pd_pnt, + vx_opt.cdf_info.cdf_ta, i_bin); + else pd = pd_pnt; + + // Store thresholds + pct_info[i_bin].fthresh = vx_opt.fpct_ta; + pct_info[i_bin].othresh = vx_opt.ocat_ta[i_thr]; + pct_info[i_bin].allocate_n_alpha(vx_opt.get_n_ci_alpha()); + + for(i=0; i 0 || (double) (n_vld/pd_ens.n_ens) >= conf_info.vld_data_thresh) { + pd_pnt.add_grid_pair((double) n_evt/n_vld, pd_ens.o_na[i_obs], + pd_ens.cmn_na[i_obs], pd_ens.csd_na[i_obs], + pd_ens.wgt_na[i_obs]); + } + + } // end for i_obs + + // Initialize + pct_info[i_bin].clear(); + + // Store thresholds + pct_info[i_bin].fthresh = vx_opt.fpct_ta; + pct_info[i_bin].othresh = cdp_thresh[i_bin]; + pct_info[i_bin].allocate_n_alpha(vx_opt.get_n_ci_alpha()); + + for(i=0; i 1) { + + PCTInfo pct_mean; + compute_pct_mean(pct_info, n_bin, pct_mean, sum_total); + + // Write out PSTD + if(vx_opt.output_flag[i_pstd] != STATOutputType_None) { + write_pstd_row(shc, pct_mean, + vx_opt.output_flag[i_pstd], + -1, n_bin, stat_at, i_stat_row, + txt_at[i_pstd], i_txt_row[i_pstd]); + } + } // end if n_bin > 1 + + return; +} + +//////////////////////////////////////////////////////////////////////// + void write_ens_nc(EnsVarInfo * ens_info, int i_var, DataPlane &ens_dp) { int i, j, k, l; double t, v; @@ -3121,14 +3405,14 @@ void clean_up() { // Deallocate threshold count arrays if(thresh_cnt_na) { - for(i=0; icat_ta.n() > max_n_thresh) { - max_n_thresh = ens_info->cat_ta.n(); + if(ens_info->cat_ta.n() > max_n_ens_thresh) { + max_n_ens_thresh = ens_info->cat_ta.n(); } } @@ -430,7 +431,7 @@ void EnsembleStatConfInfo::process_config(GrdFileType etype, } // Parse settings for each verification task - for(i=0; i " + << "The number of forecast (" << fcat_ta.n() + << ") and observation (" << ocat_ta.n() + << ") probability category thresholds in \"" + << conf_key_prob_cat_thresh << "\" must match.\n\n"; + exit(1); + } + + // Conf: prob_pct_thresh + fpct_ta = fdict.lookup_thresh_array(conf_key_prob_pct_thresh); + fpct_ta = string_to_prob_thresh(fpct_ta.get_str().c_str()); // Conf: duplicate_flag duplicate_flag = parse_conf_duplicate_flag(&odict); @@ -1107,7 +1160,12 @@ void EnsembleStatVxOpt::set_vx_pd(EnsembleStatConfInfo *conf_info, int ctrl_inde void EnsembleStatVxOpt::set_perc_thresh(const PairDataEnsemble *pd_ptr) { - if(!othr_ta.need_perc()) return; + // + // Check if percentile thresholds were requested + // + if(!othr_ta.need_perc() && + !fcat_ta.need_perc() && + !ocat_ta.need_perc()) return; // // Sort the input arrays @@ -1121,10 +1179,12 @@ void EnsembleStatVxOpt::set_perc_thresh(const PairDataEnsemble *pd_ptr) { csort.sort_array(); // - // Compute percentiles, passing the observation thresholds in for - // the fcst and obs slots. + // Compute percentiles, passing the observation filtering + // thresholds in for the fcst and obs slots. // othr_ta.set_perc(&fsort, &osort, &csort, &othr_ta, &othr_ta); + fcat_ta.set_perc(&fsort, &osort, &csort, &fcat_ta, &ocat_ta); + ocat_ta.set_perc(&fsort, &osort, &csort, &fcat_ta, &ocat_ta); return; } @@ -1154,8 +1214,7 @@ int EnsembleStatVxOpt::n_txt_row(int i_txt_row) const { // Point Vx: Message Types * Masks * Interpolations * Obs Thresholds * Alphas // Grid Vx: Masks * Interpolations * Obs Thresholds * Alphas n = (get_n_msg_typ() + 1) * get_n_mask() * get_n_interp() * - get_n_o_thresh() * get_n_ci_alpha(); - + get_n_obs_thresh() * get_n_ci_alpha(); break; case(i_rhist): @@ -1166,8 +1225,7 @@ int EnsembleStatVxOpt::n_txt_row(int i_txt_row) const { // Point Vx: Message Types * Masks * Interpolations * Obs Thresholds // Grid Vx: Masks * Interpolations * Obs Thresholds n = (get_n_msg_typ() + 1) * get_n_mask() * get_n_interp() * - get_n_o_thresh(); - + get_n_obs_thresh(); break; case(i_orank): @@ -1177,8 +1235,7 @@ int EnsembleStatVxOpt::n_txt_row(int i_txt_row) const { // Number of ORANK lines possible = // Number of pairs * Obs Thresholds - n = vx_pd.get_n_pair() * get_n_o_thresh(); - + n = vx_pd.get_n_pair() * get_n_obs_thresh(); break; case(i_ssvar): @@ -1188,6 +1245,35 @@ int EnsembleStatVxOpt::n_txt_row(int i_txt_row) const { n = 0; break; + case(i_pct): + case(i_pjc): + case(i_prc): + + // Maximum number of PCT, PJC, and PRC lines possible = + // Point Vx: Message Types * Masks * Interpolations * Categorical Thresholds + // Grid Vx: Masks * Interpolations * Categorical Thresholds + n = (get_n_msg_typ() + 1) * get_n_mask() * get_n_interp() * + get_n_prob_cat_thresh(); + break; + + case(i_pstd): + + // Maximum number of PSTD lines possible = + // Point Vx: Message Types * Masks * Interpolations * Categorical Thresholds * Alphas + // Grid Vx: Masks * Interpolations * Categorical Thresholds * Alphas + n = (get_n_msg_typ() + 1) * get_n_mask() * get_n_interp() * + get_n_prob_cat_thresh() * get_n_ci_alpha(); + break; + + case(i_eclv): + + // Maximum number of ECLV lines possible = + // Point Vx: Message Types * Masks * Interpolations * Probability Thresholds + // Grid Vx: Masks * Interpolations * Probability Thresholds + n = (get_n_msg_typ() + 1) * get_n_mask() * get_n_interp() * + get_n_prob_cat_thresh() * get_n_prob_cat_thresh(); + break; + default: mlog << Error << "\nEnsembleStatVxOpt::n_txt_row(int) -> " << "unexpected output type index value: " << i_txt_row @@ -1198,6 +1284,15 @@ int EnsembleStatVxOpt::n_txt_row(int i_txt_row) const { return(n); } +//////////////////////////////////////////////////////////////////////// + +int EnsembleStatVxOpt::get_n_prob_cat_thresh() const { + + // Probability categories can be defined by the prob_cat_thresh or + // climo_cdf.bins configuration file options. + return(max(fcat_ta.n(), cdf_info.cdf_ta.n())); +} + //////////////////////////////////////////////////////////////////////// // // Code for struct EnsembleStatNcOutInfo diff --git a/met/src/tools/core/ensemble_stat/ensemble_stat_conf_info.h b/met/src/tools/core/ensemble_stat/ensemble_stat_conf_info.h index ceaedf50d0..e2d823a504 100644 --- a/met/src/tools/core/ensemble_stat/ensemble_stat_conf_info.h +++ b/met/src/tools/core/ensemble_stat/ensemble_stat_conf_info.h @@ -34,12 +34,18 @@ static const int i_phist = 3; static const int i_orank = 4; static const int i_ssvar = 5; static const int i_relp = 6; -static const int n_txt = 7; +static const int i_pct = 7; +static const int i_pstd = 8; +static const int i_pjc = 9; +static const int i_prc = 10; +static const int i_eclv = 11; +static const int n_txt = 12; // Text file type static const STATLineType txt_file_type[n_txt] = { stat_ecnt, stat_rps, stat_rhist, stat_phist, - stat_orank, stat_ssvar, stat_relp + stat_orank, stat_ssvar, stat_relp, stat_pct, + stat_pstd, stat_pjc, stat_prc, stat_eclv }; //////////////////////////////////////////////////////////////////////// @@ -114,6 +120,8 @@ class EnsembleStatVxOpt { ThreshArray othr_ta; // Observation filtering thresholds + NumArray eclv_points; // ECLV points + ClimoCDFInfo cdf_info; // Climo CDF info NumArray ci_alpha; // Alpha value for confidence intervals @@ -122,7 +130,10 @@ class EnsembleStatVxOpt { double ssvar_bin_size; // SSVAR bin size double phist_bin_size; // PHIST bin size - ThreshArray prob_cat_ta; // Categorical thresholds for probabilities + + ThreshArray fcat_ta; // Forecast categorical probability-definition thresholds, including RPS + ThreshArray ocat_ta; // Observation categorical event-definition thresholds + ThreshArray fpct_ta; // Forecast PCT thresholds DuplicateType duplicate_flag; // Duplicate observations ObsSummary obs_summary; // Summarize observations @@ -154,20 +165,28 @@ class EnsembleStatVxOpt { int get_n_mask() const; int get_n_mask_area() const; - int get_n_o_thresh() const; - int get_n_cdf_bin() const; - int get_n_ci_alpha() const; + int get_n_obs_thresh() const; + int get_n_prob_cat_thresh() const; + int get_n_prob_pct_thresh() const; + + int get_n_eclv_points() const; + int get_n_cdf_bin() const; + int get_n_ci_alpha() const; }; //////////////////////////////////////////////////////////////////////// -inline int EnsembleStatVxOpt::get_n_msg_typ() const { return(msg_typ.n()); } -inline int EnsembleStatVxOpt::get_n_interp() const { return(interp_info.n_interp); } -inline int EnsembleStatVxOpt::get_n_mask() const { return(mask_name.n()); } -inline int EnsembleStatVxOpt::get_n_mask_area() const { return(mask_name_area.n()); } -inline int EnsembleStatVxOpt::get_n_o_thresh() const { return(othr_ta.n()); } -inline int EnsembleStatVxOpt::get_n_cdf_bin() const { return(cdf_info.n_bin); } -inline int EnsembleStatVxOpt::get_n_ci_alpha() const { return(ci_alpha.n()); } +inline int EnsembleStatVxOpt::get_n_msg_typ() const { return(msg_typ.n()); } +inline int EnsembleStatVxOpt::get_n_interp() const { return(interp_info.n_interp); } +inline int EnsembleStatVxOpt::get_n_mask() const { return(mask_name.n()); } +inline int EnsembleStatVxOpt::get_n_mask_area() const { return(mask_name_area.n()); } + +inline int EnsembleStatVxOpt::get_n_obs_thresh() const { return(othr_ta.n()); } +inline int EnsembleStatVxOpt::get_n_prob_pct_thresh() const { return(fpct_ta.n()); } + +inline int EnsembleStatVxOpt::get_n_eclv_points() const { return(eclv_points.n()); } +inline int EnsembleStatVxOpt::get_n_cdf_bin() const { return(cdf_info.n_bin); } +inline int EnsembleStatVxOpt::get_n_ci_alpha() const { return(ci_alpha.n()); } //////////////////////////////////////////////////////////////////////// @@ -178,12 +197,12 @@ class EnsembleStatConfInfo { void init_from_scratch(); // Ensemble processing - int n_ens_var; // Number of ensemble fields to be processed - int max_n_thresh; // Maximum number of ensemble thresholds - int max_hira_size; // Maximum size of a HiRA neighborhoods + int n_ens_var; // Number of ensemble fields to be processed + int max_n_ens_thresh; // Maximum number of ensemble thresholds // Ensemble verification - int n_vx; // Number of ensemble fields to be verified + int n_vx; // Number of ensemble fields to be verified + int max_hira_size; // Maximum size of a HiRA neighborhoods public: @@ -256,20 +275,23 @@ class EnsembleStatConfInfo { int n_stat_row() const; // Maximum across all verification tasks - int get_max_n_thresh() const; - int get_max_hira_size() const; + int get_max_n_ens_thresh() const; + int get_max_hira_size() const; + int get_max_n_prob_cat_thresh() const; + int get_max_n_prob_pct_thresh() const; + int get_max_n_eclv_points() const; int get_compression_level(); }; //////////////////////////////////////////////////////////////////////// -inline int EnsembleStatConfInfo::get_n_ens_var() const { return(n_ens_var); } -inline int EnsembleStatConfInfo::get_n_nbrhd() const { return(n_nbrhd); } -inline int EnsembleStatConfInfo::get_n_vx() const { return(n_vx); } -inline int EnsembleStatConfInfo::get_max_n_thresh() const { return(max_n_thresh); } -inline int EnsembleStatConfInfo::get_max_hira_size() const { return(max_hira_size); } -inline int EnsembleStatConfInfo::get_compression_level() { return(conf.nc_compression()); } +inline int EnsembleStatConfInfo::get_n_ens_var() const { return(n_ens_var); } +inline int EnsembleStatConfInfo::get_n_nbrhd() const { return(n_nbrhd); } +inline int EnsembleStatConfInfo::get_n_vx() const { return(n_vx); } +inline int EnsembleStatConfInfo::get_max_n_ens_thresh() const { return(max_n_ens_thresh); } +inline int EnsembleStatConfInfo::get_max_hira_size() const { return(max_hira_size); } +inline int EnsembleStatConfInfo::get_compression_level() { return(conf.nc_compression()); } //////////////////////////////////////////////////////////////////////// diff --git a/met/src/tools/core/point_stat/point_stat.cc b/met/src/tools/core/point_stat/point_stat.cc index 73dfa2af24..5f40c314be 100644 --- a/met/src/tools/core/point_stat/point_stat.cc +++ b/met/src/tools/core/point_stat/point_stat.cc @@ -158,7 +158,6 @@ static void do_pct (const PointStatVxOpt &, const PairDataPoint *); static void do_hira_ens ( int, const PairDataPoint *); static void do_hira_prob ( int, const PairDataPoint *); - static void finish_txt_files(); static void clean_up(); diff --git a/test/config/EnsembleStatConfig b/test/config/EnsembleStatConfig index 296555748e..1b12b3e973 100644 --- a/test/config/EnsembleStatConfig +++ b/test/config/EnsembleStatConfig @@ -28,13 +28,23 @@ obtype = "ANALYS"; // Verification grid // regrid = { - to_grid = NONE; - method = NEAREST; - width = 1; + to_grid = NONE; + method = NEAREST; + width = 1; + vld_thresh = 0.5; + shape = SQUARE; } //////////////////////////////////////////////////////////////////////////////// +// +// May be set separately in each "ens.field" entry +// +censor_thresh = []; +censor_val = []; +cat_thresh = []; +nc_var_str = ""; + // // Ensemble product fields to be processed // @@ -92,13 +102,26 @@ nmep_smooth = { //////////////////////////////////////////////////////////////////////////////// +// +// May be set separately in each "fcst.field" and "obs.field" entry +// +prob_cat_thresh = []; + +// +// May be set separately in each "fcst.field" entry +// +prob_pct_thresh = [ ==0.25 ]; + +// +// May be set separately in each "obs.field" entry +// +eclv_points = 0.05; + // // Forecast and observation fields to be verified // fcst = { message_type = [ "ADPSFC" ]; - sid_inc = []; - sid_exc = []; obs_quality_inc = []; obs_quality_exc = []; @@ -111,7 +134,7 @@ fcst = { name = "APCP"; level = "A24"; obs_thresh = [ NA, >0, >=2.54 ]; - obs_error = { + obs_error = { flag = ${OBS_ERROR_FLAG}; dist_type = NONE; } @@ -126,11 +149,16 @@ obs = fcst; // Point observation filtering options // May be set separately in each "obs.field" entry // -obs_thresh = [ NA ]; -duplicate_flag = NONE; -obs_summary = NONE; -obs_perc_value = 50; -skip_const = ${SKIP_CONST}; +message_type = []; +sid_inc = []; +sid_exc = []; +obs_thresh = [ NA ]; +obs_quality_inc = []; +obs_quality_exc = []; +duplicate_flag = NONE; +obs_summary = NONE; +obs_perc_value = 50; +skip_const = ${SKIP_CONST}; // // Observation error options @@ -154,12 +182,6 @@ obs_error = { ens_ssvar_bin_size = 1.0; ens_phist_bin_size = 0.05; -// -// Categorical thresholds to define ensemble probabilities -// May be set separately in each "fcst.field" entry -// -prob_cat_thresh = []; - //////////////////////////////////////////////////////////////////////////////// // @@ -185,12 +207,20 @@ mask = { //////////////////////////////////////////////////////////////////////////////// +// +// Confidence interval settings +// +ci_alpha = [ 0.05 ]; + +//////////////////////////////////////////////////////////////////////////////// + // // Interpolation methods // interp = { field = BOTH; vld_thresh = 1.0; + shape = SQUARE; type = [ { @@ -213,6 +243,11 @@ output_flag = { orank = BOTH; ssvar = BOTH; relp = BOTH; + pct = NONE; + pstd = NONE; + pjc = NONE; + prc = NONE; + eclv = NONE; } //////////////////////////////////////////////////////////////////////////////// @@ -221,6 +256,7 @@ output_flag = { // Ensemble product output types // ensemble_flag = { + latlon = TRUE; mean = TRUE; stdev = TRUE; minus = TRUE; diff --git a/test/config/EnsembleStatConfig_MASK_SID b/test/config/EnsembleStatConfig_MASK_SID index 764ad8280c..76e4adb43f 100644 --- a/test/config/EnsembleStatConfig_MASK_SID +++ b/test/config/EnsembleStatConfig_MASK_SID @@ -28,13 +28,23 @@ obtype = "ANALYS"; // Verification grid // regrid = { - to_grid = NONE; - method = NEAREST; - width = 1; + to_grid = NONE; + method = NEAREST; + width = 1; + vld_thresh = 0.5; + shape = SQUARE; } //////////////////////////////////////////////////////////////////////////////// +// +// May be set separately in each "ens.field" entry +// +censor_thresh = []; +censor_val = []; +cat_thresh = []; +nc_var_str = ""; + // // Ensemble product fields to be processed // @@ -89,15 +99,26 @@ nmep_smooth = { //////////////////////////////////////////////////////////////////////////////// +// +// May be set separately in each "fcst.field" and "obs.field" entry +// +prob_cat_thresh = []; + +// +// May be set separately in each "fcst.field" entry +// +prob_pct_thresh = [ ==0.25 ]; + +// +// May be set separately in each "obs.field" entry +// +eclv_points = 0.05; + // // Forecast and observation fields to be verified // fcst = { message_type = [ "ADPSFC" ]; - sid_inc = []; - sid_exc = []; - obs_quality_inc = []; - obs_quality_exc = []; field = [ { @@ -116,11 +137,16 @@ obs = fcst; // Point observation filtering options // May be set separately in each "obs.field" entry // -obs_thresh = [ NA ]; -duplicate_flag = NONE; -obs_summary = NONE; -obs_perc_value = 50; -skip_const = FALSE; +message_type = []; +sid_inc = []; +sid_exc = []; +obs_thresh = [ NA ]; +obs_quality_inc = []; +obs_quality_exc = []; +duplicate_flag = NONE; +obs_summary = NONE; +obs_perc_value = 50; +skip_const = FALSE; // // Observation error options @@ -144,12 +170,6 @@ obs_error = { ens_ssvar_bin_size = 1.0; ens_phist_bin_size = 0.05; -// -// Categorical thresholds to define ensemble probabilities -// May be set separately in each "fcst.field" entry -// -prob_cat_thresh = []; - //////////////////////////////////////////////////////////////////////////////// // @@ -176,12 +196,20 @@ mask = { //////////////////////////////////////////////////////////////////////////////// +// +// Confidence interval settings +// +ci_alpha = [ 0.05 ]; + +//////////////////////////////////////////////////////////////////////////////// + // // Interpolation methods // interp = { field = BOTH; vld_thresh = 1.0; + shape = SQUARE; type = [ { @@ -204,6 +232,11 @@ output_flag = { orank = BOTH; ssvar = STAT; relp = STAT; + pct = NONE; + pstd = NONE; + pjc = NONE; + prc = NONE; + eclv = NONE; } //////////////////////////////////////////////////////////////////////////////// @@ -212,6 +245,7 @@ output_flag = { // Ensemble product output types // ensemble_flag = { + latlon = TRUE; mean = TRUE; stdev = TRUE; minus = FALSE; diff --git a/test/config/EnsembleStatConfig_climo b/test/config/EnsembleStatConfig_climo index 478ceb1552..fea0dc9180 100644 --- a/test/config/EnsembleStatConfig_climo +++ b/test/config/EnsembleStatConfig_climo @@ -28,13 +28,23 @@ obtype = "ANALYS"; // Verification grid // regrid = { - to_grid = NONE; - method = NEAREST; - width = 1; + to_grid = NONE; + method = NEAREST; + width = 1; + vld_thresh = 0.5; + shape = SQUARE; } //////////////////////////////////////////////////////////////////////////////// +// +// May be set separately in each "ens.field" entry +// +censor_thresh = []; +censor_val = []; +cat_thresh = []; +nc_var_str = ""; + // // Ensemble product fields to be processed // @@ -84,6 +94,21 @@ nmep_smooth = { //////////////////////////////////////////////////////////////////////////////// +// +// May be set separately in each "fcst.field" and "obs.field" entry +// +prob_cat_thresh = []; + +// +// May be set separately in each "fcst.field" entry +// +prob_pct_thresh = [ ==0.25 ]; + +// +// May be set separately in each "obs.field" entry +// +eclv_points = 0.05; + // // Forecast and observation fields to be verified // @@ -97,6 +122,45 @@ obs = fcst; //////////////////////////////////////////////////////////////////////////////// +// +// Point observation filtering options +// May be set separately in each "obs.field" entry +// +message_type = []; +sid_inc = []; +sid_exc = []; +obs_thresh = [ NA ]; +obs_quality_inc = []; +obs_quality_exc = []; +duplicate_flag = NONE; +obs_summary = NONE; +obs_perc_value = 50; +skip_const = FALSE; + +// +// Observation error options +// Set dist_type to NONE to use the observation error table instead +// May be set separately in each "obs.field" entry +// +obs_error = { + flag = FALSE; + dist_type = NONE; + dist_parm = []; + inst_bias_scale = 1.0; + inst_bias_offset = 0.0; + min = NA; + max = NA; +} + +// +// Ensemble bin sizes +// May be set separately in each "obs.field" entry +// +ens_ssvar_bin_size = 1.0; +ens_phist_bin_size = 0.05; + +//////////////////////////////////////////////////////////////////////////////// + // // Climatology data // @@ -126,52 +190,12 @@ climo_stdev = { climo_cdf = { cdf_bins = 10; center_bins = FALSE; + write_bins = TRUE; + direct_prob = FALSE; } //////////////////////////////////////////////////////////////////////////////// -// -// Point observation filtering options -// May be set separately in each "obs.field" entry -// -obs_thresh = [ NA ]; -obs_quality_inc = []; -obs_quality_exc = []; -duplicate_flag = NONE; -obs_summary = NONE; -obs_perc_value = 50; -skip_const = FALSE; - -// -// Observation error options -// Set dist_type to NONE to use the observation error table instead -// May be set separately in each "obs.field" entry -// -obs_error = { - flag = FALSE; - dist_type = NONE; - dist_parm = []; - inst_bias_scale = 1.0; - inst_bias_offset = 0.0; - min = NA; - max = NA; -} - -// -// Ensemble bin sizes -// May be set separately in each "obs.field" entry -// -ens_ssvar_bin_size = 1.0; -ens_phist_bin_size = 0.05; - -// -// Categorical thresholds to define ensemble probabilities -// May be set separately in each "fcst.field" entry -// -prob_cat_thresh = []; - -//////////////////////////////////////////////////////////////////////////////// - // // Point observation time window // @@ -195,12 +219,20 @@ mask = { //////////////////////////////////////////////////////////////////////////////// +// +// Confidence interval settings +// +ci_alpha = [ 0.05 ]; + +//////////////////////////////////////////////////////////////////////////////// + // // Interpolation methods // interp = { field = BOTH; vld_thresh = 1.0; + shape = SQUARE; type = [ { @@ -223,6 +255,11 @@ output_flag = { orank = BOTH; ssvar = STAT; relp = STAT; + pct = BOTH; + pstd = BOTH; + pjc = BOTH; + prc = BOTH; + eclv = BOTH; } //////////////////////////////////////////////////////////////////////////////// diff --git a/test/config/EnsembleStatConfig_grid_weight b/test/config/EnsembleStatConfig_grid_weight index dbd72bc580..19325f310a 100644 --- a/test/config/EnsembleStatConfig_grid_weight +++ b/test/config/EnsembleStatConfig_grid_weight @@ -32,10 +32,19 @@ regrid = { method = NEAREST; width = 1; vld_thresh = 0.5; + shape = SQUARE; } //////////////////////////////////////////////////////////////////////////////// +// +// May be set separately in each "ens.field" entry +// +censor_thresh = []; +censor_val = []; +cat_thresh = []; +nc_var_str = ""; + tmp_field = [ { name = "TMP"; level = [ "Z2" ]; } ]; // @@ -83,6 +92,21 @@ nmep_smooth = { //////////////////////////////////////////////////////////////////////////////// +// +// May be set separately in each "fcst.field" and "obs.field" entry +// +prob_cat_thresh = []; + +// +// May be set separately in each "fcst.field" entry +// +prob_pct_thresh = [ ==0.25 ]; + +// +// May be set separately in each "obs.field" entry +// +eclv_points = 0.05; + // // Forecast and observation fields to be verified // @@ -102,11 +126,16 @@ obs = fcst; // Point observation filtering options // May be set separately in each "obs.field" entry // -obs_thresh = [ NA ]; -duplicate_flag = NONE; -obs_summary = NONE; -obs_perc_value = 50; -skip_const = FALSE; +message_type = []; +sid_inc = []; +sid_exc = []; +obs_thresh = [ NA ]; +obs_quality_inc = []; +obs_quality_exc = []; +duplicate_flag = NONE; +obs_summary = NONE; +obs_perc_value = 50; +skip_const = FALSE; // // Observation error options @@ -130,16 +159,10 @@ obs_error = { ens_ssvar_bin_size = 1.0; ens_phist_bin_size = 0.05; -// -// Categorical thresholds to define ensemble probabilities -// May be set separately in each "fcst.field" entry -// -prob_cat_thresh = []; - //////////////////////////////////////////////////////////////////////////////// // -// Climatology mean data +// Climatology data // climo_mean = { @@ -194,6 +217,7 @@ ci_alpha = [ 0.05 ]; interp = { field = BOTH; vld_thresh = 1.0; + shape = SQUARE; type = [ { @@ -216,6 +240,11 @@ output_flag = { orank = NONE; ssvar = STAT; relp = STAT; + pct = NONE; + pstd = NONE; + pjc = NONE; + prc = NONE; + eclv = NONE; } //////////////////////////////////////////////////////////////////////////////// @@ -224,6 +253,7 @@ output_flag = { // Ensemble product output types // ensemble_flag = { + latlon = TRUE; mean = TRUE; stdev = FALSE; minus = FALSE; diff --git a/test/config/EnsembleStatConfig_one_cdf_bin b/test/config/EnsembleStatConfig_one_cdf_bin index 2a02b9475e..1197e77517 100644 --- a/test/config/EnsembleStatConfig_one_cdf_bin +++ b/test/config/EnsembleStatConfig_one_cdf_bin @@ -28,13 +28,23 @@ obtype = "ANALYS"; // Verification grid // regrid = { - to_grid = NONE; - method = NEAREST; - width = 1; + to_grid = NONE; + method = NEAREST; + width = 1; + vld_thresh = 0.5; + shape = SQUARE; } //////////////////////////////////////////////////////////////////////////////// +// +// May be set separately in each "ens.field" entry +// +censor_thresh = []; +censor_val = []; +cat_thresh = []; +nc_var_str = ""; + // // Ensemble product fields to be processed // @@ -85,43 +95,30 @@ nmep_smooth = { //////////////////////////////////////////////////////////////////////////////// // -// Forecast and observation fields to be verified +// May be set separately in each "fcst.field" and "obs.field" entry // -fcst = { - field = [ - { name = "TMP"; level = "Z2"; message_type = [ "ADPSFC" ]; }, - { name = "TMP"; level = "P850"; message_type = [ "ADPUPA" ]; } - ]; -} -obs = fcst; - -//////////////////////////////////////////////////////////////////////////////// +prob_cat_thresh = []; // -// Climatology data +// May be set separately in each "fcst.field" entry // -climo_mean = fcst; -climo_mean = { - - file_name = [ ${CLIMO_MEAN_FILE_LIST} ]; - regrid = { - method = BILIN; - width = 2; - vld_thresh = 0.5; - } - - time_interp_method = DW_MEAN; - day_interval = 1; - hour_interval = 6; -} +prob_pct_thresh = [ ==0.25 ]; // // May be set separately in each "obs.field" entry // -climo_cdf = { - cdf_bins = 1; - center_bins = FALSE; +eclv_points = 0.05; + +// +// Forecast and observation fields to be verified +// +fcst = { + field = [ + { name = "TMP"; level = "Z2"; message_type = [ "ADPSFC" ]; }, + { name = "TMP"; level = "P850"; message_type = [ "ADPUPA" ]; } + ]; } +obs = fcst; //////////////////////////////////////////////////////////////////////////////// @@ -129,13 +126,16 @@ climo_cdf = { // Point observation filtering options // May be set separately in each "obs.field" entry // -obs_thresh = [ NA ]; -obs_quality_inc = []; -obs_quality_exc = []; -duplicate_flag = NONE; -obs_summary = NONE; -obs_perc_value = 50; -skip_const = FALSE; +message_type = []; +sid_inc = []; +sid_exc = []; +obs_thresh = [ NA ]; +obs_quality_inc = []; +obs_quality_exc = []; +duplicate_flag = NONE; +obs_summary = NONE; +obs_perc_value = 50; +skip_const = FALSE; // // Observation error options @@ -159,11 +159,34 @@ obs_error = { ens_ssvar_bin_size = 1.0; ens_phist_bin_size = 0.05; +//////////////////////////////////////////////////////////////////////////////// + // -// Categorical thresholds to define ensemble probabilities -// May be set separately in each "fcst.field" entry +// Climatology data // -prob_cat_thresh = []; +climo_mean = fcst; +climo_mean = { + + file_name = [ ${CLIMO_MEAN_FILE_LIST} ]; + regrid = { + method = BILIN; + width = 2; + vld_thresh = 0.5; + shape = SQUARE; + } + + time_interp_method = DW_MEAN; + day_interval = 1; + hour_interval = 6; +} + +// +// May be set separately in each "obs.field" entry +// +climo_cdf = { + cdf_bins = 1; + center_bins = FALSE; +} //////////////////////////////////////////////////////////////////////////////// @@ -189,12 +212,20 @@ mask = { //////////////////////////////////////////////////////////////////////////////// +// +// Confidence interval settings +// +ci_alpha = [ 0.05 ]; + +//////////////////////////////////////////////////////////////////////////////// + // // Interpolation methods // interp = { field = BOTH; vld_thresh = 1.0; + shape = SQUARE; type = [ { @@ -217,6 +248,11 @@ output_flag = { orank = NONE; ssvar = NONE; relp = NONE; + pct = NONE; + pstd = NONE; + pjc = NONE; + prc = NONE; + eclv = NONE; } //////////////////////////////////////////////////////////////////////////////// diff --git a/test/config/EnsembleStatConfig_python b/test/config/EnsembleStatConfig_python index 59ae439495..b8261dcde3 100644 --- a/test/config/EnsembleStatConfig_python +++ b/test/config/EnsembleStatConfig_python @@ -26,7 +26,6 @@ obtype = "ANALYS"; // // Verification grid -// May be set separately in each "field" entry // regrid = { to_grid = NONE; @@ -39,7 +38,7 @@ regrid = { //////////////////////////////////////////////////////////////////////////////// // -// May be set separately in each "field" entry +// May be set separately in each "ens.field" entry // censor_thresh = []; censor_val = []; @@ -92,6 +91,21 @@ nmep_smooth = { //////////////////////////////////////////////////////////////////////////////// +// +// May be set separately in each "fcst.field" and "obs.field" entry +// +prob_cat_thresh = []; + +// +// May be set separately in each "fcst.field" entry +// +prob_pct_thresh = [ ==0.25 ]; + +// +// May be set separately in each "obs.field" entry +// +eclv_points = 0.05; + // // Forecast and observation fields to be verified // @@ -107,16 +121,16 @@ obs = { // Point observation filtering options // May be set separately in each "obs.field" entry // -message_type = []; -sid_inc = []; -sid_exc = []; -obs_thresh = [ NA ]; -obs_quality_inc = []; -obs_quality_exc = []; -duplicate_flag = NONE; -obs_summary = NONE; -obs_perc_value = 50; -skip_const = FALSE; +message_type = []; +sid_inc = []; +sid_exc = []; +obs_thresh = [ NA ]; +obs_quality_inc = []; +obs_quality_exc = []; +duplicate_flag = NONE; +obs_summary = NONE; +obs_perc_value = 50; +skip_const = FALSE; // // Observation error options @@ -134,7 +148,7 @@ obs_error = { } // -// Mapping of message type group name to comma-separated list of values. +// Mapping of message type group name to comma-separated list of values // message_type_group_map = [ { key = "SURFACE"; val = "ADPSFC,SFCSHP,MSONET"; }, @@ -150,39 +164,6 @@ message_type_group_map = [ ens_ssvar_bin_size = 1.0; ens_phist_bin_size = 0.05; -// -// Categorical thresholds to define ensemble probabilities -// May be set separately in each "fcst.field" entry -// -prob_cat_thresh = []; - -//////////////////////////////////////////////////////////////////////////////// - -// -// Climatology data -// -climo_mean = { - - file_name = []; - field = []; - - regrid = { - method = NEAREST; - width = 1; - vld_thresh = 0.5; - shape = SQUARE; - } - - time_interp_method = DW_MEAN; - day_interval = 31; - hour_interval = 6; -} - -climo_stdev = climo_mean; -climo_stdev = { - file_name = []; -} - //////////////////////////////////////////////////////////////////////////////// // @@ -220,7 +201,7 @@ ci_alpha = [ 0.05 ]; interp = { field = BOTH; vld_thresh = 1.0; - shape = SQUARE; + shape = SQUARE; type = [ { @@ -243,6 +224,11 @@ output_flag = { orank = NONE; ssvar = NONE; relp = NONE; + pct = NONE; + pstd = NONE; + pjc = NONE; + prc = NONE; + eclv = NONE; } //////////////////////////////////////////////////////////////////////////////// diff --git a/test/config/EnsembleStatConfig_qty_inc_exc b/test/config/EnsembleStatConfig_qty_inc_exc index 620511dd4c..62b42d75c2 100644 --- a/test/config/EnsembleStatConfig_qty_inc_exc +++ b/test/config/EnsembleStatConfig_qty_inc_exc @@ -28,13 +28,23 @@ obtype = "ANALYS"; // Verification grid // regrid = { - to_grid = NONE; - method = NEAREST; - width = 1; + to_grid = NONE; + method = NEAREST; + width = 1; + vld_thresh = 0.5; + shape = SQUARE; } //////////////////////////////////////////////////////////////////////////////// +// +// May be set separately in each "ens.field" entry +// +censor_thresh = []; +censor_val = []; +cat_thresh = []; +nc_var_str = ""; + // // Ensemble product fields to be processed // @@ -87,6 +97,21 @@ nmep_smooth = { //////////////////////////////////////////////////////////////////////////////// +// +// May be set separately in each "fcst.field" and "obs.field" entry +// +prob_cat_thresh = []; + +// +// May be set separately in each "fcst.field" entry +// +prob_pct_thresh = [ ==0.25 ]; + +// +// May be set separately in each "obs.field" entry +// +eclv_points = 0.05; + // // Forecast and observation fields to be verified // @@ -129,11 +154,16 @@ obs = fcst; // Point observation filtering options // May be set separately in each "obs.field" entry // -obs_thresh = [ NA ]; -duplicate_flag = NONE; -obs_summary = NONE; -obs_perc_value = 50; -skip_const = FALSE; +message_type = []; +sid_inc = []; +sid_exc = []; +obs_thresh = [ NA ]; +obs_quality_inc = []; +obs_quality_exc = []; +duplicate_flag = NONE; +obs_summary = NONE; +obs_perc_value = 50; +skip_const = FALSE; // // Observation error options @@ -157,12 +187,6 @@ obs_error = { ens_ssvar_bin_size = 1.0; ens_phist_bin_size = 0.05; -// -// Categorical thresholds to define ensemble probabilities -// May be set separately in each "fcst.field" entry -// -prob_cat_thresh = []; - //////////////////////////////////////////////////////////////////////////////// // @@ -187,12 +211,20 @@ mask = { //////////////////////////////////////////////////////////////////////////////// +// +// Confidence interval settings +// +ci_alpha = [ 0.05 ]; + +//////////////////////////////////////////////////////////////////////////////// + // // Interpolation methods // interp = { field = BOTH; vld_thresh = 1.0; + shape = SQUARE; type = [ { @@ -215,6 +247,11 @@ output_flag = { orank = BOTH; ssvar = NONE; relp = NONE; + pct = NONE; + pstd = NONE; + pjc = NONE; + prc = NONE; + eclv = NONE; } //////////////////////////////////////////////////////////////////////////////// @@ -223,6 +260,7 @@ output_flag = { // Ensemble product output types // ensemble_flag = { + latlon = TRUE; mean = TRUE; stdev = TRUE; minus = TRUE; diff --git a/test/config/EnsembleStatConfig_single_file_grib b/test/config/EnsembleStatConfig_single_file_grib index 76bb0bf3b5..27c802e30a 100644 --- a/test/config/EnsembleStatConfig_single_file_grib +++ b/test/config/EnsembleStatConfig_single_file_grib @@ -26,7 +26,6 @@ obtype = "ANALYS"; // // Verification grid -// May be set separately in each "field" entry // regrid = { to_grid = NONE; @@ -39,7 +38,7 @@ regrid = { //////////////////////////////////////////////////////////////////////////////// // -// May be set separately in each "field" entry +// May be set separately in each "ens.field" entry // censor_thresh = []; censor_val = []; @@ -56,9 +55,9 @@ ens = { field = [ { name = "PRMSL"; - level = "L0"; - lead_time = "06"; - GRIB_ens = "MET_ENS_MEMBER_ID"; + level = "L0"; + lead_time = "06"; + GRIB_ens = "MET_ENS_MEMBER_ID"; } ]; } @@ -99,17 +98,31 @@ nmep_smooth = { //////////////////////////////////////////////////////////////////////////////// +// +// May be set separately in each "fcst.field" and "obs.field" entry +// +prob_cat_thresh = []; + +// +// May be set separately in each "fcst.field" entry +// +prob_pct_thresh = [ ==0.25 ]; + +// +// May be set separately in each "obs.field" entry +// +eclv_points = 0.05; + // // Forecast and observation fields to be verified // fcst = { - field = [ { name = "PRMSL"; - level = "L0"; - lead_time = "06"; - GRIB_ens = "MET_ENS_MEMBER_ID"; + level = "L0"; + lead_time = "06"; + GRIB_ens = "MET_ENS_MEMBER_ID"; } ]; } @@ -118,9 +131,9 @@ obs = { field = [ { name = "PRMSL"; - level = "L0"; - lead_time = "06"; - GRIB_ens = "hi_res_ctl"; + level = "L0"; + lead_time = "06"; + GRIB_ens = "hi_res_ctl"; } ]; } @@ -130,16 +143,16 @@ obs = { // Point observation filtering options // May be set separately in each "obs.field" entry // -message_type = [ "ADPUPA" ]; -sid_inc = []; -sid_exc = []; -obs_thresh = [ NA ]; -obs_quality_inc = []; -obs_quality_exc = []; -duplicate_flag = NONE; -obs_summary = NONE; -obs_perc_value = 50; -skip_const = FALSE; +message_type = [ "ADPUPA" ]; +sid_inc = []; +sid_exc = []; +obs_thresh = [ NA ]; +obs_quality_inc = []; +obs_quality_exc = []; +duplicate_flag = NONE; +obs_summary = NONE; +obs_perc_value = 50; +skip_const = FALSE; // // Observation error options @@ -173,12 +186,6 @@ message_type_group_map = [ ens_ssvar_bin_size = 1.0; ens_phist_bin_size = 0.05; -// -// Categorical thresholds to define ensemble probabilities -// May be set separately in each "fcst.field" entry -// -prob_cat_thresh = []; - //////////////////////////////////////////////////////////////////////////////// // @@ -212,6 +219,8 @@ climo_stdev = { climo_cdf = { cdf_bins = 10; center_bins = FALSE; + write_bins = TRUE; + direct_prob = FALSE; } //////////////////////////////////////////////////////////////////////////////// @@ -251,7 +260,7 @@ ci_alpha = [ 0.05 ]; interp = { field = BOTH; vld_thresh = 1.0; - shape = SQUARE; + shape = SQUARE; type = [ { @@ -274,6 +283,11 @@ output_flag = { orank = NONE; ssvar = NONE; relp = NONE; + pct = NONE; + pstd = NONE; + pjc = NONE; + prc = NONE; + eclv = NONE; } //////////////////////////////////////////////////////////////////////////////// diff --git a/test/config/EnsembleStatConfig_single_file_nc b/test/config/EnsembleStatConfig_single_file_nc index 67bd0e3399..5f29e6c671 100644 --- a/test/config/EnsembleStatConfig_single_file_nc +++ b/test/config/EnsembleStatConfig_single_file_nc @@ -26,7 +26,6 @@ obtype = "ANALYS"; // // Verification grid -// May be set separately in each "field" entry // regrid = { to_grid = NONE; @@ -39,7 +38,7 @@ regrid = { //////////////////////////////////////////////////////////////////////////////// // -// May be set separately in each "field" entry +// May be set separately in each "ens.field" entry // censor_thresh = []; censor_val = []; @@ -106,6 +105,21 @@ nmep_smooth = { //////////////////////////////////////////////////////////////////////////////// +// +// May be set separately in each "fcst.field" and "obs.field" entry +// +prob_cat_thresh = []; + +// +// May be set separately in each "fcst.field" entry +// +prob_pct_thresh = [ ==0.25 ]; + +// +// May be set separately in each "obs.field" entry +// +eclv_points = 0.05; + // // Forecast and observation fields to be verified // @@ -126,22 +140,23 @@ obs = { } ]; } + //////////////////////////////////////////////////////////////////////////////// // // Point observation filtering options // May be set separately in each "obs.field" entry // -message_type = [ "ADPUPA" ]; -sid_inc = []; -sid_exc = []; -obs_thresh = [ NA ]; -obs_quality_inc = []; -obs_quality_exc = []; -duplicate_flag = NONE; -obs_summary = NONE; -obs_perc_value = 50; -skip_const = FALSE; +message_type = [ "ADPUPA" ]; +sid_inc = []; +sid_exc = []; +obs_thresh = [ NA ]; +obs_quality_inc = []; +obs_quality_exc = []; +duplicate_flag = NONE; +obs_summary = NONE; +obs_perc_value = 50; +skip_const = FALSE; // // Observation error options @@ -175,12 +190,6 @@ message_type_group_map = [ ens_ssvar_bin_size = 1.0; ens_phist_bin_size = 0.05; -// -// Categorical thresholds to define ensemble probabilities -// May be set separately in each "fcst.field" entry -// -prob_cat_thresh = []; - //////////////////////////////////////////////////////////////////////////////// // @@ -214,6 +223,8 @@ climo_stdev = { climo_cdf = { cdf_bins = 10; center_bins = FALSE; + write_bins = TRUE; + direct_prob = FALSE; } //////////////////////////////////////////////////////////////////////////////// @@ -253,7 +264,7 @@ ci_alpha = [ 0.05 ]; interp = { field = BOTH; vld_thresh = 1.0; - shape = SQUARE; + shape = SQUARE; type = [ { @@ -276,6 +287,11 @@ output_flag = { orank = NONE; ssvar = NONE; relp = NONE; + pct = NONE; + pstd = NONE; + pjc = NONE; + prc = NONE; + eclv = NONE; } //////////////////////////////////////////////////////////////////////////////// diff --git a/test/xml/unit_climatology_1.0deg.xml b/test/xml/unit_climatology_1.0deg.xml index 6f458f13c8..a357b5c8b1 100644 --- a/test/xml/unit_climatology_1.0deg.xml +++ b/test/xml/unit_climatology_1.0deg.xml @@ -264,6 +264,11 @@ &OUTPUT_DIR;/climatology_1.0deg/ensemble_stat_NCEP_1.0DEG_20120410_120000V.stat &OUTPUT_DIR;/climatology_1.0deg/ensemble_stat_NCEP_1.0DEG_20120410_120000V_ecnt.txt &OUTPUT_DIR;/climatology_1.0deg/ensemble_stat_NCEP_1.0DEG_20120410_120000V_orank.txt + &OUTPUT_DIR;/climatology_1.0deg/ensemble_stat_NCEP_1.0DEG_20120410_120000V_pct.txt + &OUTPUT_DIR;/climatology_1.0deg/ensemble_stat_NCEP_1.0DEG_20120410_120000V_pstd.txt + &OUTPUT_DIR;/climatology_1.0deg/ensemble_stat_NCEP_1.0DEG_20120410_120000V_prc.txt + &OUTPUT_DIR;/climatology_1.0deg/ensemble_stat_NCEP_1.0DEG_20120410_120000V_pjc.txt + &OUTPUT_DIR;/climatology_1.0deg/ensemble_stat_NCEP_1.0DEG_20120410_120000V_eclv.txt &OUTPUT_DIR;/climatology_1.0deg/ensemble_stat_NCEP_1.0DEG_20120410_120000V_ens.nc &OUTPUT_DIR;/climatology_1.0deg/ensemble_stat_NCEP_1.0DEG_20120410_120000V_orank.nc diff --git a/test/xml/unit_met_test_scripts.xml b/test/xml/unit_met_test_scripts.xml index 80d05f9877..9d0176c561 100644 --- a/test/xml/unit_met_test_scripts.xml +++ b/test/xml/unit_met_test_scripts.xml @@ -461,10 +461,18 @@ -v 2 - &OUTPUT_DIR;/ensemble_stat/ensemble_stat_20100101_120000V_orank.txt &OUTPUT_DIR;/ensemble_stat/ensemble_stat_20100101_120000V_ecnt.txt + &OUTPUT_DIR;/ensemble_stat/ensemble_stat_20100101_120000V_rps.txt &OUTPUT_DIR;/ensemble_stat/ensemble_stat_20100101_120000V_rhist.txt &OUTPUT_DIR;/ensemble_stat/ensemble_stat_20100101_120000V_phist.txt + &OUTPUT_DIR;/ensemble_stat/ensemble_stat_20100101_120000V_orank.txt + &OUTPUT_DIR;/ensemble_stat/ensemble_stat_20100101_120000V_ssvar.txt + &OUTPUT_DIR;/ensemble_stat/ensemble_stat_20100101_120000V_relp.txt + &OUTPUT_DIR;/ensemble_stat/ensemble_stat_20100101_120000V_pct.txt + &OUTPUT_DIR;/ensemble_stat/ensemble_stat_20100101_120000V_pstd.txt + &OUTPUT_DIR;/ensemble_stat/ensemble_stat_20100101_120000V_pjc.txt + &OUTPUT_DIR;/ensemble_stat/ensemble_stat_20100101_120000V_prc.txt + &OUTPUT_DIR;/ensemble_stat/ensemble_stat_20100101_120000V_eclv.txt &OUTPUT_DIR;/ensemble_stat/ensemble_stat_20100101_120000V.stat &OUTPUT_DIR;/ensemble_stat/ensemble_stat_20100101_120000V_ens.nc &OUTPUT_DIR;/ensemble_stat/ensemble_stat_20100101_120000V_orank.nc