Feature 1761 percent thresh (#1956)

* Per issue #1761 in set_perc() adding code to get FBIAS numeric value, like 1.0 or 0.9, etc. SL

* Per issue #1761: in set_perc(), modified actual percentile calculation at end to use the extracted FBIAS numeric value (float). SL

* Per issue #1761: modified the check on the perc_thresh_freq_bias, just has to be > 0 now. SL

* Per issue #1761: cleaned up code in set_perc(). SL

* Per #1761, updates to Simple_Node::set_perc() to handle variable frequency bias amounts.
Changes include:
- Reverting the formatting of this back to how it originally was in the develop branch. In general, just match the formatting of the existing file, so as the minimize the number of difference lines.
- Add logic to adjust the percentile to be found based on the requested FBIAS value. Multiplying or dividing the percentile by the FBIAS value depends on the inequality type and whether we're bias adjusting the forecast or observation data.
- Adjust the log messages slightly.

Please be aware that I'm not totally confident in these changes. They warrant much more testing. This logic is very, very confusing.

* Per #1761, call compute_percentile() when double-checking the percentile values.

* Per #1761, remove unused variable.

* Per #1761, add warning for percentiles > 100.

* Per #1761. In set_perc(), after testing cleaned up code. SL

* Per issue #1761: adding new config file for testing dynamic FBIAS values. SL

* Per issue #1761: added new unit test for dynamic FBIAS values when running grid_stat. SL

* Per issue #1761, modified FBIAS section to indicated that the user can use dynamic values that are not 1.0. SL

* Update met/docs/Users_Guide/config_options.rst

Co-authored-by: johnhg <johnhg@ucar.edu>

* Update met/docs/Users_Guide/config_options.rst

Co-authored-by: johnhg <johnhg@ucar.edu>

* Update met/docs/Users_Guide/config_options.rst

Co-authored-by: johnhg <johnhg@ucar.edu>

* Update test/config/GridStatConfig_fbias_perc_thresh

Co-authored-by: johnhg <johnhg@ucar.edu>

* Update test/config/GridStatConfig_fbias_perc_thresh

Co-authored-by: johnhg <johnhg@ucar.edu>

* Update test/config/GridStatConfig_fbias_perc_thresh

Co-authored-by: johnhg <johnhg@ucar.edu>

* Update test/config/GridStatConfig_fbias_perc_thresh

Co-authored-by: johnhg <johnhg@ucar.edu>

* Per issue #1761, set nc_pairs_flag = FALSE. SL

Co-authored-by: Seth Linden <linden@kiowa.rap.ucar.edu>
Co-authored-by: John Halley Gotway <johnhg@ucar.edu>

met/docs/Users_Guide/config_options.rst

@@ -93,12 +93,14 @@ The configuration file language supports the following data types:
       e.g. "<USP90(2.5)" means less than the 90-th percentile values which
       the user has already determined to be 2.5 outside of MET.
 
-    * "==FBIAS1" to automatically de-bias the data. This option must be
-      used in conjunction with a simple threshold in the other field.
-      For example, when "obs.cat_thresh = >5.0" and
-      "fcst.cat_thresh = ==FBIAS1;", MET applies the >5.0 threshold to the
-      observations and then chooses a forecast threshold which results in a
-      frequency bias of 1.
+    * "==FBIAS" for a user-specified frequency bias value.
+      e.g. "==FBIAS1" to automatically de-bias the data, "==FBIAS0.9" to select a low-bias threshold, or "==FBIAS1.1" to select a high-bias threshold.
+      This option must be used in
+      conjunction with a simple threshold in the other field. For example,
+      when "obs.cat_thresh = >5.0" and "fcst.cat_thresh = ==FBIAS1;",
+      MET applies the >5.0 threshold to the observations and then chooses a
+      forecast threshold which results in a frequency bias of 1.
+      The frequency bias can be any float value > 0.0.
 
     * "CDP" for climatological distribution percentile thresholds.
       These thresholds require that the climatological mean and standard
@@ -119,7 +121,7 @@ The configuration file language supports the following data types:
     For example, "==CDP25" is automatically expanded to 4 percentile bins:
     >=CDP0&&<CDP25,>=CDP25&&<CDP50,>=CDP50&&<CDP75,>=CDP75&&<=CDP100
 
-  * When sample percentile thresholds of type SFP, SOP, SCP, or FBIAS1 are
+  * When sample percentile thresholds of type SFP, SOP, SCP, or FBIAS are
     requested, MET recomputes the actual percentile that the threshold
     represents. If the requested percentile and actual percentile differ by
     more than 5%, a warning message is printed. This may occur when the

met/src/basic/vx_config/config.tab.yy

@@ -1661,7 +1661,7 @@ if ( s->PT < 0 || s->PT > 100 )  {
 
 }
 
-if ( s->Ptype == perc_thresh_freq_bias && !is_eq(s->PT, 1.0) )  {
+if ( s->Ptype == perc_thresh_freq_bias && s->PT <= 0 )  {
 
    mlog << Error << "\ndo_simple_perc_thresh() -> "
         << "unsupported frequency bias percentile threshold!\n\n";

met/src/basic/vx_config/threshold.cc

@@ -934,10 +934,11 @@ void Simple_Node::set_perc(const NumArray *fptr, const NumArray *optr, const Num
 
 {
 
-int i, count;
+int i;
 double ptile, diff;
 NumArray data;
 const NumArray * ptr = 0;
+bool fbias_fcst = false;
 
    //
    //  handle sample percentile types
@@ -972,14 +973,16 @@ else if ( Ptype == perc_thresh_freq_bias )  {
         fthr->get_ptype() == no_perc_thresh_type   &&
         fthr->get_type()  != thresh_complex )  {
 
+      fbias_fcst = false;
+
       ptr = optr;
       op  = fthr->get_type();
       PT  = fptr->compute_percentile(fthr->get_value(),
                                      is_inclusive(fthr->get_type()));
 
       mlog << Debug(3)
-           << "The forecast threshold \"" << fthr->get_str()
-           << "\" includes " << PT * 100.0 << "% of the data.\n";
+           << "The forecast threshold value \"" << fthr->get_str()
+           << "\" represents the " << PT * 100.0 << "-th percentile.\n";
 
    }
 
@@ -991,14 +994,16 @@ else if ( Ptype == perc_thresh_freq_bias )  {
              othr->get_ptype() == no_perc_thresh_type   &&
              othr->get_type()  != thresh_complex )  {
 
+      fbias_fcst = true;
+
       ptr = fptr;
       op  = othr->get_type();
       PT  = optr->compute_percentile(othr->get_value(),
                                      is_inclusive(othr->get_type()));
 
       mlog << Debug(3)
-           << "The observation threshold \"" << othr->get_str()
-           << "\" includes " << PT * 100.0 << "% of the data.\n";
+           << "The observation threshold value \"" << othr->get_str()
+           << "\" represents the " << PT * 100.0 << "-th percentile.\n";
 
    }
 
@@ -1029,7 +1034,8 @@ else if ( Ptype == perc_thresh_freq_bias )  {
 
    PT *= 100.0;
 
-}
+} // end else if PT == perc_thresh_freq_bias
+
    //
    //  nothing to do
    //
@@ -1082,7 +1088,7 @@ if ( data.n() == 0 )  {
         << " threshold \"" << s
         << "\" because no valid data was provided.\n\n";
 
-      exit ( 1 );
+   exit ( 1 );
 
 }
 else  {
@@ -1094,6 +1100,64 @@ else  {
    s.strip_paren();
    abbr_s.strip_paren();
 
+   //
+   //  parse the frequency bias value from the threshold string
+   //
+
+   if ( Ptype == perc_thresh_freq_bias )  {
+
+      ConcatString fs = s;
+
+      fs.replace("==FBIAS", " ", false);
+
+      double fbias_val = atof(fs.c_str());
+
+      //
+      //  range check requested bias value
+      //
+
+      if ( fbias_val <= 0.0 )  {
+
+         mlog << Error << "\nSimple_Node::set_perc() -> "
+              << "the requested frequency bias value (" << fbias_val
+              << ") must be > 0 in threshold \"" << s << "\".\n\n";
+
+      }
+
+      //
+      //  adjust PT by the requested frequency bias amount
+      //
+
+      double PT_new;
+
+      if ( fbias_fcst )  {
+               if ( op == thresh_le || op == thresh_lt )  PT_new = PT * fbias_val;
+          else if ( op == thresh_ge || op == thresh_gt )  PT_new = PT / fbias_val;     
+      }
+      else  {
+               if ( op == thresh_le || op == thresh_lt )  PT_new = PT / fbias_val;
+          else if ( op == thresh_ge || op == thresh_gt )  PT_new = PT * fbias_val;     
+      }
+
+      if ( PT_new > 100.0 )  {
+         mlog << Warning << "\nSimple_Node::set_perc() -> "
+              << "For " << (fbias_fcst ? "forecast" : "observation" )
+              << " threshold \"" << s << "\" the required percentile of "
+              << PT_new << " exceeds the maximum possible value. "
+              << "Resetting to 100.\n\n";
+
+         PT_new = 100.0;
+      }
+
+      mlog << Debug(3)
+           << "For " << (fbias_fcst ? "forecast" : "observation" )
+           << " threshold \"" << s << "\" with type \"" << thresh_type_str[op]
+           << "\" update the requested percentile from " << PT << " to "
+           << PT_new << ".\n";
+
+      PT = PT_new;
+   }
+
    //
    //  compute the percentile and update the strings
    //
@@ -1117,27 +1181,16 @@ else  {
    //  compute the actual percentile and check tolerance
    //
 
-   if ( op == thresh_le || op == thresh_ge || op == thresh_eq )  {
-
-      for ( i=count=0; i<data.n(); i++ )  if ( data[i] <= T ) count++;
-
-   }
-   else  {
-
-      for ( i=count=0; i<data.n(); i++ )  if ( data[i] <  T ) count++;
-
-   }
-
-   ptile = (double) count / data.n();
+   ptile = data.compute_percentile(T, is_inclusive(op));
    diff  = abs(PT / 100.0 - ptile);
 
    if ( !is_eq(PT / 100.0, ptile, perc_thresh_default_tol) )  {
 
       mlog << Warning << "\nSimple_Node::set_perc() -> "
            << "the requested percentile (" << PT
-           << "%) for threshold \"" << s
+           << ") for threshold \"" << s
            << "\" differs from the actual percentile ("
-           << ptile * 100.0 << ") by " << diff * 100.0 << "%.\n"
+           << ptile * 100.0 << ") by " << diff * 100.0 << ".\n"
            << "This is common for small samples or data that contains "
            << "ties.\n\n";
 
@@ -1146,8 +1199,8 @@ else  {
 
       mlog << Debug(3)
            << "The requested percentile (" << PT
-           << "%) for threshold threshold \"" << s
-           << "\" includes " << ptile * 100.0 << "% of the data.\n";
+           << ") for threshold \"" << s << "\" includes "
+           << ptile * 100.0 << "% of the data.\n";
 
    }