From 151f480a38964b8a07985268fb00dc0619f06f65 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 19 Jun 2024 10:02:55 -0600 Subject: [PATCH] Update develop-ref after dtcenter/MET#2916 (#2919) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 2673 Moved dvariable declaration after include * #2673 Move down namespace below include * Feature #2395 wdir (#2820) * Per #2395, add new columns to VL1L2, VAL1L2, and VCNT line types for wind direction statistics. Work still in progress. * Per #2395, write the new VCNT columns to the output and document the additions to the VL1L2, VAL1L2, and VCNT columns. * Per #2395, add the definition of new statistics to Appendix G. * Per #2395, update file version history. * Per #2395, tweak warning message about zero wind vectors and update grid-stat and point-stat to log calls to the do_vl1l2() function. * Per #2395, refine the weights for wind direction stats, ignoring the undefined directions. * Update src/tools/core/stat_analysis/aggr_stat_line.cc * Update src/tools/core/stat_analysis/parse_stat_line.cc * Update src/tools/core/stat_analysis/aggr_stat_line.cc * Recent changes to branch protection rules for the develop branch have broken the logic of the update_truth.yml GHA workflow. Instead of submitting a PR to merge develop into develop-ref directly, use an intermediate update_truth_for_develop branch. * Feature #2280 ens_prob (#2823) * Per #2280, update to support probability threshold strings like ==8, where 8 is the number of ensemble members, to create probability bins centered on the n/8 for n = 0 ... 8. * Per #2280, update docs about probability threshold settings. * Per #2280, use a loose tolerance when checking for consistent bin widths. * Per #2280, add a new unit test for grid_stat to demonstrate processing the output from gen_ens_prod. * Per #2280, when verifying NMEP probability forecasts, smooth the obs data first. * Per #2280, only request STAT output for the PCT line type to match unit_grid_stat.xml and minimize the new output files. * Per #2280, update config option docs. * Per #2280, update config option docs. * #2673 Change 0 to nullptr * #2673 Change 0 to nullptr * #2673 Change 0 to nullptr * #2673 Change 0 to nullptr * #2673 Change 0 to nullptr * #2673 Removed the redundant parentheses with return * #2673 Removed the redundant parentheses with return * #2673 Removed the redundant parentheses with return * #2673 Removed the redundant parentheses with return * #2673 Removed the redundant parentheses with return * #2673 restored return statement * #2673 Added std namespace * #2673 Moved down 'using namespace' statement. Removed trailing spaces * #2673 Moved down 'using namespace' statement. * #2673 Moved down 'using namespace' statement. * #2673 Moved down 'using namespace' statement. * #2673 Moved down 'using namespace' statement. * #2673 Added std namespace * #2673 Added std namespace * #2673 Added std namespace * #2673 Changed literal 1 to boolean value, true * Feature #2673 enum_to_string (#2835) * Feature #2583 ecnt (#2825) * Unrelated to #2583, fix typo in code comments. * Per #2583, add hooks write 3 new ECNT columns for observation error data. * Per #2583, make error messages about mis-matched array lengths more informative. * Per #2583, switch to more concise variable naming conventions of ign_oerr_cnv, ign_oerr_cor, and dawid_seb. * Per #2583, fix typo to enable compilation * Per #2583, define the 5 new ECNT column names. * Per #2583, add 5 new columns to the ECNT table in the Ensemble-Stat chapter * Per #2583, update stat_columns.cc to write these 5 new ECNT columns * Per #2583, update ECNTInfo class to compute the 5 new ECNT statistics. * Per #2583, update stat-analysis to parse the 5 new ECNT columns. * Per #2583, update aggregate_stat logic for 5 new ECNT columns. * Per #2583, update PairDataEnsemble logic for 5 new ECNT columns * Per #2583, update vx_statistics library with obs_error handling logic for the 5 new ECNT columns * Per #2583, changes to make it compile * Per #2583, changes to make it compile * Per #2583, switch to a consistent ECNT column naming convention with OERR at the end. Using IGN_CONV_OERR and IGN_CORR_OERR. * Per #2583, define ObsErrorEntry::variance() with a call to the dist_var() utility function. * Per #2583, update PairDataEnsemble::compute_pair_vals() to compute the 5 new stats with the correct inputs. * Per #2583, add DEBUG(10) log messages about computing these new stats. * Per #2583, update Stat-Analysis to compute these 5 new stats from the ORANK line type. * Per #2583, whitespace and comments. * Per #2583, update the User's Guide. * Per #2583, remove the DS_ADD_OERR and DS_MULT_OERR ECNT columns and rename DS_OERR as DSS, since observation error is not actually involved in its computation. * Per #2583, minor update to Appendix C * Per #2583, rename ECNT line type statistic DSS to IDSS. * Per #2583, fix a couple of typos * Per #2583, more error checking. * Per #2583, remove the ECNT IDSS column since its just 2*pi*IGN, the existing ignorance score, and only provides meaningful information when combined with the other Dawid-Sebastiani statistics that have already been removed. * Per #2583, add Eric's documentation of these new stats to Appendix C. Along the way, update the DOI links in the references based on this APA style guide: https://apastyle.apa.org/style-grammar-guidelines/references/dois-urls#:~:text=Include%20a%20DOI%20for%20all,URL%2C%20include%20only%20the%20DOI. * Per #2583, fix new equations with embedded underscores for PDF by defining both html and pdf formatting options. * Per #2583, update the ign_conv_oerr equation to include a 2 *pi multiplier for consistency with the existing ignorance score. Also, fix the documented equations. * Per #2583, remove log file that was inadvertently added on this branch. * Per #2583, simplify ObsErrorEntry::variance() implementation. For the distribution type of NONE, return a variance of 0.0 rather than bad data, as discussed with @michelleharrold and @JeffBeck-NOAA on 3/8/2024. --------- Co-authored-by: MET Tools Test Account * Revert #2825 since more documentation and testing is needed (#2837) This reverts commit 108a8958b206d6712197823a083666ab039bf818. * Feature #2583 ecnt fix IGN_OERR_CORR (#2838) * Unrelated to #2583, fix typo in code comments. * Per #2583, add hooks write 3 new ECNT columns for observation error data. * Per #2583, make error messages about mis-matched array lengths more informative. * Per #2583, switch to more concise variable naming conventions of ign_oerr_cnv, ign_oerr_cor, and dawid_seb. * Per #2583, fix typo to enable compilation * Per #2583, define the 5 new ECNT column names. * Per #2583, add 5 new columns to the ECNT table in the Ensemble-Stat chapter * Per #2583, update stat_columns.cc to write these 5 new ECNT columns * Per #2583, update ECNTInfo class to compute the 5 new ECNT statistics. * Per #2583, update stat-analysis to parse the 5 new ECNT columns. * Per #2583, update aggregate_stat logic for 5 new ECNT columns. * Per #2583, update PairDataEnsemble logic for 5 new ECNT columns * Per #2583, update vx_statistics library with obs_error handling logic for the 5 new ECNT columns * Per #2583, changes to make it compile * Per #2583, changes to make it compile * Per #2583, switch to a consistent ECNT column naming convention with OERR at the end. Using IGN_CONV_OERR and IGN_CORR_OERR. * Per #2583, define ObsErrorEntry::variance() with a call to the dist_var() utility function. * Per #2583, update PairDataEnsemble::compute_pair_vals() to compute the 5 new stats with the correct inputs. * Per #2583, add DEBUG(10) log messages about computing these new stats. * Per #2583, update Stat-Analysis to compute these 5 new stats from the ORANK line type. * Per #2583, whitespace and comments. * Per #2583, update the User's Guide. * Per #2583, remove the DS_ADD_OERR and DS_MULT_OERR ECNT columns and rename DS_OERR as DSS, since observation error is not actually involved in its computation. * Per #2583, minor update to Appendix C * Per #2583, rename ECNT line type statistic DSS to IDSS. * Per #2583, fix a couple of typos * Per #2583, more error checking. * Per #2583, remove the ECNT IDSS column since its just 2*pi*IGN, the existing ignorance score, and only provides meaningful information when combined with the other Dawid-Sebastiani statistics that have already been removed. * Per #2583, add Eric's documentation of these new stats to Appendix C. Along the way, update the DOI links in the references based on this APA style guide: https://apastyle.apa.org/style-grammar-guidelines/references/dois-urls#:~:text=Include%20a%20DOI%20for%20all,URL%2C%20include%20only%20the%20DOI. * Per #2583, fix new equations with embedded underscores for PDF by defining both html and pdf formatting options. * Per #2583, update the ign_conv_oerr equation to include a 2 *pi multiplier for consistency with the existing ignorance score. Also, fix the documented equations. * Per #2583, remove log file that was inadvertently added on this branch. * Per #2583, simplify ObsErrorEntry::variance() implementation. For the distribution type of NONE, return a variance of 0.0 rather than bad data, as discussed with @michelleharrold and @JeffBeck-NOAA on 3/8/2024. * Per #2583, updates to ensemble-stat.rst recommended by @michelleharrold and @JeffBeck-NOAA. * Per #2583, implement changes to the IGN_CORR_OERR corrected as directed by @ericgilleland. --------- Co-authored-by: MET Tools Test Account * Update the pull request template to include a question about expected impacts to existing METplus Use Cases. * #2830 Changed enum Builtin to enum class * #2830 Converted enum to enum class at config_constants.h * Feature #2830 bootstrap enum (#2843) * Bugfix #2833 develop azimuth (#2840) * Per #2833, fix n-1 bug when defining the azimuth delta for range/azimuth grids. * Per #2833, when definng TcrmwData:range_max_km, divide by n_range - 1 since the range values start at 0. * Per #2833, remove max_range_km from the TC-RMW config file. Set the default rmw_scale to NA so that its not used by default. And update the documentation. Still actually need to make the logic of the code work as it should. * Per #2833, update tc_rmw to define the range as either a function of rmw or using explicit spacing in km. * Per #2833, update the TCRMW Config files to remove the max_range_km entry, and update the unit test for one call to use RMW ranges and the other to use ranges defined in kilometers. * Per #2833, just correct code comments. * Per #2833, divide by n - 1 when computing the range delta, rather than n. * Per #2833, correct the handling of the maximum range in the tc-rmw tool. For fixed delta km, need to define the max range when setting up the grid at the beginning. --------- Co-authored-by: MET Tools Test Account * #2830 Changed enum PadSize to enum class * #2830 Removed redundant parantheses * #2830 Removed commenyted out code * #2830 Use auto * #2830 Changed enum to enum class for DistType, InterpMthd, GridTemplates, and NormalizeType * #2830 Moved enum_class_as_integer from header file to cc files * #2830 Added enum_as_int.hpp * #2830 Added enum_as_int.hpp * Deleted enum_class_as_integer and renamed it to enum_class_as_int * Removed redundant paranthese * #2830 Changed enum to enumclass * #2830 Changed enum_class_as_integer to enum_class_as_int * Feature #2379 sonarqube gha (#2847) * Per #2379, testing initial GHA SonarQube setup. * Per #2379, switch to only analyzing the src directory. * Per #2379, move more config logic from sonar-project.properties into the workflow. #ci-skip-all * Per #2379, try removing + symbols * Per #2379, move projectKey into xml workflow and remove sonar-project.properties. * Per #2379, try following the instructions at https://github.com/sonarsource-cfamily-examples/linux-autotools-gh-actions-sq/blob/main/.github/workflows/build.yml ci-skip-all * Per #2379, see details of progress described in this issue comment: https://github.com/dtcenter/MET/issues/2379#issuecomment-2000242425 * Unrelated to #2379, just removing spurious space that gets flagged as a diff when re-running enum_to_string on seneca. * Per #2379, try running SonarQube through GitHub. * Per #2379, remove empty env section and also disable the testing workflow temporarily during sonarqube development. * Per #2379, fix docker image name. * Per #2379, delete unneeded script. * Per #2379, update GHA to scan Python code and push to the correct SonarQube projects. * Per #2379, update GHA SonarQube project names * Per #2379, update the build job name * Per #2379, update the comile step name * Per #2379, switch to consistent SONAR variable names. * Per #2379, fix type in sed expressions. * Per #2379, just rename the log artifact * Per #2379, use time_command wrapper instead of run_command. * Per #2379, fix bad env var name * Per #2379, switch from egrep to grep. * Per #2379, just try cat-ting the logfile * Per #2379, test whether cat-ting the log file actually works. * Per #2379, revert back * Per #2379, mention SonarQube in the PR template. Make workflow name more succinct. * Per #2379, add SONAR_REFERENCE_BRANCH setting to define the sonar.newCode.referenceBranch property. The goal is to define the comparison reference branch for each SonarQube scan. * Per #2379, have the sonarqube.yml job print the reference branch it's using * Per #2379, intentionally introduce a new code smell to see if SonarQube correctly flag it as appearing in new code. * Per #2379, trying adding the SonarQube quality gate check. * Per #2379, add logic for using the report-task.txt output files to check the quality gate status for both the python and cxx scans. * Per #2379 must use unique GHA id's * Per #2379, working on syntax for quality gate checks * Per #2379, try again. * Per #2379, try again * Per #2379, try again * Per #2379, try again * Per #2379, try again * Per #2379, try again * Per #2379, try yet again * Per #2379 * Per #2379, add more debug * Per #2379, remove -it option from docker run commands * Per #2379, again * Per #2379, now that the scan works as expected, remove the intentional SonarQube code smell as well as debug logging. * Hotfix related to #2379. The sonar.newCode.referenceBranch and sonar.branch.name cannot be set to the same string! Only add the newCode definition when they differ. * #2830 Changed enum STATJobType to enum class * #2830 Changed STATLineType to enum class * #2830 Changed Action to enum class * #2830 Changed ModeDataType to enum class * #2830 Changed StepCase to enum class * #2830 Changed enum to enum class * #2830 Changed GenesisPairCategory to enum class * #2830 Removed rediundabt parenrthese * #2830 Reduced same if checking * #2830 Cleanup * #2830 USe empty() instead of lebgth checking * #2830 Adjusted indentations * Feature #2379 develop sonarqube updates (#2850) * Per #2379, move rgb2ctable.py into the python utility scripts directory for better organization and to enable convenient SonarQube scanning. * Per #2379, remove point.py from the vx_python3_utils directory which cleary was inadvertenlty added during development 4 years ago. As far as I can tell it isn't being called by any other code and doesn't belong in the repository. Note that scripts/python/met/point.py has the same name but is entirely different. * Per #2379, update the GHA SonarQube scan to do a single one with Python and C++ combined. The nightly build script is still doing 2 separate scans for now. If this all works well, they could also be combined into a single one. * Per #2379, eliminate MET_CONFIG_OPTIONS from the SonarQube workflow since it doesn't need to be and probably shouldn't be configurable. * Per #2379, trying to copy report-task.txt out of the image * Per #2379, update build_met_sonarqube.sh to check the scan return status * Per #2379, fix bash assignment syntax * Per #2379, remove unused SCRIPT_DIR envvar * Per #2379, switch to a single SonarQube scan for MET's nightly build as well * Feature 2654 ascii2nc polar buoy support (#2846) * Added iabp data type, and modified file_handler to filter based on time range, which was added as a command line option * handle time using input year, hour, min, and doy * cleanup and switch to position day of year for time computations * Added an ascii2nc unit test for iabp data * Added utility scripts to pull iabp data from the web and find files in a time range * Modified iabp_handler to always output a placeholder 'location' observation with value 1 * added description of IABP data python utility scripts * Fixed syntax error * Fixed Another syntax error. * Slight reformat of documentation * Per #2654, update the Makefiles in scripts/python/utility to include all the python scripts that should be installed. * Per #2654, remove unused code from get_iabp_from_web.py that is getting flagged as a bug by SonarQube. * Per #2654, fix typo in docs --------- Co-authored-by: John Halley Gotway Co-authored-by: MET Tools Test Account * Feature #2786 rpss_from_prob (#2861) * Per #2786, small change to a an error message unrelated to this development. * Per #2786, add RPSInfo::set_climo_prob() function to derive the RPS line type from climatology probability bins. And update Ensemble-Stat to call it. * Per #2786, minor change to clarify error log message. * Per #2786, for is_prob = TRUE input, the RPS line type is the only output option. Still need to update docs! * Per #2786, add new call to Ensemble-Stat to test computing RPS from climo probabilities * Per #2786, use name rps_climo_bin_prob to be very explicit. * Per #2786, redefine logic of RPSInfo::set_climo_bin_prob() to match the CPC definition. Note that reliability, resolution, uncertainty, and RPSS based on the sample climatology are all set to bad data. Need to investigate whether they can be computed using these inputs. * Per #2786, remove the requirement that any fcst.prob_cat_thresh thresholds must be defined. If they are defined, pass them through to the FCST_THRESH output column. If not, write NA. Add check to make sure the event occurs in exactly 1 category. * Per #2786, don't enforce fcst.prob_cat_thresh == obs.prob_cat_thresh for probabilistic inputs. And add more is_prob checks so that only the RPS line type can be written when given probabilistic inputs. * updated documentation * Per #2786, call rescale_probability() function to convert from 0-100 probs to 0-1 probs. --------- Co-authored-by: j-opatz * Feature #2862 v12.0.0-beta4 (#2864) * Feature #2379 develop single_sq_project (#2865) * Hotfix to the documentation in the develop branch. Issue #2858 was closed as a duplicate of #2857. I had included it in the MET-12.0.0-beta4 release notes, but the work is not yet actually complete. * Feature 2842 ugrid config (#2852) * #2842 Removed UGrid related setting * #2842 Corrected vertical level for data_plane_array * #2842 Do not allow the time range * #2842 The UGridConfig file can be passed as ugrid_dataset * #2842 Changed -config option to -ugrid_config * #2842 Deleted UGrid configurations * 2842 Fix a compile error when UGrid is disabled * #2842 Cleanup * #2842 Added an unittest point_stat_ugrid_mpas_config * #2842 Added a PointStatConfig without UGrid dataset. * #2842 Corrected ty[po at the variable name * Switched from time_centered to time_instant. I think time_centered is the center of the forecast lead window and time_instant is the time the forecast is valid (end of forecast window). * #2842 Removed ugrid_max_distance_km and unused metadata names * #2842 Restored time variable time_instant for LFric * #2842 Adjust lon between -180 and 180 * #2842 Adjust lon between -180 and 180 * #2842 Adjust lon between -180 and 180 * #2842 Adjusted lon to between -180 to 180 * #2842 Changed variable names * Per #2842, switch from degrees east to west right when the longitudes are read. * #2842, switch from degrees east to west right when the longitudes are read * #2842 Cleanup debug messages --------- Co-authored-by: Howard Soh Co-authored-by: Daniel Adriaansen Co-authored-by: John Halley Gotway * Feature 2753 comp script config (#2868) * set dynamic library file extension to .dylib if running on MacOS and .so otherwise * Added disabling of jasper documentation for compiliation on Hera * Updated * remove extra export of compiler env vars * include full path to log file so it is easier to file the log file to examine when a command fails * send cmake output to a log file * remove redundant semi-colon * use full path to log file so it is easier to examine on failure * use run_cmd to catch if rm command fails * Modifications for compilation on hera, gaea, and orion * Updating * fixed variable name * clean up if/else statements * set TIFF_LIBRARY_RELEASE argument to use full path to dynamic library file to prevent failure installing proj library * set LDFLAGS so that LDFLAGS value set in the user's environment will also be used * Updated based on gaea, orion, and hera installs * Updated * change extension of dynamic library files only if architecture is arm64 because older Macs still use .so * added netcdf library to args to prevent error installing NetCDF-CXX when PROJ has been installed in the same run of the script -- PATH is set in the COMPILE_PROJ if block that causes this flag from being added automatically * clean up how rpath and -L are added to LDFLAGS so that each entry is separate -- prevents errors installing on Mac arm64 because multiple rpath values aren't read using :. Also use MET_PROJLIB * Updated * removed -ltiff from MET libs * only add path to rpath and -L arguments if they are not already included in LDFLAGS * changed from using LIB_TIFF (full path to tiff lib file) to use TIFF_LIB_DIR (dir containing tiff lib file). Added TIFF_INCLUDE_DIR to proj compilation and -DJAS_ENABLE_DOC to jasper compliation taken from @jprestop branch * update comments * ensure all MET_* and MET_*LIB variables are added to the rpath for consistency * remove unnecessary if block and only export LDFLAGS at the end of setting locally * Updated * Added section for adding /lib64 and rearranged placement of ADDTL_DIR * Commenting out the running of the Jasper lib tests * Updating and/or removing files * Updating and/or removing files * Latest udpates which include the addition of the tiff library for proj * Remove commented out line. Co-authored-by: John Halley Gotway * Make indentation consistent. Co-authored-by: John Halley Gotway * Make indentation consistent. Co-authored-by: John Halley Gotway * Make indentation consistent. Co-authored-by: John Halley Gotway * Per 2753, added -lm to configure_lib_args for NetCDF-CXX * Per #2753 updating acorn files * Per #2753, update wcoss2 files * Per #2753, updating acorn file to include MET_PYTHON_EXE * Per #2753, updated files for 12.0.0 for derecho * Per #2753, updated derecho file adding MET_PYTHON_EXE and made corrections * Updating config files * Updating orion files * Updates for gaea's files * Updating gaea modulefile * Removing modulefile for cheyenne * Added MET_PYTHON_EXE * Added MET_PYTHON_EXE to hera too * Adding file for hercules * Removing equals sign from setenv * Adding file for hercules * Updated script to add libjpeg installation for grib2c * Per #2753, Adding file for casper --------- Co-authored-by: George McCabe <23407799+georgemccabe@users.noreply.github.com> Co-authored-by: John Halley Gotway * Feature #2795 level_mismatch_warning (#2873) * Per #2795, move the warning message about level mismatch from the config validation step to when the forecast files are being processed. Only check this when the number of forecast fields is greater than 1, but no longer limit the check to pressure levels only. * Per #2795, add comments * Whitespace * Per #2795, port level mismatch fix over to Ensemble-Stat. Check it for each verification task, but only print it once for each task, rather than once for each task * ensemble member. * Feature #2870 removing_MISSING_warning (#2872) * Per #2870, define utility functions for parsing the file type from a file list and for logging missing files, checking for the MISSING keyword. Also, update Ensemble-Stat and Gen-Ens-Prod to call these functions. * Per #2870, update the gen_ens_prod tests to demonstrate the use of the MISSING keyword for missing files. METplus uses this keyword for Ensemble-Stat and Gen-Ens-Prod. * Feature 2842 ugrid config (#2875) * #2842 Removed UGrid related setting * #2842 Corrected vertical level for data_plane_array * #2842 Do not allow the time range * #2842 The UGridConfig file can be passed as ugrid_dataset * #2842 Changed -config option to -ugrid_config * #2842 Deleted UGrid configurations * 2842 Fix a compile error when UGrid is disabled * #2842 Cleanup * #2842 Added an unittest point_stat_ugrid_mpas_config * #2842 Added a PointStatConfig without UGrid dataset. * #2842 Corrected ty[po at the variable name * Switched from time_centered to time_instant. I think time_centered is the center of the forecast lead window and time_instant is the time the forecast is valid (end of forecast window). * #2842 Removed ugrid_max_distance_km and unused metadata names * #2842 Restored time variable time_instant for LFric * #2842 Adjust lon between -180 and 180 * #2842 Adjust lon between -180 and 180 * #2842 Adjust lon between -180 and 180 * #2842 Adjusted lon to between -180 to 180 * #2842 Changed variable names * Per #2842, switch from degrees east to west right when the longitudes are read. * #2842, switch from degrees east to west right when the longitudes are read * #2842 Cleanup debug messages * #2842 Disabled output types except STAT for sl1l2 * #2842 Disabled output types except STAT for sl1l2 and MPR * #2842 Reduced output files for UGrid --------- Co-authored-by: Howard Soh Co-authored-by: Daniel Adriaansen Co-authored-by: John Halley Gotway * Hotfix to develop branch to remove duplicate test named 'point_stat_ugrid_mpas_config'. That was causing unit_ugrid.xml to fail because it was still looking for .txt output files that are no longer being generated. * Feature 2748 document ugrid (#2869) * Initial documentation of the UGRID capability. * Fixes error in references, adds appendix to index, and adds sub-section for configuration entries and a table for metadata map items. * Corrects LFRic, rewords section on UGRID conventions, updates description of using GridStat, and removes mention of nodes. * Forgot one more mention of UGRID conventions. * Incorporates more suggestions from @willmayfield. * Switches to numerical table reference. * Feature #2781 Convert MET NetCDF point obs to Pandas DataFrame (#2877) * Per #2781, added function to convert MET NetCDF point observation data to pandas so it can be read and modified in a python embedding script. Added example python embedding script * ignore python cache files * fixed function call * reduce cognitive complexity to satisfy SonarQube and add boolean return value to catch if function fails to read data * clean up script and add comments * replace call to object function that doesn't exist, handle exception when file passed to script cannot be read by the NetCDF library * rename example script * add new example script to makefiles * fix logic to build pandas DataFrame to properly get header information from observation header IDs * Per #2781, add unit test to demonstrate python embedding script that reads MET NetCDF point observation file and converts it to a pandas DataFrame * Per #2781, added init function for nc_point_obs to take an input filename. Also raise TypeError exception from nc_point_obs.read_data() if input file cannot be read * call parent class init function to properly initialize nc_point_obs * Feature #2833 pcp_combine_missing (#2886) * Per #2883, add -input_thresh command line option to configure allowable missing input files. * Per #2883, update pcp_combine usage statement. * Per #2883, update existing pcp_combine -derive unit test example by adding 3 new missing file inputs at the beginning, middle, and end of the file list. The first two are ignored since they include the MISSING keyword, but the third without that keyword triggers a warning message as desired. The -input_thresh option is added to only require 70% of the input files be present. This should produce the exact same output data. * Per #2883, update the pcp_combine logic for the sum command to allow missing data files based on the -input_thresh threshold. Add a test in unit_pcp_combine.xml to demonstrate. * Update docs/Users_Guide/reformat_grid.rst Co-authored-by: George McCabe <23407799+georgemccabe@users.noreply.github.com> * Per #2883, update pcp_combine usage statement in the code to be more simliar to the User's Guide. * Per #2883, switch to using derive_file_list_missing as the one containing missing files and recreate derive_file_list as it had existed for the test named pcp_combine_derive_VLD_THRESH. * Per #2883, move initialization inside the same loop to resolve SonarQube issues. * Per #2883, update sum_data_files() to switch from allocating memory to using STL vectors to satisfy SonarQube. * Per #2883, changes to declarations of variables to satisfy SonarQube. * Per #2883, address more SonarQube issues * Per #2883, backing out an unintended change I made to tcrmw_grid.cc. This change belongs on a different branch. * Per #2883, update logic of parse_file_list_type() function to handle python input strings. Also update pcp_combine to parse the type of input files being read and log non-missing python input files expected. --------- Co-authored-by: George McCabe <23407799+georgemccabe@users.noreply.github.com> * Per #2888, update STATAnalysisJob::dump_stat_line() to support dumping stat line types VCNT, RPS, DMAP, and SSIDX. (#2891) * Per #2659, making updates as proposed at the 20240516 MET Eng. Mtg. (#2895) * Feature #2395 TOTAL_DIR (#2892) * Per #2395, remove the n_dir_undef and n_dira_undef variables that are superceded by the new dcount and dacount VL1L2Info members to keep track of the number of valid wind direction vectors. * Per #2395, add TOTAL_DIR columns to the VL1L2, VAL1L2, and VCNT line types and update the header column tables. * Per #2395, update the User's Guide to list the new TOTAL_DIR columns in the VL1L2, VAL1L2, and VCNT line types. * Per #2395, update stat_analysis to parse the new TOTAL_DIR columns and use the values to aggregate results when needed. * Per #2395, for SonarQube change 'const char *' to 'const char * const' to satisfy the finding that 'Global variables should be const.' Should probably switch from 'char char *' to strings eventually. But for now, I'm just making up for some SonarQube technical debt. * Per #2395, fix typo in placement of the DIR_ME column name in the met_header_columns_V12.0.txt file * Per #2395, add 2 new Stat-Analysis jobs to demonstrate the processing of VL1L2 lines. * Per #2395, update logic of is_vector_dir_stat(). Instead of just checking 'DIR_', check 'DIR_ME', 'DIR_MAE', and 'DIR_MSE' to avoid an false positive match for the 'DIR_ERR' column which is computed from the vector partial sums rather than the individual direction differences. * Bugfix #2897 develop python_valid_time (#2899) * Per #2897, fix typos in 2 log messages. Also fix the bug in storing the valid time strings. The time string in vld_array should exactly correspond to the numeric unixtime values in vld_num_array. Therefore they need to be updated inside the same if block. The bug is that we were storing only the unique unixtime values but storing ALL of the valid time string, not just the unique ones. * Per #2897, minor change to formatting of log message * MET #2897, don’t waste time searching, just set the index to n - 1 * Per #2897, remove unused add_prec_point_obs(...) function * Per #2897, update add_point_obs(...) logic for DEBUG(9) to print very detailed log messages about what obs are being rejected and which are being used for each verification task. * Per #2897, refine the 'using' log message to make the wording consistent with the summary rejection reason counts log message * Per #2897, update the User's Guide about -v 9 for Point-Stat --------- Co-authored-by: j-opatz Co-authored-by: MET Tools Test Account * Bugfix 2867 point2grid qc flag (#2890) * #2867 Added compute_adp_qc_flag and adjusted ADP QC flags * #2867 Added point2grid_GOES_16_ADP_Enterprise_high. Changed AOD QC flags to 0,1,2 (was 1,2,3) * #2867 Added get_nc_att_values_ * #2867 Added get_nc_att_values. Added the argument allow_conversion to get_nc_data(netCDF::NcVar *, uchar *data) * #2867 Read the ADP QC flag values and meanings attributes from DQF variable and set the QC high, meduium, low values to support Enterprise algorithm. Adjusted the ADP QC values by using AOD qc values * #2867 Cleanup * #2867 Corrected indent * #2867 Changed log message * #2867 Removed unused argument * #2867 Removed unused argument * Cleanup * #2867 Fix SonarQube findings * #2867 Deleted protected section with no members * #2867 Cleanup * #2867 FIxed SonarQube findings; unused local variables, decalare as const, etc * #2867 MOved include directives to top * #2867 Changed some argumenmt with references to avoid copying objects * #2867 Do not filter by QC flag if -qc is not given * #2867 Use enumj class for GOES QC: HIGH, MEDIUM, and LOW * #2867 Added log message back which were deleted accidently * #2867 Chaned statci const to constexpr * #2867 Initial release. Separated from nc_utils.h * @2867 Added nc_utils_core.h * #2867 Moved some blocks to nc_utils_core.h * #2867 Include nc_utils_core.h * #2867 Added const references * Per #2867, fixing typo in comments. --------- Co-authored-by: Howard Soh Co-authored-by: j-opatz * Hotfix to develop to fix the update_truth.yml workflow logic. This testing workflow run failed (https://github.com/dtcenter/MET/actions/runs/9209471209). Here we switch to a unique update truth branch name to avoid conflicts. * Avoid pushing directly to the develop or main_vX.Y branches since that is not necessary for the automation logic in MET. * #2904 Changed R path to R-4.4.0 (#2905) Co-authored-by: Howard Soh * Feature #2912 pb2nc error (#2914) * Feature 2717 convert unit.pl to unit.py (#2871) * created unit.py module in new internal/test_unit/python directory * added xml parsing to unit.py * added repl_env function * added reading of the remaining xml tags in build_tests function * progress on main function (putting together test commands) * a few more lines in the main function * minor updates * fixed how the test command was being run * added if name/main and command line parsing * fixed handling of no 'env' in cmd_only mode * handle params from xml that have \ after filename without space in between * added logging * added some more pieces to unit * more updates to unit.py, including running checks on output files * bug fixes, improved handling of output file names, improved handling of env vars, improved logging output * fixed how shell commands are run, and other minor fixes * added last bits from the perl script, fixed some bugs * created unit.py module in new internal/test_unit/python directory * added xml parsing to unit.py * added repl_env function * added reading of the remaining xml tags in build_tests function * progress on main function (putting together test commands) * a few more lines in the main function * minor updates * update scripts to call python unit test script instead of the old perl script * fix she-bang line to allow script to be run without python3 before it * add missing test_dir and exit_on_fail tags that are found in the rest of the unit test xml files * fix call to logger.warning * change tags named 'exists' to 'exist' to match the rest of the xml files * added logger to function * removed tab at end of line that was causing output file path to be excluded from the command * fix broken checks for output files * incorporated george's recommended changes * changed default to overwrite logs; allow for more than one xml file to be passed in command --------- Co-authored-by: Natalie babij Co-authored-by: Natalie babij Co-authored-by: Natalie babij Co-authored-by: Natalie Babij Co-authored-by: John Halley Gotway Co-authored-by: George McCabe <23407799+georgemccabe@users.noreply.github.com> Co-authored-by: j-opatz * Bugfix 2867 point2grid qc unittest (#2913) * #2867 Added compute_adp_qc_flag and adjusted ADP QC flags * #2867 Added point2grid_GOES_16_ADP_Enterprise_high. Changed AOD QC flags to 0,1,2 (was 1,2,3) * #2867 Added get_nc_att_values_ * #2867 Added get_nc_att_values. Added the argument allow_conversion to get_nc_data(netCDF::NcVar *, uchar *data) * #2867 Read the ADP QC flag values and meanings attributes from DQF variable and set the QC high, meduium, low values to support Enterprise algorithm. Adjusted the ADP QC values by using AOD qc values * #2867 Cleanup * #2867 Corrected indent * #2867 Changed log message * #2867 Removed unused argument * #2867 Removed unused argument * Cleanup * #2867 Fix SonarQube findings * #2867 Deleted protected section with no members * #2867 Cleanup * #2867 FIxed SonarQube findings; unused local variables, decalare as const, etc * #2867 MOved include directives to top * #2867 Changed some argumenmt with references to avoid copying objects * #2867 Do not filter by QC flag if -qc is not given * #2867 Use enumj class for GOES QC: HIGH, MEDIUM, and LOW * #2867 Added log message back which were deleted accidently * #2867 Chaned statci const to constexpr * #2867 Initial release. Separated from nc_utils.h * @2867 Added nc_utils_core.h * #2867 Moved some blocks to nc_utils_core.h * #2867 Include nc_utils_core.h * #2867 Added const references * #2867 Some 'static const' were chnaged to constexpr * #2867 Changed -qc options (1,2,3 to 0,1 - high & medium) for AOD * #2867 Merged develop branch * #2867 Corrected the unit test name --------- Co-authored-by: Howard Soh * Feature #2911 tc_stat_set_hdr (#2916) * Per #2911, no real changes for Stat-Analysis. Just changing order of variables for consistency. * Per #2911, add StatHdrColumns::apply_set_hdr_opts(...) function to be used by TC-Stat. * Per #2911, move ByColumn to the TCStatJob base class and add HdrName and HdrValue to support the -set_hdr job command. * Per #2911, update GSI tools to call the newly added StatHdrColumns::apply_set_hdr_opts(...) function. * Per #2911, update logic of Stat-Analysis for consistency to make use of common apply_set_hdr_opts() function. * Per #2911, add DataLine::set_item() function to support -set_hdr options. * Per #2911, just update contents of error message * Per #2911, add TCStatLine member functions for has() and get_offset(). * Per #2911, update tc_stat to support applying -set_hdr to TC-Stat filter jobs. * Per #2911, revise TC-Stat config files to exercise the -set_hdr job command option * Per #2911, update TC-Stat documentation to mention the -set_hdr job command option * Per #2911, add note * Per #2911, as recommended by SonarQube, make some of these member functions const. --------- Co-authored-by: Howard Soh Co-authored-by: John Halley Gotway Co-authored-by: Howard Soh Co-authored-by: MET Tools Test Account Co-authored-by: davidalbo Co-authored-by: j-opatz Co-authored-by: Daniel Adriaansen Co-authored-by: Julie Prestopnik Co-authored-by: George McCabe <23407799+georgemccabe@users.noreply.github.com> Co-authored-by: natalieb-noaa <146213121+natalieb-noaa@users.noreply.github.com> Co-authored-by: Natalie babij Co-authored-by: Natalie babij Co-authored-by: Natalie babij Co-authored-by: Natalie Babij Co-authored-by: metplus-bot <97135045+metplus-bot@users.noreply.github.com> --- docs/Users_Guide/tc-stat.rst | 14 ++ .../test_unit/config/TCStatConfig_ALAL2010 | 6 +- .../test_unit/config/TCStatConfig_PROBRIRW | 2 +- src/basic/vx_util/data_line.cc | 27 +++ src/basic/vx_util/data_line.h | 6 + src/libcode/vx_analysis_util/stat_job.cc | 14 +- src/libcode/vx_analysis_util/stat_job.h | 9 +- src/libcode/vx_stat_out/stat_hdr_columns.cc | 214 ++++++++++++++++++ src/libcode/vx_stat_out/stat_hdr_columns.h | 8 + src/libcode/vx_tc_util/tc_columns.cc | 19 +- src/libcode/vx_tc_util/tc_columns.h | 2 + src/libcode/vx_tc_util/tc_hdr_columns.cc | 86 +++++++ src/libcode/vx_tc_util/tc_hdr_columns.h | 3 + src/libcode/vx_tc_util/tc_stat_line.cc | 30 +++ src/libcode/vx_tc_util/tc_stat_line.h | 2 + .../core/stat_analysis/aggr_stat_line.cc | 153 +++---------- src/tools/core/stat_analysis/aggr_stat_line.h | 8 +- src/tools/other/gsi_tools/gsi_util.cc | 167 ++------------ src/tools/other/gsi_tools/gsi_util.h | 28 +-- src/tools/other/gsi_tools/gsid2mpr.cc | 27 ++- src/tools/tc_utils/tc_stat/tc_stat.cc | 1 + src/tools/tc_utils/tc_stat/tc_stat_job.cc | 103 +++++---- src/tools/tc_utils/tc_stat/tc_stat_job.h | 22 +- 23 files changed, 577 insertions(+), 374 deletions(-) diff --git a/docs/Users_Guide/tc-stat.rst b/docs/Users_Guide/tc-stat.rst index bdcbc35327..3cddda63f2 100644 --- a/docs/Users_Guide/tc-stat.rst +++ b/docs/Users_Guide/tc-stat.rst @@ -400,6 +400,8 @@ The output generated from the TC-Stat tool contains statistics produced by the a This job command finds and filters TCST lines down to those meeting the criteria selected by the filter's options. The filtered TCST lines are written to a file specified by the **-dump_row** option. The TCST output from this job follows the TCST output description in :numref:`tc-dland` and :numref:`tc-pairs`. + The "-set_hdr" job command option can be used to override any of the output header strings (e.g. "-set_hdr DESC EVENT_EQUAL" sets the output DESC column to "EVENT_EQUAL"). + **Job: Summary** This job produces summary statistics for the column name specified by the **-column** option. The output of the summary job consists of three rows: @@ -475,6 +477,18 @@ Users may also specify the **-out_alpha** option to define the alpha value for t Users may also specify the **-out_stat** option to write the contingency table counts and statistics (for the CTC and CTS output line types) to an output STAT file. Information about the RIRW timing information and filtering criteria are written to the STAT header columns while the contingency table counts and/or statistics are written to the CTC and/or CTS output columns. +When using the "-out_stat" option to create a .stat output file and stratifying results using one or more "-by" job command options, those columns may be referenced in the "-set_hdr" option. + +.. code-block:: none + + -job rirw -line_type TCMPR -by CYCLONE -out_stat ctc.stat -set_hdr DESC CYCLONE + +When using multiple "-by" options, use "CASE" to reference the full case information string. + +.. code-block:: none + + -job rirw -line_type TCMPR -by CYCLONE,LEAD -out_stat ctc.stat -set_hdr DESC CASE + **Job: PROBRIRW** The PROBRIRW job produces probabilistic contingency table counts and statistics defined by placing forecast probabilities and BEST track rapid intensification events into an Nx2 contingency table. Users may specify several job command options to configure the behavior of this job: diff --git a/internal/test_unit/config/TCStatConfig_ALAL2010 b/internal/test_unit/config/TCStatConfig_ALAL2010 index dd0de6084d..b46d0f72c3 100644 --- a/internal/test_unit/config/TCStatConfig_ALAL2010 +++ b/internal/test_unit/config/TCStatConfig_ALAL2010 @@ -208,11 +208,11 @@ jobs = [ "-job summary -column TRACK -by AMODEL -dump_row ${MET_TEST_OUTPUT}/tc_stat/ALAL2010_AHWI_vs_BCLP.tcst", "-job summary -column TRACK -by AMODEL,INIT -dump_row ${MET_TEST_OUTPUT}/tc_stat/ALAL2010_AHWI_vs_BCLP_by_INIT.tcst", "-job summary -column WIND -by AMODEL -column_union true", - "-job filter -amodel AHWI -rirw_track BDECK -rirw_thresh >=30 -rirw_exact FALSE -dump_row ${MET_TEST_OUTPUT}/tc_stat/ALAL2010_AHWI_ri.tcst", - "-job filter -amodel AHWI -rirw_track BDECK -rirw_thresh <=-30 -rirw_exact TRUE -dump_row ${MET_TEST_OUTPUT}/tc_stat/ALAL2010_AHWI_rw.tcst", + "-job filter -amodel AHWI -rirw_track BDECK -rirw_thresh >=30 -rirw_exact FALSE -set_hdr DESC RI -dump_row ${MET_TEST_OUTPUT}/tc_stat/ALAL2010_AHWI_ri.tcst", + "-job filter -amodel AHWI -rirw_track BDECK -rirw_thresh <=-30 -rirw_exact TRUE -set_hdr DESC RW -dump_row ${MET_TEST_OUTPUT}/tc_stat/ALAL2010_AHWI_rw.tcst", "-job rirw -rirw_window 00 -rirw_thresh <=-15 -out_line_type CTC,CTS,MPR", "-job rirw -rirw_window 12 -rirw_thresh <=-15 -out_line_type CTC,CTS,MPR", - "-job rirw -rirw_window 12 -rirw_thresh <=-15 -out_line_type CTC,CTS -by amodel -out_stat ${MET_TEST_OUTPUT}/tc_stat/ALAL2010_rirw.stat" + "-job rirw -rirw_window 12 -rirw_thresh <=-15 -out_line_type CTC,CTS -by amodel -set_hdr DESC AMODEL -out_stat ${MET_TEST_OUTPUT}/tc_stat/ALAL2010_rirw.stat" ]; // diff --git a/internal/test_unit/config/TCStatConfig_PROBRIRW b/internal/test_unit/config/TCStatConfig_PROBRIRW index f8645a1e1e..679b681e59 100644 --- a/internal/test_unit/config/TCStatConfig_PROBRIRW +++ b/internal/test_unit/config/TCStatConfig_PROBRIRW @@ -205,7 +205,7 @@ out_valid_mask = ""; // Array of TCStat analysis jobs to be performed on the filtered data // jobs = [ - "-job filter -dump_row ${MET_TEST_OUTPUT}/tc_stat/PROBRIRW_filter_ee.tcst", + "-job filter -set_hdr DESC EVENT_EQUAL -dump_row ${MET_TEST_OUTPUT}/tc_stat/PROBRIRW_filter_ee.tcst", "-job summary -column TK_ERR -by AMODEL -probrirw_thresh 30 -column_thresh PROBRIRW_PROB >0 -dump_row ${MET_TEST_OUTPUT}/tc_stat/PROBRIRW_summary_tk_err.tcst", "-job probrirw -column_thresh RIRW_WINDOW ==24 -by AMODEL -probrirw_thresh 30 -probrirw_bdelta_thresh >=30 -out_line_type PCT,PSTD,PRC,PJC -dump_row ${MET_TEST_OUTPUT}/tc_stat/PROBRIRW_probrirw.tcst", "-job summary -column TK_ERR -by AMODEL,LEAD -amodel GPMI,GPMN -event_equal TRUE", diff --git a/src/basic/vx_util/data_line.cc b/src/basic/vx_util/data_line.cc index 62b7e0c923..5c283e258d 100644 --- a/src/basic/vx_util/data_line.cc +++ b/src/basic/vx_util/data_line.cc @@ -232,6 +232,33 @@ return; //////////////////////////////////////////////////////////////////////// +void DataLine::set_item(int k, const ConcatString &item_cs) + +{ + +if ( (k < 0) || (k >= N_items) ) { + + ConcatString cs = (File ? File->filename() : ""); + + mlog << Error << "\nDataLine::set_item(int) -> " + << "range check error setting line number " << LineNumber + << ", item number " << k+1 << " of " << N_items + << " from file \"" << cs << "\"\n\n"; + + exit ( 1 ); + +} + +Items[k] = item_cs; + +return; + +} + + +//////////////////////////////////////////////////////////////////////// + + const char * DataLine::get_item(int k) const { diff --git a/src/basic/vx_util/data_line.h b/src/basic/vx_util/data_line.h index bb9de49c8e..b3ef288b5e 100644 --- a/src/basic/vx_util/data_line.h +++ b/src/basic/vx_util/data_line.h @@ -98,6 +98,12 @@ class DataLine { void dump(std::ostream &, int depth = 0) const; + // + // set stuff + // + + void set_item(int, const ConcatString &); + // // retrieve stuff // diff --git a/src/libcode/vx_analysis_util/stat_job.cc b/src/libcode/vx_analysis_util/stat_job.cc index dda0a08bd9..ced4361068 100644 --- a/src/libcode/vx_analysis_util/stat_job.cc +++ b/src/libcode/vx_analysis_util/stat_job.cc @@ -1174,13 +1174,13 @@ void STATAnalysisJob::parse_job_command(const char *jobstring) { else if(jc_array[i] == "-column_str_exc" ) { column_str_exc_map.clear(); } + else if(jc_array[i] == "-by" ) { + by_column.clear(); + } else if(jc_array[i] == "-set_hdr" ) { hdr_name.clear(); hdr_value.clear(); } - else if(jc_array[i] == "-by" ) { - by_column.clear(); - } else if(jc_array[i] == "-out_line_type" ) { out_line_type.clear(); } @@ -1450,15 +1450,15 @@ void STATAnalysisJob::parse_job_command(const char *jobstring) { } i+=2; } + else if(jc_array[i] == "-by") { + by_column.add_css(to_upper(jc_array[i+1])); + i+=1; + } else if(jc_array[i] == "-set_hdr") { hdr_name.add_css(to_upper(jc_array[i+1])); hdr_value.add_css(jc_array[i+2]); i+=2; } - else if(jc_array[i] == "-by") { - by_column.add_css(to_upper(jc_array[i+1])); - i+=1; - } else if(jc_array[i] == "-dump_row") { set_dump_row(jc_array[i+1].c_str()); i++; diff --git a/src/libcode/vx_analysis_util/stat_job.h b/src/libcode/vx_analysis_util/stat_job.h index 208c6d4894..88b709adf7 100644 --- a/src/libcode/vx_analysis_util/stat_job.h +++ b/src/libcode/vx_analysis_util/stat_job.h @@ -234,14 +234,17 @@ class STATAnalysisJob { std::map column_str_inc_map; std::map column_str_exc_map; - StringArray hdr_name; - StringArray hdr_value; - // // Store the case information for the -by option // StringArray by_column; + // + // Options for -set_hdr output + // + StringArray hdr_name; + StringArray hdr_value; + // // Variables used to the store the analysis job specification // diff --git a/src/libcode/vx_stat_out/stat_hdr_columns.cc b/src/libcode/vx_stat_out/stat_hdr_columns.cc index 005499d8ed..3c3560b587 100644 --- a/src/libcode/vx_stat_out/stat_hdr_columns.cc +++ b/src/libcode/vx_stat_out/stat_hdr_columns.cc @@ -15,6 +15,9 @@ using namespace std; +//////////////////////////////////////////////////////////////////////// + +static const string case_str = "CASE"; //////////////////////////////////////////////////////////////////////// // @@ -337,6 +340,217 @@ void StatHdrColumns::set_alpha(const double a) { //////////////////////////////////////////////////////////////////////// +void StatHdrColumns::apply_set_hdr_opts( + const StringArray &hdr_cols, const StringArray &hdr_vals) { + StringArray case_cols; + StringArray case_vals; + + // Call other implementation without case information + apply_set_hdr_opts(hdr_cols, hdr_vals, case_cols, case_vals); + + return; +} + +//////////////////////////////////////////////////////////////////////// +// +// Use the current -set_hdr options to populate the STAT header columns, +// substituting in case-specific values, as needed. +// +//////////////////////////////////////////////////////////////////////// + +void StatHdrColumns::apply_set_hdr_opts( + const StringArray &hdr_cols, const StringArray &hdr_vals, + const StringArray &case_cols, const StringArray &case_vals) { + + // No updates needed + if(hdr_cols.n() == 0) return; + + // Sanity check lengths + if(hdr_cols.n() != hdr_vals.n()) { + mlog << Error << "\nStatHdrColumns::apply_set_hdr_opts() -> " + << "the number of -set_hdr columns names (" << hdr_cols.n() + << " and values (" << hdr_vals.n() << " must match!\n\n"; + exit(1); + } + if(case_cols.n() != case_vals.n()) { + mlog << Error << "\nStatHdrColumns::apply_set_hdr_opts() -> " + << "the number of case columns names (" << case_cols.n() + << " and values (" << case_vals.n() << " must match!\n\n"; + exit(1); + } + + int index; + ConcatString cs; + SingleThresh st; + + // MODEL + if(hdr_cols.has("MODEL", index)) { + cs = get_set_hdr_str(hdr_vals[index], case_cols, case_vals); + set_model(cs.c_str()); + } + + // DESC + if(hdr_cols.has("DESC", index)) { + cs = get_set_hdr_str(hdr_vals[index], case_cols, case_vals); + set_desc(cs.c_str()); + } + + // FCST_LEAD + if(hdr_cols.has("FCST_LEAD", index)) { + cs = get_set_hdr_str(hdr_vals[index], case_cols, case_vals); + set_fcst_lead_sec(timestring_to_sec(cs.c_str())); + } + + // FCST_VALID_BEG, FCST_VALID_END + if(hdr_cols.has("FCST_VALID_BEG", index)) { + cs = get_set_hdr_str(hdr_vals[index], case_cols, case_vals); + set_fcst_valid_beg(timestring_to_unix(cs.c_str())); + } + if(hdr_cols.has("FCST_VALID_END", index)) { + cs = get_set_hdr_str(hdr_vals[index], case_cols, case_vals); + set_fcst_valid_end(timestring_to_unix(cs.c_str())); + } + + // OBS_LEAD + if(hdr_cols.has("OBS_LEAD", index)) { + cs = get_set_hdr_str(hdr_vals[index], case_cols, case_vals); + set_obs_lead_sec(timestring_to_sec(cs.c_str())); + } + + // OBS_VALID_BEG, OBS_VALID_END + if(hdr_cols.has("OBS_VALID_BEG", index)) { + cs = get_set_hdr_str(hdr_vals[index], case_cols, case_vals); + set_obs_valid_beg(timestring_to_unix(cs.c_str())); + } + if(hdr_cols.has("OBS_VALID_END", index)) { + cs = get_set_hdr_str(hdr_vals[index], case_cols, case_vals); + set_obs_valid_end(timestring_to_unix(cs.c_str())); + } + + // FCST_VAR + if(hdr_cols.has("FCST_VAR", index)) { + cs = get_set_hdr_str(hdr_vals[index], case_cols, case_vals); + set_fcst_var(cs.c_str()); + } + + // FCST_UNITS + if(hdr_cols.has("FCST_UNITS", index)) { + cs = get_set_hdr_str(hdr_vals[index], case_cols, case_vals); + set_fcst_units(cs.c_str()); + } + + // FCST_LEV + if(hdr_cols.has("FCST_LEV", index)) { + cs = get_set_hdr_str(hdr_vals[index], case_cols, case_vals); + set_fcst_lev(cs.c_str()); + } + + // OBS_VAR + if(hdr_cols.has("OBS_VAR", index)) { + cs = get_set_hdr_str(hdr_vals[index], case_cols, case_vals); + set_obs_var(cs.c_str()); + } + + // OBS_UNITS + if(hdr_cols.has("OBS_UNITS", index)) { + cs = get_set_hdr_str(hdr_vals[index], case_cols, case_vals); + set_obs_units(cs.c_str()); + } + + // OBS_LEV + if(hdr_cols.has("OBS_LEV", index)) { + cs = get_set_hdr_str(hdr_vals[index], case_cols, case_vals); + set_obs_lev(cs.c_str()); + } + + // OBTYPE + if(hdr_cols.has("OBTYPE", index)) { + cs = get_set_hdr_str(hdr_vals[index], case_cols, case_vals); + set_obtype(cs.c_str()); + } + + // VX_MASK + if(hdr_cols.has("VX_MASK", index)) { + cs = get_set_hdr_str(hdr_vals[index], case_cols, case_vals); + set_mask(cs.c_str()); + } + + // INTERP_MTHD + if(hdr_cols.has("INTERP_MTHD", index)) { + cs = get_set_hdr_str(hdr_vals[index], case_cols, case_vals); + set_interp_mthd(cs.c_str()); + } + + // INTERP_PNTS + if(hdr_cols.has("INTERP_PNTS", index)) { + cs = get_set_hdr_str(hdr_vals[index], case_cols, case_vals); + set_interp_wdth(nint(sqrt(atof(cs.c_str())))); + } + + // FCST_THRESH + if(hdr_cols.has("FCST_THRESH", index)) { + cs = get_set_hdr_str(hdr_vals[index], case_cols, case_vals); + st.set(cs.c_str()); + set_fcst_thresh(st); + } + + // OBS_THRESH + if(hdr_cols.has("OBS_THRESH", index)) { + cs = get_set_hdr_str(hdr_vals[index], case_cols, case_vals); + st.set(cs.c_str()); + set_obs_thresh(st); + } + + // COV_THRESH + if(hdr_cols.has("COV_THRESH", index)) { + cs = get_set_hdr_str(hdr_vals[index], case_cols, case_vals); + st.set(cs.c_str()); + set_cov_thresh(st); + } + + // ALPHA + if(hdr_cols.has("ALPHA", index)) { + cs = get_set_hdr_str(hdr_vals[index], case_cols, case_vals); + set_alpha(atof(cs.c_str())); + } + + // LINE_TYPE + if(hdr_cols.has("LINE_TYPE", index)) { + cs = get_set_hdr_str(hdr_vals[index], case_cols, case_vals); + set_line_type(cs.c_str()); + } + + return; +} + +//////////////////////////////////////////////////////////////////////// + +ConcatString StatHdrColumns::get_set_hdr_str(const std::string &hdr_val, + const StringArray &case_cols, const StringArray &case_vals) const { + ConcatString cs; + int index; + + // Check for the full CASE string + if(case_str.compare(hdr_val) == 0) { + cs = case_vals.serialize(":"); + } + // Check for one of the case columns + else if(case_cols.has(hdr_val, index)) { + cs = case_vals[index]; + } + // Otherwise, use the current header value + else { + cs = hdr_val; + } + + // Sanity check for empty strings + if(cs.empty()) cs = na_str; + + return cs; +} + +//////////////////////////////////////////////////////////////////////// + ConcatString StatHdrColumns::get_fcst_thresh_str() const { ConcatString cs; diff --git a/src/libcode/vx_stat_out/stat_hdr_columns.h b/src/libcode/vx_stat_out/stat_hdr_columns.h index 6a1c5da0fb..8fb9e4f564 100644 --- a/src/libcode/vx_stat_out/stat_hdr_columns.h +++ b/src/libcode/vx_stat_out/stat_hdr_columns.h @@ -151,6 +151,14 @@ class StatHdrColumns { void set_cov_thresh (const ThreshArray); void set_alpha (const double); + // Apply -set_hdr overrides + void apply_set_hdr_opts(const StringArray &, const StringArray &); + void apply_set_hdr_opts(const StringArray &, const StringArray &, + const StringArray &, const StringArray &); + + ConcatString get_set_hdr_str(const std::string &, + const StringArray &, const StringArray &) const; + // Get functions ConcatString get_model () const; ConcatString get_desc () const; diff --git a/src/libcode/vx_tc_util/tc_columns.cc b/src/libcode/vx_tc_util/tc_columns.cc index 6f7e7f2d6c..f91431718f 100644 --- a/src/libcode/vx_tc_util/tc_columns.cc +++ b/src/libcode/vx_tc_util/tc_columns.cc @@ -170,7 +170,8 @@ void write_prob_rirw_header_row(int hdr_flag, int n_thresh, AsciiTable &at, //////////////////////////////////////////////////////////////////////// void write_track_pair_info(TcHdrColumns &hdr, const TrackPairInfo &p, - AsciiTable &at, int &i_row) { + AsciiTable &at, int &i_row, + const StringArray &hdr_name, const StringArray &hdr_value) { // Loop through the TrackPairInfo points for(int i=0; iget_item("DESC", false)); } + // Apply -set_hdr options + if(hdr_name.n() > 0) hdr.apply_set_hdr_opts(hdr_name, hdr_value); + // Write the header columns write_tc_header_cols(hdr, at, i_row); @@ -221,6 +225,19 @@ void write_track_pair_info(TcHdrColumns &hdr, const TrackPairInfo &p, //////////////////////////////////////////////////////////////////////// +void write_track_pair_info(TcHdrColumns &hdr, const TrackPairInfo &p, + AsciiTable &at, int &i_row) { + + StringArray hdr_name; + StringArray hdr_value; + + write_track_pair_info(hdr, p, at, i_row, hdr_name, hdr_value); + + return; +} + +//////////////////////////////////////////////////////////////////////// + void write_prob_rirw_pair_info(TcHdrColumns &hdr, const ProbRIRWPairInfo &p, AsciiTable &at, int &i_row) { diff --git a/src/libcode/vx_tc_util/tc_columns.h b/src/libcode/vx_tc_util/tc_columns.h index 9a5aee0181..0d3a83e0a8 100644 --- a/src/libcode/vx_tc_util/tc_columns.h +++ b/src/libcode/vx_tc_util/tc_columns.h @@ -166,6 +166,8 @@ extern void write_prob_rirw_header_row(int, int, AsciiTable &, int, int); // Write out the data lines extern void write_track_pair_info (TcHdrColumns &, const TrackPairInfo &, AsciiTable &, int &); +extern void write_track_pair_info (TcHdrColumns &, const TrackPairInfo &, AsciiTable &, int &, + const StringArray &, const StringArray &); extern void write_prob_rirw_pair_info(TcHdrColumns &, const ProbRIRWPairInfo &, AsciiTable &, int &); // Write out the header entries diff --git a/src/libcode/vx_tc_util/tc_hdr_columns.cc b/src/libcode/vx_tc_util/tc_hdr_columns.cc index 3708417560..9cb4c1a77b 100644 --- a/src/libcode/vx_tc_util/tc_hdr_columns.cc +++ b/src/libcode/vx_tc_util/tc_hdr_columns.cc @@ -64,3 +64,89 @@ void TcHdrColumns::clear() { } //////////////////////////////////////////////////////////////////////// + +void TcHdrColumns::apply_set_hdr_opts( + const StringArray &hdr_cols, const StringArray &hdr_vals) { + + // No updates needed + if(hdr_cols.n() == 0) return; + + int index; + + // Sanity check lengths + if(hdr_cols.n() != hdr_vals.n()) { + mlog << Error << "\nTcHdrColumns::apply_set_hdr_opts() -> " + << "the number of -set_hdr columns names (" << hdr_cols.n() + << " and values (" << hdr_vals.n() << " must match!\n\n"; + exit(1); + } + + // AMODEL + if(hdr_cols.has("AMODEL", index)) { + set_adeck_model(hdr_vals[index]); + } + + // BMODEL + if(hdr_cols.has("BMODEL", index)) { + set_bdeck_model(hdr_vals[index]); + } + + // DESC + if(hdr_cols.has("DESC", index)) { + set_desc(hdr_vals[index]); + } + + // STORM_ID + if(hdr_cols.has("STORM_ID", index)) { + set_storm_id(hdr_vals[index]); + } + + // BASIN + if(hdr_cols.has("BASIN", index)) { + set_basin(hdr_vals[index]); + } + + // CYCLONE + if(hdr_cols.has("CYCLONE", index)) { + set_cyclone(hdr_vals[index]); + } + + // STORM_NAME + if(hdr_cols.has("STORM_NAME", index)) { + set_storm_name(hdr_vals[index]); + } + + // INIT + if(hdr_cols.has("INIT", index)) { + set_init(timestring_to_sec(hdr_vals[index].c_str())); + } + + // LEAD + if(hdr_cols.has("LEAD", index)) { + set_lead(timestring_to_sec(hdr_vals[index].c_str())); + } + + // VALID + if(hdr_cols.has("VALID", index)) { + set_valid(timestring_to_sec(hdr_vals[index].c_str())); + } + + // INIT_MASK + if(hdr_cols.has("INIT_MASK", index)) { + set_init_mask(hdr_vals[index]); + } + + // VALID_MASK + if(hdr_cols.has("VALID_MASK", index)) { + set_valid_mask(hdr_vals[index]); + } + + // LINE_TYPE + if(hdr_cols.has("LINE_TYPE", index)) { + set_line_type(hdr_vals[index]); + } + + return; +} + +//////////////////////////////////////////////////////////////////////// diff --git a/src/libcode/vx_tc_util/tc_hdr_columns.h b/src/libcode/vx_tc_util/tc_hdr_columns.h index f632dfaf56..60ecf39260 100644 --- a/src/libcode/vx_tc_util/tc_hdr_columns.h +++ b/src/libcode/vx_tc_util/tc_hdr_columns.h @@ -70,6 +70,9 @@ class TcHdrColumns { void set_valid_mask (const ConcatString &); void set_line_type (const ConcatString &); + // Apply -set_hdr overrides + void apply_set_hdr_opts(const StringArray &, const StringArray &); + // Get functions ConcatString adeck_model () const; ConcatString bdeck_model () const; diff --git a/src/libcode/vx_tc_util/tc_stat_line.cc b/src/libcode/vx_tc_util/tc_stat_line.cc index a8556a7dd2..d46dc41c77 100644 --- a/src/libcode/vx_tc_util/tc_stat_line.cc +++ b/src/libcode/vx_tc_util/tc_stat_line.cc @@ -164,6 +164,36 @@ bool TCStatLine::is_header() const { //////////////////////////////////////////////////////////////////////// +bool TCStatLine::has(const char *col_str) const { + return !is_bad_data(get_offset(col_str)); +} + +//////////////////////////////////////////////////////////////////////// + +int TCStatLine::get_offset(const char *col_str) const { + int offset = bad_data_int; + + // + // Search for matching header column + // + offset = HdrLine->col_offset(col_str); + + // + // If not found, check extra header columns + // + if(is_bad_data(offset)) { + if(!get_file()->header().has(col_str, offset)) offset = bad_data_int; + } + + // + // Return the offset value + // + + return offset; +} + +//////////////////////////////////////////////////////////////////////// + ConcatString TCStatLine::get(const char *col_str, bool check_na) const { ConcatString cs = (string)get_item(col_str, check_na); diff --git a/src/libcode/vx_tc_util/tc_stat_line.h b/src/libcode/vx_tc_util/tc_stat_line.h index 276fad1638..e39dcf7f70 100644 --- a/src/libcode/vx_tc_util/tc_stat_line.h +++ b/src/libcode/vx_tc_util/tc_stat_line.h @@ -75,6 +75,8 @@ class TCStatLine : public DataLine { // Retrieve values of the header columns // + bool has (const char *) const; + int get_offset (const char *) const; ConcatString get (const char *, bool check_na = true) const; const char * get_item (const char *, bool check_na = true) const; const char * get_item (int, bool check_na = true) const; diff --git a/src/tools/core/stat_analysis/aggr_stat_line.cc b/src/tools/core/stat_analysis/aggr_stat_line.cc index c46c35c134..c40fb0eb64 100644 --- a/src/tools/core/stat_analysis/aggr_stat_line.cc +++ b/src/tools/core/stat_analysis/aggr_stat_line.cc @@ -41,6 +41,7 @@ // to VL1L2, VAL1L2, and VCNT. // 019 02/21/24 Halley Gotway MET #2583 Add observation error // ECNT statistics. +// 020 06/14/24 Halley Gotway MET #2911 Call apply_set_hdr_opts(). // //////////////////////////////////////////////////////////////////////// @@ -62,7 +63,6 @@ using namespace std; - //////////////////////////////////////////////////////////////////////// static bool is_precip_var_name(const ConcatString &s); @@ -350,25 +350,17 @@ StatHdrColumns StatHdrInfo::get_shc(const ConcatString &cur_case, const StringArray &hdr_cols, const StringArray &hdr_vals, const STATLineType lt) { - ConcatString css; - StringArray case_vals; ThreshArray ta; + ConcatString css; double out_alpha; - int index, wdth; + int wdth; StatHdrColumns shc; - // Split up the current case into values - case_vals = cur_case.split(":"); - // MODEL - shc.set_model( - get_shc_str(cur_case, case_cols, case_vals, hdr_cols, hdr_vals, - "MODEL", model, false).c_str()); + shc.set_model(get_col_css(cur_case, "MODEL", model, false).c_str()); // DESC - shc.set_desc( - get_shc_str(cur_case, case_cols, case_vals, hdr_cols, hdr_vals, - "DESC", desc, false).c_str()); + shc.set_desc(get_col_css(cur_case, "DESC", desc, false).c_str()); // FCST_LEAD css = write_css_hhmmss(fcst_lead); @@ -378,26 +370,11 @@ StatHdrColumns StatHdrInfo::get_shc(const ConcatString &cur_case, << fcst_lead.n() << " unique FCST_LEAD values: " << css << "\n"; } - if(hdr_cols.has("FCST_LEAD", index)) { - shc.set_fcst_lead_sec(timestring_to_sec(hdr_vals[index].c_str())); - } - else { - shc.set_fcst_lead_sec(fcst_lead.max()); - } + shc.set_fcst_lead_sec(fcst_lead.max()); // FCST_VALID_BEG, FCST_VALID_END - if(hdr_cols.has("FCST_VALID_BEG", index)) { - shc.set_fcst_valid_beg(timestring_to_unix(hdr_vals[index].c_str())); - } - else { - shc.set_fcst_valid_beg(fcst_valid_beg); - } - if(hdr_cols.has("FCST_VALID_END", index)) { - shc.set_fcst_valid_end(timestring_to_unix(hdr_vals[index].c_str())); - } - else { - shc.set_fcst_valid_end(fcst_valid_end); - } + shc.set_fcst_valid_beg(fcst_valid_beg); + shc.set_fcst_valid_end(fcst_valid_end); // OBS_LEAD css = write_css_hhmmss(obs_lead); @@ -407,71 +384,38 @@ StatHdrColumns StatHdrInfo::get_shc(const ConcatString &cur_case, << obs_lead.n() << " unique OBS_LEAD values: " << css << "\n"; } - if(hdr_cols.has("OBS_LEAD", index)) { - shc.set_obs_lead_sec(timestring_to_sec(hdr_vals[index].c_str())); - } - else { - shc.set_obs_lead_sec(obs_lead.max()); - } + shc.set_obs_lead_sec(obs_lead.max()); // OBS_VALID_BEG, OBS_VALID_END - if(hdr_cols.has("OBS_VALID_BEG", index)) { - shc.set_obs_valid_beg(timestring_to_unix(hdr_vals[index].c_str())); - } - else { - shc.set_obs_valid_beg(obs_valid_beg); - } - if(hdr_cols.has("OBS_VALID_END", index)) { - shc.set_obs_valid_end(timestring_to_unix(hdr_vals[index].c_str())); - } - else { - shc.set_obs_valid_end(obs_valid_end); - } + shc.set_obs_valid_beg(obs_valid_beg); + shc.set_obs_valid_end(obs_valid_end); // FCST_VAR - shc.set_fcst_var( - get_shc_str(cur_case, case_cols, case_vals, hdr_cols, hdr_vals, - "FCST_VAR", fcst_var, false)); + shc.set_fcst_var(get_col_css(cur_case, "FCST_VAR", fcst_var, false)); // FCST_UNITS - shc.set_fcst_units( - get_shc_str(cur_case, case_cols, case_vals, hdr_cols, hdr_vals, - "FCST_UNITS", fcst_units, false)); + shc.set_fcst_units(get_col_css(cur_case, "FCST_UNITS", fcst_units, false)); // FCST_LEV - shc.set_fcst_lev( - get_shc_str(cur_case, case_cols, case_vals, hdr_cols, hdr_vals, - "FCST_LEV", fcst_lev, false).c_str()); + shc.set_fcst_lev(get_col_css(cur_case, "FCST_LEV", fcst_lev, false).c_str()); // OBS_VAR - shc.set_obs_var( - get_shc_str(cur_case, case_cols, case_vals, hdr_cols, hdr_vals, - "OBS_VAR", obs_var, false)); + shc.set_obs_var(get_col_css(cur_case, "OBS_VAR", obs_var, false)); // OBS_UNITS - shc.set_obs_units( - get_shc_str(cur_case, case_cols, case_vals, hdr_cols, hdr_vals, - "OBS_UNITS", obs_units, false)); + shc.set_obs_units(get_col_css(cur_case, "OBS_UNITS", obs_units, false)); // OBS_LEV - shc.set_obs_lev( - get_shc_str(cur_case, case_cols, case_vals, hdr_cols, hdr_vals, - "OBS_LEV", obs_lev, false).c_str()); + shc.set_obs_lev(get_col_css(cur_case, "OBS_LEV", obs_lev, false).c_str()); // OBTYPE - shc.set_obtype( - get_shc_str(cur_case, case_cols, case_vals, hdr_cols, hdr_vals, - "OBTYPE", obtype, false).c_str()); + shc.set_obtype(get_col_css(cur_case, "OBTYPE", obtype, false).c_str()); // VX_MASK - shc.set_mask( - get_shc_str(cur_case, case_cols, case_vals, hdr_cols, hdr_vals, - "VX_MASK", vx_mask, false).c_str()); + shc.set_mask(get_col_css(cur_case, "VX_MASK", vx_mask, false).c_str()); // INTERP_MTHD - shc.set_interp_mthd( - get_shc_str(cur_case, case_cols, case_vals, hdr_cols, hdr_vals, - "INTERP_MTHD", interp_mthd, true)); + shc.set_interp_mthd(get_col_css(cur_case, "INTERP_MTHD", interp_mthd, true)); // INTERP_PNTS css = write_css(interp_pnts); @@ -485,28 +429,21 @@ StatHdrColumns StatHdrInfo::get_shc(const ConcatString &cur_case, else { wdth = nint(sqrt(interp_pnts[0])); } - - if(hdr_cols.has("INTERP_PNTS", index)) { - wdth = nint(sqrt(atof(hdr_vals[index].c_str()))); - } shc.set_interp_wdth(wdth); // FCST_THRESH ta.clear(); - ta.add_css(get_shc_str(cur_case, case_cols, case_vals, hdr_cols, hdr_vals, - "FCST_THRESH", fcst_thresh, true).c_str()); + ta.add_css(get_col_css(cur_case, "FCST_THRESH", fcst_thresh, true).c_str()); shc.set_fcst_thresh(ta); // OBS_THRESH ta.clear(); - ta.add_css(get_shc_str(cur_case, case_cols, case_vals, hdr_cols, hdr_vals, - "OBS_THRESH", obs_thresh, true).c_str()); + ta.add_css(get_col_css(cur_case, "OBS_THRESH", obs_thresh, true).c_str()); shc.set_obs_thresh(ta); // COV_THRESH ta.clear(); - ta.add_css(get_shc_str(cur_case, case_cols, case_vals, hdr_cols, hdr_vals, - "COV_THRESH", cov_thresh, true).c_str()); + ta.add_css(get_col_css(cur_case, "COV_THRESH", cov_thresh, true).c_str()); shc.set_cov_thresh(ta); // ALPHA @@ -521,34 +458,27 @@ StatHdrColumns StatHdrInfo::get_shc(const ConcatString &cur_case, else { out_alpha = alpha[0]; } - - if(hdr_cols.has("ALPHA", index)) { - out_alpha = atof(hdr_vals[index].c_str()); - } shc.set_alpha(out_alpha); // LINE_TYPE shc.set_line_type(statlinetype_to_string(lt)); + // Apply the -set_hdr options + StringArray case_vals = cur_case.split(":"); + shc.apply_set_hdr_opts(hdr_cols, hdr_vals, case_cols, case_vals); + return shc; } //////////////////////////////////////////////////////////////////////// -ConcatString StatHdrInfo::get_shc_str(const ConcatString &cur_case, - const StringArray &case_cols, - const StringArray &case_vals, - const StringArray &hdr_cols, - const StringArray &hdr_vals, +ConcatString StatHdrInfo::get_col_css(const ConcatString &cur_case, const char *col_name, const StringArray &col_vals, - bool warning) { - ConcatString css, shc_str; - int hdr_index, case_index; - + bool warning) const { // Build comma-separated list of column values - css = write_css(col_vals); + ConcatString css(write_css(col_vals)); // Check for multiple entries if(col_vals.n() > 1) { @@ -560,28 +490,7 @@ ConcatString StatHdrInfo::get_shc_str(const ConcatString &cur_case, else mlog << Debug(2) << msg; } - // Check the header options. - if(hdr_cols.has(col_name, hdr_index)) { - - // Check for the full CASE string. - if(case_str.compare(hdr_vals[hdr_index]) == 0) { - shc_str = cur_case; - } - // Check for one of the case columns. - else if(case_cols.has(hdr_vals[hdr_index], case_index)) { - shc_str = case_vals[case_index]; - } - // Otherwise, use the constant header string. - else { - shc_str = hdr_vals[hdr_index]; - } - } - // Otherwise, use the comma-separated list of values. - else { - shc_str = css; - } - - return shc_str; + return css; } //////////////////////////////////////////////////////////////////////// diff --git a/src/tools/core/stat_analysis/aggr_stat_line.h b/src/tools/core/stat_analysis/aggr_stat_line.h index a934491a1f..e79c0155eb 100644 --- a/src/tools/core/stat_analysis/aggr_stat_line.h +++ b/src/tools/core/stat_analysis/aggr_stat_line.h @@ -80,14 +80,10 @@ struct StatHdrInfo { const StringArray &hdr_cols, const StringArray &hdr_vals, const STATLineType lt); - ConcatString get_shc_str(const ConcatString &cur_case, - const StringArray &case_cols, - const StringArray &case_vals, - const StringArray &hdr_cols, - const StringArray &hdr_vals, + ConcatString get_col_css(const ConcatString &cur_case, const char *col_name, const StringArray &col_vals, - bool warning); + bool warning) const; }; struct AggrSummaryInfo { diff --git a/src/tools/other/gsi_tools/gsi_util.cc b/src/tools/other/gsi_tools/gsi_util.cc index 5381ebe2d1..82b4000ae9 100644 --- a/src/tools/other/gsi_tools/gsi_util.cc +++ b/src/tools/other/gsi_tools/gsi_util.cc @@ -301,152 +301,35 @@ bool is_retr(const char *s) { void setup_header(StatHdrColumns &shc, const StringArray &name, const StringArray &value, const char *default_line_type) { - int index; - SingleThresh st; - - // MODEL - if(name.has("MODEL", index)) { - shc.set_model(value[index].c_str()); - } - else { - shc.set_model(default_model); - } - - // DESC - if(name.has("DESC", index)) { - shc.set_desc(value[index].c_str()); - } - else { - shc.set_desc(default_desc); - } - - // FCST_LEAD - if(name.has("FCST_LEAD", index)) { - shc.set_fcst_lead_sec(timestring_to_sec(value[index].c_str())); - } - else { - shc.set_fcst_lead_sec(default_lead); - } - - // FCST_VALID_BEG, FCST_VALID_END - if(name.has("FCST_VALID_BEG", index)) { - shc.set_fcst_valid_beg(timestring_to_unix(value[index].c_str())); - not_has_FCST_VALID_BEG = false; - } - if(name.has("FCST_VALID_END", index)) { - shc.set_fcst_valid_end(timestring_to_unix(value[index].c_str())); - not_has_FCST_VALID_END = false; - } - - // OBS_LEAD - if(name.has("OBS_LEAD", index)) { - shc.set_obs_lead_sec(timestring_to_sec(value[index].c_str())); - } - else { - shc.set_obs_lead_sec(default_lead); - } - - // OBS_VALID_BEG, OBS_VALID_END - if(name.has("OBS_VALID_BEG", index)) { - shc.set_obs_valid_beg(timestring_to_unix(value[index].c_str())); - not_has_OBS_VALID_BEG = false; - } - if(name.has("OBS_VALID_END", index)) { - shc.set_obs_valid_end(timestring_to_unix(value[index].c_str())); - not_has_OBS_VALID_END = false; - } - - // FCST_VAR - if(name.has("FCST_VAR", index)) { - shc.set_fcst_var(value[index]); - not_has_FCST_VAR = false; - } - - // FCST_LEV - if(name.has("FCST_LEV", index)) { - shc.set_fcst_lev(value[index].c_str()); - } - else { - shc.set_fcst_lev(default_lev); - } - - // OBS_VAR - if(name.has("OBS_VAR", index)) { - shc.set_obs_var(value[index]); - not_has_OBS_VAR = false; - } - - // OBS_LEV - if(name.has("OBS_LEV", index)) { - shc.set_obs_lev(value[index].c_str()); - } - else { - shc.set_obs_lev(default_lev); - } - - // OBTYPE - if(name.has("OBTYPE", index)) { - shc.set_obtype(value[index].c_str()); - not_has_OBTYPE = false; - } - else { - shc.set_obtype(default_obtype); - } - - // VX_MASK - if(name.has("VX_MASK", index)) { - shc.set_mask(value[index].c_str()); - } - else { - shc.set_mask(default_vx_mask); - } - - // INTERP_MTHD - if(name.has("INTERP_MTHD", index)) { - shc.set_interp_mthd(value[index]); - } - else { - shc.set_interp_mthd((string)default_interp_mthd); - } - - // INTERP_PNTS - if(name.has("INTERP_PNTS", index)) { - shc.set_interp_wdth(nint(sqrt(atof(value[index].c_str())))); - } - else { - shc.set_interp_wdth(default_interp_wdth); - } - - // FCST_THRESH - if(name.has("FCST_THRESH", index)) st.set(value[index].c_str()); - else st.set(default_thresh); + SingleThresh st(na_str); + + // Initialize the header + shc.set_model("GSI"); + shc.set_desc(na_str); + shc.set_fcst_lead_sec(0); + shc.set_fcst_valid_beg(0); + shc.set_fcst_valid_end(0); + shc.set_obs_lead_sec(0); + shc.set_obs_valid_beg(0); + shc.set_obs_valid_end(0); + shc.set_fcst_var(na_str); + shc.set_fcst_units(na_str); + shc.set_fcst_lev(na_str); + shc.set_obs_var(na_str); + shc.set_obs_units(na_str); + shc.set_obs_lev(na_str); + shc.set_obtype(na_str); + shc.set_mask(na_str); + shc.set_interp_mthd(na_str); + shc.set_interp_wdth(0); shc.set_fcst_thresh(st); - - // OBS_THRESH - if(name.has("OBS_THRESH", index)) st.set(value[index].c_str()); - else st.set(default_thresh); shc.set_obs_thresh(st); - - // COV_THRESH - if(name.has("COV_THRESH", index)) st.set(value[index].c_str()); - else st.set(default_thresh); shc.set_cov_thresh(st); + shc.set_alpha(bad_data_double); + shc.set_line_type(default_line_type); - // ALPHA - if(name.has("ALPHA", index)) { - shc.set_alpha(atof(value[index].c_str())); - } - else { - shc.set_alpha(default_alpha); - } - - // LINE_TYPE - if(name.has("LINE_TYPE", index)) { - shc.set_line_type(value[index].c_str()); - } - else { - shc.set_line_type(default_line_type); - } + // Apply the -set_hdr options + shc.apply_set_hdr_opts(name, value); return; } diff --git a/src/tools/other/gsi_tools/gsi_util.h b/src/tools/other/gsi_tools/gsi_util.h index b635884ed2..dfd4390247 100644 --- a/src/tools/other/gsi_tools/gsi_util.h +++ b/src/tools/other/gsi_tools/gsi_util.h @@ -18,21 +18,9 @@ //////////////////////////////////////////////////////////////////////// // Constants -static const char default_model[] = "GSI"; -static const char default_desc[] = "NA"; -static const int default_lead = 0; -static const char default_lev[] = "NA"; -static const char default_obtype[] = "NA"; -static const char default_vx_mask[] = "NA"; -static const char default_interp_mthd[] = "NA"; -static const int default_interp_wdth = 0; -static const char default_thresh[] = "NA"; -static const double default_alpha = bad_data_double; - -static const int bad_setup_qc = -999; -static const char key_sep[] = ":"; - -static const char conv_id_str[] = "conv"; +static const int bad_setup_qc = -999; +static const char key_sep[] = ":"; +static const char conv_id_str[] = "conv"; static const char *micro_id_str [] = { "amsua", "amsub", "mhs", @@ -83,16 +71,6 @@ struct RadData { //////////////////////////////////////////////////////////////////////// -static bool not_has_FCST_VALID_BEG = true; -static bool not_has_FCST_VALID_END = true; -static bool not_has_OBS_VALID_BEG = true; -static bool not_has_OBS_VALID_END = true; -static bool not_has_FCST_VAR = true; -static bool not_has_OBS_VAR = true; -static bool not_has_OBTYPE = true; - -//////////////////////////////////////////////////////////////////////// - ConvData parse_conv_data(const ConvRecord &r, const int i); RadData parse_rad_data (const RadRecord &r, const int i, const int chval, const int use); diff --git a/src/tools/other/gsi_tools/gsid2mpr.cc b/src/tools/other/gsi_tools/gsid2mpr.cc index 8915b73ce6..579d41bd19 100644 --- a/src/tools/other/gsi_tools/gsid2mpr.cc +++ b/src/tools/other/gsi_tools/gsid2mpr.cc @@ -47,7 +47,6 @@ using namespace std; - //////////////////////////////////////////////////////////////////////// static void process_conv(const char *conv_filename, const char *output_filename); @@ -365,13 +364,13 @@ void write_mpr_row_conv(AsciiTable &at, int row, const ConvData &d) { int col; // Update header for current data - if(not_has_FCST_VALID_BEG) shc.set_fcst_valid_beg(d.fcst_ut); - if(not_has_FCST_VALID_END) shc.set_fcst_valid_end(d.fcst_ut); - if(not_has_OBS_VALID_BEG) shc.set_obs_valid_beg(d.obs_ut); - if(not_has_OBS_VALID_END) shc.set_obs_valid_end(d.obs_ut); - if(not_has_FCST_VAR) shc.set_fcst_var(d.var); - if(not_has_OBS_VAR) shc.set_obs_var(d.var); - if(not_has_OBTYPE) shc.set_obtype(d.obtype.c_str()); + if(!hdr_name.has("FCST_VALID_BEG")) shc.set_fcst_valid_beg(d.fcst_ut); + if(!hdr_name.has("FCST_VALID_END")) shc.set_fcst_valid_end(d.fcst_ut); + if(!hdr_name.has("OBS_VALID_BEG")) shc.set_obs_valid_beg(d.obs_ut); + if(!hdr_name.has("OBS_VALID_END")) shc.set_obs_valid_end(d.obs_ut); + if(!hdr_name.has("FCST_VAR")) shc.set_fcst_var(d.var); + if(!hdr_name.has("OBS_VAR")) shc.set_obs_var(d.var); + if(!hdr_name.has("OBTYPE")) shc.set_obtype(d.obtype.c_str()); // Write header columns write_header_cols(shc, at, row); @@ -420,12 +419,12 @@ void write_mpr_row_rad(AsciiTable &at, int row, const RadData & d) { int col; // Update header for current data - if(not_has_FCST_VALID_BEG) shc.set_fcst_valid_beg(d.fcst_ut); - if(not_has_FCST_VALID_END) shc.set_fcst_valid_end(d.fcst_ut); - if(not_has_OBS_VALID_BEG) shc.set_obs_valid_beg(d.obs_ut); - if(not_has_OBS_VALID_END) shc.set_obs_valid_end(d.obs_ut); - if(not_has_FCST_VAR) shc.set_fcst_var(d.var); - if(not_has_OBS_VAR) shc.set_obs_var(d.var); + if(!hdr_name.has("FCST_VALID_BEG")) shc.set_fcst_valid_beg(d.fcst_ut); + if(!hdr_name.has("FCST_VALID_END")) shc.set_fcst_valid_end(d.fcst_ut); + if(!hdr_name.has("OBS_VALID_BEG")) shc.set_obs_valid_beg(d.obs_ut); + if(!hdr_name.has("OBS_VALID_END")) shc.set_obs_valid_end(d.obs_ut); + if(!hdr_name.has("FCST_VAR")) shc.set_fcst_var(d.var); + if(!hdr_name.has("OBS_VAR")) shc.set_obs_var(d.var); // Write header columns write_header_cols(shc, at, row); diff --git a/src/tools/tc_utils/tc_stat/tc_stat.cc b/src/tools/tc_utils/tc_stat/tc_stat.cc index 28025107aa..a5592c7b15 100644 --- a/src/tools/tc_utils/tc_stat/tc_stat.cc +++ b/src/tools/tc_utils/tc_stat/tc_stat.cc @@ -22,6 +22,7 @@ // 004 07/06/22 Howard Soh METplus-Internal #19 Rename main to met_main // 005 09/28/22 Prestopnik MET #2227 Remove namespace std from header files // 006 10/06/22 Halley Gotway MET #392 Incorporate diagnostics +// 007 06/14/24 Halley Gotway MET #2911 Support -set_hdr job command option // //////////////////////////////////////////////////////////////////////// diff --git a/src/tools/tc_utils/tc_stat/tc_stat_job.cc b/src/tools/tc_utils/tc_stat/tc_stat_job.cc index ec8172776c..2bfca419fd 100644 --- a/src/tools/tc_utils/tc_stat/tc_stat_job.cc +++ b/src/tools/tc_utils/tc_stat/tc_stat_job.cc @@ -184,6 +184,8 @@ void TCStatJob::init_from_scratch() { ValidMask.set_ignore_case(1); LineType.set_ignore_case(1); TrackWatchWarn.set_ignore_case(1); + ByColumn.set_ignore_case(1); + HdrName.set_ignore_case(1); clear(); @@ -230,6 +232,10 @@ void TCStatJob::clear() { EventEqualLead.clear(); EventEqualCases.clear(); + ByColumn.clear(); + HdrName.clear(); + HdrValue.clear(); + DumpFile.clear(); close_dump_file(); JobOut = (ofstream *) nullptr; @@ -315,6 +321,11 @@ void TCStatJob::assign(const TCStatJob & j) { InitDiagThreshMap = j.InitDiagThreshMap; PrintDiagWarning = j.PrintDiagWarning; + ByColumn = j.ByColumn; + + HdrName = j.HdrName; + HdrValue = j.HdrValue; + DumpFile = j.DumpFile; open_dump_file(); @@ -514,6 +525,15 @@ void TCStatJob::dump(ostream & out, int depth) const { out << prefix << "OutValidMask = " << (OutValidMaskName.nonempty() ? OutValidMaskName.text() : na_str) << "\n"; + out << prefix << "ByColumn ...\n"; + ByColumn.dump(out, depth + 1); + + out << prefix << "HdrName ...\n"; + HdrName.dump(out, depth + 1); + + out << prefix << "HdrValue ...\n"; + HdrValue.dump(out, depth + 1); + out << prefix << "DumpFile = " << (DumpFile.nonempty() ? DumpFile.text() : na_str) << "\n"; out << prefix << "StatFile = " << (StatFile.nonempty() ? StatFile.text() : na_str) << "\n"; @@ -1086,6 +1106,9 @@ StringArray TCStatJob::parse_job_command(const char *jobstring) { else if(c.compare("-event_equal_lead" ) == 0) { EventEqualLead.add_css_sec(a[i+1].c_str()); a.shift_down(i, 1); } else if(c.compare("-out_init_mask" ) == 0) { set_out_init_mask(a[i+1].c_str()); a.shift_down(i, 1); } else if(c.compare("-out_valid_mask" ) == 0) { set_out_valid_mask(a[i+1].c_str()); a.shift_down(i, 1); } + else if(c.compare("-by" ) == 0) { ByColumn.add_css(to_upper(a[i+1])); a.shift_down(i, 1); } + else if(c.compare("-set_hdr" ) == 0) { HdrName.add(to_upper(a[i+1])); + HdrValue.add(a[i+2]); a.shift_down(i, 2); } else if(c.compare("-dump_row" ) == 0) { DumpFile = a[i+1]; open_dump_file(); a.shift_down(i, 1); } else if(c.compare("-out_stat" ) == 0) { StatFile = a[i+1]; open_stat_file(); a.shift_down(i, 1); } else { b.add(a[i]); } @@ -1206,7 +1229,8 @@ void TCStatJob::close_stat_file() { //////////////////////////////////////////////////////////////////////// -void TCStatJob::dump_pair(const TrackPairInfo &pair, ofstream *out) { +void TCStatJob::dump_pair(const TrackPairInfo &pair, ofstream *out, + bool do_set_hdr) const { if(!out || pair.n_points() == 0) return; @@ -1258,7 +1282,8 @@ void TCStatJob::dump_pair(const TrackPairInfo &pair, ofstream *out) { // Write the TrackPairInfo object i_row = hdr_row; - write_track_pair_info(tchc, pair, out_at, i_row); + if(do_set_hdr) write_track_pair_info(tchc, pair, out_at, i_row, HdrName, HdrValue); + else write_track_pair_info(tchc, pair, out_at, i_row); // Write the AsciiTable to the file *out << out_at; @@ -1268,11 +1293,23 @@ void TCStatJob::dump_pair(const TrackPairInfo &pair, ofstream *out) { //////////////////////////////////////////////////////////////////////// -void TCStatJob::dump_line(const TCStatLine &line, ofstream *out) { +void TCStatJob::dump_line(const TCStatLine &line, ofstream *out, + bool do_set_hdr) const { if(!out) return; - *out << line; + // Apply -set_hdr options, if requested + if(do_set_hdr) { + TCStatLine line_set_hdr = line; + for(int i=0; i 0) s << "-dump_row " << DumpFile << " "; if(StatFile.length() > 0) @@ -1851,7 +1892,7 @@ void TCStatJobFilter::filter_tracks(TCPointCounts &n) { mlog << Debug(4) << "Processing track pair: " << pair.case_info() << "\n"; - if(DumpOut) dump_pair(pair, DumpOut); + if(DumpOut) dump_pair(pair, DumpOut, true); } } // end while } // end else @@ -1888,7 +1929,7 @@ void TCStatJobFilter::filter_lines(TCPointCounts &n) { // Check if this line should be kept if(!is_keeper_line(line, n)) continue; - if(DumpOut) dump_line(line, DumpOut); + if(DumpOut) dump_line(line, DumpOut, true); } // end while } // end else @@ -1955,7 +1996,6 @@ void TCStatJobSummary::init_from_scratch() { // Ignore case when performing comparisons ReqColumn.set_ignore_case(1); Column.set_ignore_case(1); - ByColumn.set_ignore_case(1); clear(); @@ -1972,7 +2012,6 @@ void TCStatJobSummary::clear() { ReqColumn.clear(); Column.clear(); - ByColumn.clear(); SummaryMap.clear(); // Set to default value @@ -1992,7 +2031,6 @@ void TCStatJobSummary::assign(const TCStatJobSummary & j) { ReqColumn = j.ReqColumn; Column = j.Column; ColumnUnion = j.ColumnUnion; - ByColumn = j.ByColumn; SummaryMap = j.SummaryMap; OutAlpha = j.OutAlpha; FSPThresh = j.FSPThresh; @@ -2026,7 +2064,6 @@ StringArray TCStatJobSummary::parse_job_command(const char *jobstring) { if(c.compare("-column" ) == 0) { ReqColumn.add_css(to_upper(a[i+1])); add_column(a[i+1].c_str()); a.shift_down(i, 1); } else if(c.compare("-column_union") == 0) { ColumnUnion = string_to_bool(a[i+1].c_str()); a.shift_down(i, 1); } - else if(c.compare("-by" ) == 0) { ByColumn.add_css(to_upper(a[i+1])); a.shift_down(i, 1); } else if(c.compare("-out_alpha" ) == 0) { OutAlpha = atof(a[i+1].c_str()); a.shift_down(i, 1); } else if(c.compare("-fsp_thresh" ) == 0) { FSPThresh.set(a[i+1].c_str()); a.shift_down(i, 1); } else { b.add(a[i]); } @@ -2090,8 +2127,6 @@ ConcatString TCStatJobSummary::serialize() const { s << "-column " << ReqColumn[i] << " "; if(ColumnUnion != default_column_union) s << "-column_union " << bool_to_string(ColumnUnion) << " "; - for(i=0; isecond.InitMask); sa.add_uniq(it->second.ValidMask); - + // Use the basin names instead if(sa.n() == 1 && sa[0] == na_str) { sa.clear(); @@ -3889,16 +3917,21 @@ void TCStatJobRIRW::do_stat_output(ostream &out) { // c = 0; + // Split the current map key, eliminating the job name in the first entry + StringArray ByValue = it->first.split(":"); + ByValue.shift_down(0, 1); + // // CTC output line // if(OutLineType.has(stat_ctc_str)) { shc.set_alpha(bad_data_double); shc.set_line_type(stat_ctc_str); + shc.apply_set_hdr_opts(HdrName, HdrValue, ByColumn, ByValue); write_header_cols(shc, stat_at, stat_row); write_ctc_cols(it->second.Info, stat_at, stat_row++, n_header_columns); } - + // // CTS output line // @@ -3910,12 +3943,12 @@ void TCStatJobRIRW::do_stat_output(ostream &out) { it->second.Info.allocate_n_alpha(1); it->second.Info.alpha[0] = OutAlpha; shc.set_alpha(OutAlpha); - + // // Compute the stats and confidence intervals for this // CTSInfo object // - + it->second.Info.compute_stats(); it->second.Info.compute_ci(); @@ -3923,14 +3956,15 @@ void TCStatJobRIRW::do_stat_output(ostream &out) { // Write the data line // shc.set_line_type(stat_cts_str); + shc.apply_set_hdr_opts(HdrName, HdrValue, ByColumn, ByValue); write_header_cols(shc, stat_at, stat_row); write_cts_cols(it->second.Info, 0, stat_at, stat_row++, n_header_columns); } } // end for it - + // Write the table out << stat_at << "\n" << flush; - + return; } @@ -3981,9 +4015,6 @@ void TCStatJobProbRIRW::init_from_scratch() { TCStatJob::init_from_scratch(); - // Ignore case when performing comparisons - ByColumn.set_ignore_case(1); - clear(); return; @@ -3997,7 +4028,6 @@ void TCStatJobProbRIRW::clear() { JobType = TCStatJobType::ProbRIRW; - ByColumn.clear(); ProbRIRWMap.clear(); // Set to default values @@ -4023,7 +4053,6 @@ void TCStatJobProbRIRW::assign(const TCStatJobProbRIRW & j) { ProbRIRWExact = j.ProbRIRWExact; ProbRIRWBDeltaThresh = j.ProbRIRWBDeltaThresh; ProbRIRWProbThresh = j.ProbRIRWProbThresh; - ByColumn = j.ByColumn; MaxNThresh = j.MaxNThresh; NDumpLines = j.NDumpLines; OutAlpha = j.OutAlpha; @@ -4060,8 +4089,7 @@ StringArray TCStatJobProbRIRW::parse_job_command(const char *jobstring) { } // Check job command options - if(c.compare("-by" ) == 0) { ByColumn.add_css(to_upper(a[i+1])); a.shift_down(i, 1); } - else if(c.compare("-out_alpha" ) == 0) { OutAlpha = atof(a[i+1].c_str()); a.shift_down(i, 1); } + if(c.compare("-out_alpha" ) == 0) { OutAlpha = atof(a[i+1].c_str()); a.shift_down(i, 1); } else if(c.compare("-out_line_type" ) == 0) { OutLineType.add_css(to_upper(a[i+1])); a.shift_down(i, 1); } else if(c.compare("-probrirw_exact" ) == 0) { ProbRIRWExact = string_to_bool(a[i+1].c_str()); a.shift_down(i, 1); } else if(c.compare("-probrirw_bdelta_thresh") == 0) { ProbRIRWBDeltaThresh.set(a[i+1].c_str()); a.shift_down(i, 1); } @@ -4157,9 +4185,6 @@ ConcatString TCStatJobProbRIRW::serialize() const { s << "-probrirw_prob_thresh " << prob_thresh_to_string(ProbRIRWProbThresh) << " "; // Add ProbRIRW job-specific options - for(i=0; i InitDiagThreshMap; StringArray PrintDiagWarning; + // Store the case information + StringArray ByColumn; + + // Options for -set_hdr output + StringArray HdrName; + StringArray HdrValue; + // Variables to the store the analysis job specification ConcatString DumpFile; // Dump TrackPairInfo used to a file std::ofstream *DumpOut; // Dump output file stream @@ -448,9 +457,6 @@ class TCStatJobSummary : public TCStatJob { StringArray Column; bool ColumnUnion; - // Store the case information - StringArray ByColumn; - // Confidence interval alpha value double OutAlpha; @@ -504,9 +510,6 @@ class TCStatJobRIRW : public TCStatJob { void do_cts_output (std::ostream &); void do_mpr_output (std::ostream &); void do_stat_output(std::ostream &); - - // Store the case information - StringArray ByColumn; // Confidence interval alpha value double OutAlpha; @@ -555,9 +558,6 @@ class TCStatJobProbRIRW : public TCStatJob { SingleThresh ProbRIRWBDeltaThresh; // Threshold the BEST track change ThreshArray ProbRIRWProbThresh; // Array of probabilities for PCT bins - // Store the case information - StringArray ByColumn; - // Maximum number of thresholds encountered int MaxNThresh; int NDumpLines;