From 7063744d421edce58395e0312a99b359ffc9175c Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Wed, 13 Sep 2023 10:46:54 -0600 Subject: [PATCH] Feature #2609 tc_diag_missing_data (#2680) Co-authored-by: John Halley Gotway Co-authored-by: Seth Linden Co-authored-by: jprestop Co-authored-by: Daniel Adriaansen Co-authored-by: John and Cindy Co-authored-by: Howard Soh Co-authored-by: George McCabe <23407799+georgemccabe@users.noreply.github.com> Co-authored-by: hsoh-u Co-authored-by: MET Tools Test Account Co-authored-by: Seth Linden Co-authored-by: lisagoodrich <33230218+lisagoodrich@users.noreply.github.com> Co-authored-by: davidalbo Co-authored-by: Lisa Goodrich Co-authored-by: metplus-bot <97135045+metplus-bot@users.noreply.github.com> Co-authored-by: j-opatz <59586397+j-opatz@users.noreply.github.com> Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: Jonathan Vigh Co-authored-by: Tracy Hertneky <39317287+hertneky@users.noreply.github.com> Fix Python environment issue (#2407) fix definitions of G172 and G220 based on comments in NOAA-EMC/NCEPLIBS-w3emc#157. (#2406) fix #2380 develop override (#2382) fix #2408 develop empty config (#2410) fix #2390 develop compile zlib (#2404) fix #2412 develop climo (#2422) fix #2437 develop convert (#2439) fix for develop, for #2437, forgot one reference to the search_parent for a dictionary lookup. fix #2452 develop airnow (#2454) fix #2449 develop pdf (#2464) fix #2402 develop sonarqube (#2468) fix #2426 develop buoy (#2475) fix 2518 dtypes appf docs (#2519) fix 2531 compilation errors (#2533) fix #2531 compilation_errors_configure (#2535) fix 2596 main v11.1 rpath compilation (#2614) fix #2514 main_v11.1 clang (#2628) --- data/config/TCDiagConfig_default | 12 +- docs/Users_Guide/config_options_tc.rst | 8 +- docs/Users_Guide/tc-diag.rst | 23 +++- internal/test_unit/config/TCDiagConfig_ian | 12 +- src/basic/vx_config/config_constants.h | 15 ++- src/basic/vx_log/logger.cc | 39 +++++- src/basic/vx_log/logger.h | 5 + src/libcode/vx_data2d/data_class.cc | 43 +++++++ src/libcode/vx_data2d/data_class.h | 5 + src/libcode/vx_data2d_grib/data2d_grib.cc | 2 + src/libcode/vx_data2d_grib/data2d_grib.h | 8 +- src/libcode/vx_data2d_grib2/data2d_grib2.cc | 15 +-- .../vx_data2d_nc_pinterp/data2d_nc_pinterp.cc | 2 + src/libcode/vx_data2d_nccf/data2d_nccf.h | 1 - src/libcode/vx_data2d_python/data2d_python.cc | 1 - src/libcode/vx_series_data/series_data.cc | 121 ++++++++++++++++-- src/libcode/vx_series_data/series_data.h | 14 +- src/tools/tc_utils/tc_diag/tc_diag.cc | 112 +++++++++++++--- src/tools/tc_utils/tc_diag/tc_diag.h | 2 + .../tc_utils/tc_diag/tc_diag_conf_info.cc | 5 + .../tc_utils/tc_diag/tc_diag_conf_info.h | 5 +- 21 files changed, 391 insertions(+), 59 deletions(-) diff --git a/data/config/TCDiagConfig_default b/data/config/TCDiagConfig_default index f81ccdf86d..f4b489302e 100644 --- a/data/config/TCDiagConfig_default +++ b/data/config/TCDiagConfig_default @@ -45,9 +45,14 @@ valid_exc = []; // // Subset by the valid hour and lead time. +// By default, compute diagnostics every 6 hours out to 126 hours. // valid_hour = []; -lead = []; +lead = [ "0", "6", "12", "18", "24", + "30", "36", "42", "48", "54", + "60", "66", "72", "78", "84", + "90", "96", "102", "108", "114", + "120", "126" ]; //////////////////////////////////////////////////////////////////////////////// @@ -134,6 +139,11 @@ vortex_removal = FALSE; //////////////////////////////////////////////////////////////////////////////// +// +// Flag to control input files +// +one_time_per_file_flag = TRUE; + // // Flags to control output files // diff --git a/docs/Users_Guide/config_options_tc.rst b/docs/Users_Guide/config_options_tc.rst index 1d6d098ad8..f3c1592544 100644 --- a/docs/Users_Guide/config_options_tc.rst +++ b/docs/Users_Guide/config_options_tc.rst @@ -108,7 +108,7 @@ For example: valid_beg end inc exc --------------------- -Specify a model valid time window YYYYMMDD[_HH[MMSS]] format or provide a +Specify a model valid time window in YYYYMMDD[_HH[MMSS]] format or provide a list of specific valid times to include (inc) or exclude (exc). If a time window is specified, only tracks for which all points are contained within the window will be used. If valid times to include or exclude are specified, @@ -1077,7 +1077,7 @@ Statistics will be generated separately for each ATCF ID. init_beg, init_end ^^^^^^^^^^^^^^^^^^ -Forecast and operational initialization time window +Forecast and operational initialization time window, as strings in YYYYMMDD[_HH[MMSS]] format .. code-block:: none @@ -1087,7 +1087,7 @@ Forecast and operational initialization time window valid_beg, valid_end ^^^^^^^^^^^^^^^^^^^^ -Forecast, BEST, and operational valid time window +Forecast, BEST, and operational valid time window, as strings in YYYYMMDD[_HH[MMSS]] format .. code-block:: none @@ -1097,7 +1097,7 @@ Forecast, BEST, and operational valid time window lead ^^^^ -Forecast and operational lead times in hours +Forecast and operational lead times, as strings in HH[MMSS] format .. code-block:: none diff --git a/docs/Users_Guide/tc-diag.rst b/docs/Users_Guide/tc-diag.rst index 53c1ee1b0d..5ecfe649f0 100644 --- a/docs/Users_Guide/tc-diag.rst +++ b/docs/Users_Guide/tc-diag.rst @@ -94,12 +94,21 @@ Configuring input tracks and time valid_inc = []; valid_exc = []; valid_hour = []; - lead = []; The TC-Diag tool should be configured to filter the input track data (**-deck**) down to the subset of tracks that correspond to the gridded data files provided (**-data**). The filtered tracks should contain data for only *one initialization time* but may contain tracks for multiple models. The configuration options listed above are used to filter the input track data down to those that should be processed in the current run. These options are common to multiple MET tools and are described in :numref:`config_options_tc`. +.. code-block:: none + + lead = [ "0", "6", "12", "18", "24", + "30", "36", "42", "48", "54", + "60", "66", "72", "78", "84", + "90", "96", "102", "108", "114", + "120", "126" ]; + +The **lead** entry is an array of strings specifying lead times in HH[MMSS] format. By default, diagnostics are computed every 6 hours out to 126 hours. Lead times for which no track point or gridded model data exist produce a warning message and diagnostics set to a missing data value. + Configuring domain information ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -204,8 +213,16 @@ The **vortex_removal** flag entry is a boolean specifying whether or not vortex .. note:: As of MET version 11.1.0, vortex removal logic is not yet supported. -Configuring data output options -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Configuring data input and output options +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. code-block:: none + + one_time_per_file_flag = TRUE; + +The **one_time_per_file_flag** entry controls the logic for reading data from input files. This option describes how data is stored in the gridded input files specified with the **-data** command line option. Set this to true if each input file contains all of the data for a single initialization time and for a single valid time. If the input files contain data for multiple initialization or valid times, or if data for one valid time is spread across multiple files, set this to false. + +If true, all input fields are read efficiently from each file in a single call. If false, each field is processed separately in a less efficient manner. .. code-block:: none diff --git a/internal/test_unit/config/TCDiagConfig_ian b/internal/test_unit/config/TCDiagConfig_ian index 19f2fac84d..98e9fdd114 100644 --- a/internal/test_unit/config/TCDiagConfig_ian +++ b/internal/test_unit/config/TCDiagConfig_ian @@ -45,9 +45,14 @@ valid_exc = []; // // Subset by the valid hour and lead time. +// By default, compute diagnostics every 6 hours out to 126 hours. // valid_hour = []; -lead = [ "0", "6", "12", "18", "24" ]; +lead = [ "0", "6", "12", "18", "24", + "30", "36", "42", "48", "54", + "60", "66", "72", "78", "84", + "90", "96", "102", "108", "114", + "120", "126" ]; //////////////////////////////////////////////////////////////////////////////// @@ -128,6 +133,11 @@ vortex_removal = FALSE; //////////////////////////////////////////////////////////////////////////////// +// +// Flag to control input files +// +one_time_per_file_flag = TRUE; + // // Flags to control output files // diff --git a/src/basic/vx_config/config_constants.h b/src/basic/vx_config/config_constants.h index 34da078eb1..bd3ffe0219 100644 --- a/src/basic/vx_config/config_constants.h +++ b/src/basic/vx_config/config_constants.h @@ -1206,18 +1206,19 @@ static const char conf_key_radial_velocity_field_name[] = "radial_velocity_field static const char conf_key_tangential_velocity_field_name[] = "tangential_velocity_field_name"; static const char conf_key_radial_velocity_long_field_name[] = "radial_velocity_long_field_name"; static const char conf_key_tangential_velocity_long_field_name[] = "tangential_velocity_long_field_name"; -static const char conf_key_vortex_removal[] = "vortex_removal"; // // TC-Diag specific parameter key names // -static const char conf_key_domain_info[] = "domain_info"; -static const char conf_key_domain[] = "domain"; -static const char conf_key_diag_script[] = "diag_script"; -static const char conf_key_nc_rng_azi_flag[] = "nc_rng_azi_flag"; -static const char conf_key_nc_diag_flag[] = "nc_diag_flag"; -static const char conf_key_cira_diag_flag[] = "cira_diag_flag"; +static const char conf_key_domain_info[] = "domain_info"; +static const char conf_key_domain[] = "domain"; +static const char conf_key_diag_script[] = "diag_script"; +static const char conf_key_vortex_removal[] = "vortex_removal"; +static const char conf_key_one_time_per_file_flag[] = "one_time_per_file_flag"; +static const char conf_key_nc_rng_azi_flag[] = "nc_rng_azi_flag"; +static const char conf_key_nc_diag_flag[] = "nc_diag_flag"; +static const char conf_key_cira_diag_flag[] = "cira_diag_flag"; // // Parameter value names common to multiple tools diff --git a/src/basic/vx_log/logger.cc b/src/basic/vx_log/logger.cc index 5c531468d7..6e8785e041 100644 --- a/src/basic/vx_log/logger.cc +++ b/src/basic/vx_log/logger.cc @@ -37,8 +37,11 @@ static const int ErrorMessageLevel = -1; static const int WarningMessageLevel = 0; +static const bool DefaultPrintWarning = true; + static const bool DefaultExitOnWarning = false; + // // these need external linkage, do not make static or extern // @@ -217,6 +220,8 @@ LoggerWarning::LoggerWarning() { warn = WarningMessageLevel; + PrintWarning = DefaultPrintWarning; + ExitOnWarning = DefaultExitOnWarning; NeedToExit = false; @@ -246,6 +251,8 @@ LoggerWarning::LoggerWarning(const LoggerWarning & lw) { warn = lw.warn; + PrintWarning = lw.PrintWarning; + ExitOnWarning = lw.ExitOnWarning; NeedToExit = lw.NeedToExit; @@ -259,6 +266,8 @@ LoggerWarning & LoggerWarning::operator=(const LoggerWarning & lw) { warn = lw.warn; + PrintWarning = lw.PrintWarning; + ExitOnWarning = lw.ExitOnWarning; NeedToExit = lw.NeedToExit; @@ -491,6 +500,17 @@ void Logger::dump(ostream & dump_out, int depth) const ////////////////////////////////////////////////////////////////// +bool Logger::print_warning() const +{ + + return (Warning.PrintWarning); + +} + + +////////////////////////////////////////////////////////////////// + + void Logger::set_verbosity_level(const int i) { // @@ -509,6 +529,19 @@ void Logger::set_verbosity_level(const int i) ////////////////////////////////////////////////////////////////// +void Logger::set_print_warning(bool b) +{ + // + // if false, do not print warning messages + // + Warning.PrintWarning = b; + +} + + +////////////////////////////////////////////////////////////////// + + void Logger::set_exit_on_warning(bool b) { // @@ -519,7 +552,6 @@ void Logger::set_exit_on_warning(bool b) } - ////////////////////////////////////////////////////////////////// @@ -1604,7 +1636,10 @@ Logger & Logger::operator<<(const LoggerError e) Logger & Logger::operator<<(const LoggerWarning w) { - (*this) << level(WarningMessageLevel); + if (Warning.PrintWarning) + { + (*this) << level(WarningMessageLevel); + } if (Warning.ExitOnWarning) Warning.NeedToExit = true; diff --git a/src/basic/vx_log/logger.h b/src/basic/vx_log/logger.h index 529ffefc1b..3c431e53e4 100644 --- a/src/basic/vx_log/logger.h +++ b/src/basic/vx_log/logger.h @@ -115,6 +115,7 @@ class LoggerWarning { public: int warn; + bool PrintWarning; bool ExitOnWarning; bool NeedToExit; @@ -248,6 +249,8 @@ class Logger int verbosity_level() const; + bool print_warning() const; + bool is_open() const; // @@ -256,6 +259,8 @@ class Logger void set_verbosity_level(const int); + void set_print_warning(bool); + void set_exit_on_warning(bool); // diff --git a/src/libcode/vx_data2d/data_class.cc b/src/libcode/vx_data2d/data_class.cc index c9ab09ee7e..b44b94351e 100644 --- a/src/libcode/vx_data2d/data_class.cc +++ b/src/libcode/vx_data2d/data_class.cc @@ -309,6 +309,49 @@ return; //////////////////////////////////////////////////////////////////////// +int Met2dDataFile::data_planes(vector &vi_list, + vector &dp_list) + +{ + +int n_valid = 0; +DataPlane cur_dp; + + // + // Loop over requested VarInfo objects + // + +for(int i=0; i&, + std::vector&); + // post-process data after reading it bool process_data_plane(VarInfo *, DataPlane &); diff --git a/src/libcode/vx_data2d_grib/data2d_grib.cc b/src/libcode/vx_data2d_grib/data2d_grib.cc index 98720ea120..9ac709ef3a 100644 --- a/src/libcode/vx_data2d_grib/data2d_grib.cc +++ b/src/libcode/vx_data2d_grib/data2d_grib.cc @@ -827,3 +827,5 @@ int MetGrib1DataFile::index(VarInfo &vinfo){ return rec; } + +//////////////////////////////////////////////////////////////////////// diff --git a/src/libcode/vx_data2d_grib/data2d_grib.h b/src/libcode/vx_data2d_grib/data2d_grib.h index 563c09fb4f..ad14d6e8a3 100644 --- a/src/libcode/vx_data2d_grib/data2d_grib.h +++ b/src/libcode/vx_data2d_grib/data2d_grib.h @@ -84,10 +84,16 @@ class MetGrib1DataFile : public Met2dDataFile { int read_record( VarInfoGrib &); // returns match count (>=0), or -1 on error + // retrieve the first matching data plane + bool data_plane(VarInfo &, DataPlane &); + // retrieve all matching data planes + int data_plane_array(VarInfo &, DataPlaneArray &); - + + // retrieve the index of the first matching record + int index(VarInfo &); }; diff --git a/src/libcode/vx_data2d_grib2/data2d_grib2.cc b/src/libcode/vx_data2d_grib2/data2d_grib2.cc index 6a3f40574d..f86516c2b7 100644 --- a/src/libcode/vx_data2d_grib2/data2d_grib2.cc +++ b/src/libcode/vx_data2d_grib2/data2d_grib2.cc @@ -239,8 +239,8 @@ bool MetGrib2DataFile::data_plane(VarInfo &vinfo, DataPlane &plane) { //////////////////////////////////////////////////////////////////////// -int MetGrib2DataFile::data_plane_array( VarInfo &vinfo, - DataPlaneArray &plane_array ){ +int MetGrib2DataFile::data_plane_array(VarInfo &vinfo, + DataPlaneArray &plane_array) { // Initialize plane_array.clear(); @@ -357,10 +357,9 @@ int MetGrib2DataFile::data_plane_array( VarInfo &vinfo, //////////////////////////////////////////////////////////////////////// -void MetGrib2DataFile::find_record_matches( VarInfoGrib2* vinfo, - vector &listMatchExact, - vector &listMatchRange - ){ +void MetGrib2DataFile::find_record_matches(VarInfoGrib2* vinfo, + vector &listMatchExact, + vector &listMatchRange) { const char *method_name = "MetGrib2DataFile::find_record_matches() -> "; @@ -955,7 +954,7 @@ void MetGrib2DataFile::read_grib2_record_list() { //////////////////////////////////////////////////////////////////////// -void MetGrib2DataFile::read_grib2_grid( gribfield *gfld) { +void MetGrib2DataFile::read_grib2_grid(gribfield *gfld) { double d, r_km; int ResCompFlag; @@ -1416,7 +1415,7 @@ void MetGrib2DataFile::read_grib2_grid( gribfield *gfld) { //////////////////////////////////////////////////////////////////////// bool MetGrib2DataFile::read_grib2_record_data_plane(Grib2Record *rec, - DataPlane &plane) { + DataPlane &plane) { // attempt to read the record gribfield *gfld; diff --git a/src/libcode/vx_data2d_nc_pinterp/data2d_nc_pinterp.cc b/src/libcode/vx_data2d_nc_pinterp/data2d_nc_pinterp.cc index dc7d59be00..83872f4614 100644 --- a/src/libcode/vx_data2d_nc_pinterp/data2d_nc_pinterp.cc +++ b/src/libcode/vx_data2d_nc_pinterp/data2d_nc_pinterp.cc @@ -310,3 +310,5 @@ int MetNcPinterpDataFile::data_plane_array(VarInfo &vinfo, int MetNcPinterpDataFile::index(VarInfo &vinfo){ return -1; } + +//////////////////////////////////////////////////////////////////////// diff --git a/src/libcode/vx_data2d_nccf/data2d_nccf.h b/src/libcode/vx_data2d_nccf/data2d_nccf.h index d9a9a2df9c..e6ad78b1eb 100644 --- a/src/libcode/vx_data2d_nccf/data2d_nccf.h +++ b/src/libcode/vx_data2d_nccf/data2d_nccf.h @@ -92,7 +92,6 @@ class MetNcCFDataFile : public Met2dDataFile { int index(VarInfo &); - // // do stuff // diff --git a/src/libcode/vx_data2d_python/data2d_python.cc b/src/libcode/vx_data2d_python/data2d_python.cc index 9251fb0d21..8aa4d33ba3 100644 --- a/src/libcode/vx_data2d_python/data2d_python.cc +++ b/src/libcode/vx_data2d_python/data2d_python.cc @@ -506,4 +506,3 @@ return ( 0 ); //////////////////////////////////////////////////////////////////////// - diff --git a/src/libcode/vx_series_data/series_data.cc b/src/libcode/vx_series_data/series_data.cc index d7a0e587d9..581ea7ff50 100644 --- a/src/libcode/vx_series_data/series_data.cc +++ b/src/libcode/vx_series_data/series_data.cc @@ -8,6 +8,8 @@ //////////////////////////////////////////////////////////////////////// +using namespace std; + #include #include #include @@ -20,12 +22,27 @@ //////////////////////////////////////////////////////////////////////// -void get_series_entry(int i_series, VarInfo* data_info, +static bool read_single_entry(VarInfo*, const ConcatString&, const GrdFileType, + DataPlane&, Grid&); + +static bool read_all_entries(vector&, const ConcatString&, const GrdFileType, + vector&, Grid&); + +//////////////////////////////////////////////////////////////////////// + +bool get_series_entry(int i_series, VarInfo* data_info, const StringArray& search_files, const GrdFileType type, - DataPlane& dp, Grid& grid) { + DataPlane& dp, Grid& grid, + bool error_out, bool print_warning) { int i; bool found; + // Save the log print warning state + bool save_print_warning_state = mlog.print_warning(); + + // Suppress log warnings, if requested + if(!print_warning) mlog.set_print_warning(false); + mlog << Debug(3) << "Processing series entry " << i_series + 1 << ": " << data_info->magic_time_str() << "\n"; @@ -41,19 +58,23 @@ void get_series_entry(int i_series, VarInfo* data_info, found = read_single_entry(data_info, search_files[i_cur], type, dp, grid); + // Break out of the loop if data was found if(found) break; } // end for i - // Error out if not found - if(!found) { + // Error out if not found and specified + if(!found && error_out) { mlog << Error << "\nget_series_entry() -> " << "Could not find data for " << data_info->magic_time_str() - << " in file list:\n:" << write_css(search_files) << "\n\n"; + << " in file list:\n" << write_css(search_files) << "\n\n"; exit(1); } - return; + // Restore warnings to their original state + mlog.set_print_warning(save_print_warning_state); + + return(found); } //////////////////////////////////////////////////////////////////////// @@ -66,9 +87,9 @@ bool read_single_entry(VarInfo* info, const ConcatString& filename, // Check that file exists if(!file_exists(filename.c_str())) { - mlog << Warning << "\nread_single_entry() -> " - << "File does not exist: " << filename << "\n\n"; - return(false); + mlog << Warning << "\nread_single_entry() -> " + << "File does not exist: " << filename << "\n\n"; + return(false); } // Open data file @@ -87,3 +108,85 @@ bool read_single_entry(VarInfo* info, const ConcatString& filename, } //////////////////////////////////////////////////////////////////////// + +bool get_series_entries(int i_series, vector &vi_list, + const StringArray &search_files, const GrdFileType type, + vector &dp_list, Grid &grid, + bool error_out, bool print_warning) { + int i; + bool found; + + // Check for at least one field requested + if(vi_list.size() <= 0) return(false); + + // Save the log print warning state + bool save_print_warning_state = mlog.print_warning(); + + // Suppress log warnings, if requested + if(!print_warning) mlog.set_print_warning(false); + + mlog << Debug(3) + << "Processing series entry " << i_series + 1 << ": " + << vi_list[0]->magic_time_str() << "\n"; + + // Initialize + dp_list.clear(); + + // Search for data, beginning with the i_series index + for(i=0,found=false; i " + << "Could not find data for " << vi_list[0]->magic_time_str() + << " in file list:\n" << write_css(search_files) << "\n\n"; + exit(1); + } + + // Restore warnings to their original state + mlog.set_print_warning(save_print_warning_state); + + return(found); +} + +//////////////////////////////////////////////////////////////////////// + +bool read_all_entries(vector &vi_list, const ConcatString &filename, + const GrdFileType type, vector &dp_list, Grid &grid) { + + Met2dDataFileFactory mtddf_factory; + Met2dDataFile* mtddf = (Met2dDataFile*) 0; + + // Check that file exists + if(!file_exists(filename.c_str())) { + mlog << Warning << "\nread_all_entries() -> " + << "File does not exist: " << filename << "\n\n"; + return(false); + } + + // Open data file + mtddf = mtddf_factory.new_met_2d_data_file(filename.c_str(), type); + + // Attempt to read gridded data + int n_valid = mtddf->data_planes(vi_list, dp_list); + + // Store grid + if(n_valid > 0) grid = mtddf->grid(); + + // Cleanup + if(mtddf) { delete mtddf; mtddf = (Met2dDataFile *) 0; } + + return(n_valid > 0); +} + +//////////////////////////////////////////////////////////////////////// diff --git a/src/libcode/vx_series_data/series_data.h b/src/libcode/vx_series_data/series_data.h index d809b64edc..fd84a5ea86 100644 --- a/src/libcode/vx_series_data/series_data.h +++ b/src/libcode/vx_series_data/series_data.h @@ -13,6 +13,8 @@ //////////////////////////////////////////////////////////////////////// +#include + #include "vx_util.h" #include "vx_log.h" #include "vx_data2d.h" @@ -21,11 +23,15 @@ //////////////////////////////////////////////////////////////////////// -void get_series_entry(int, VarInfo*, const StringArray&, - const GrdFileType, DataPlane&, Grid&); +bool get_series_entry(int, VarInfo*, const StringArray&, + const GrdFileType, DataPlane&, Grid&, + bool error_out=true, + bool print_warning=true); -bool read_single_entry(VarInfo*, const ConcatString&, const GrdFileType, - DataPlane&, Grid&); +bool get_series_entries(int, std::vector&, const StringArray&, + const GrdFileType, std::vector&, Grid&, + bool error_out=true, + bool print_warning=true); //////////////////////////////////////////////////////////////////////// diff --git a/src/tools/tc_utils/tc_diag/tc_diag.cc b/src/tools/tc_utils/tc_diag/tc_diag.cc index f6ba1926f4..05e8471957 100644 --- a/src/tools/tc_utils/tc_diag/tc_diag.cc +++ b/src/tools/tc_utils/tc_diag/tc_diag.cc @@ -15,6 +15,7 @@ // Mod# Date Name Description // ---- ---- ---- ----------- // 000 09/27/22 Halley Gotway New +// 001 08/17/23 Halley Gotway MET #2609 handle missing data // //////////////////////////////////////////////////////////////////////// @@ -120,7 +121,7 @@ static void compute_lat_lon(TcrmwGrid&, double *, double *); // - Write CIRA ASCII and NetCDF diagnostics output files. // - Add support for $MET_PYTHON_EXE. // - Input data: -// - Instead of reading DataPlanes one at a time, +// - [DONE for #2609] Instead of reading DataPlanes one at a time, // read them all at once or perhaps in groups // (e.g. all pressure levels). // - Parellelize the processing of valid times. @@ -343,6 +344,10 @@ void process_tracks(TrackInfoArray& tracks) { << unix_to_yyyymmddhh(init_ta.max()) << ".\n\n"; exit(1); } + // Store the initialization time + else { + init_ut = init_ta[0]; + } return; } @@ -612,6 +617,14 @@ void set_data(const StringArray& a) { for(int i=2; i " + << "File does not exist: " << info.data_files[i] << "\n\n"; + } + } + // Store the data in the map if(data_opt_map.count(domain) == 0) data_opt_map[domain] = info; else data_opt_map[domain] += info; @@ -956,33 +969,80 @@ void process_track_points(const TrackInfoArray& tracks) { } //////////////////////////////////////////////////////////////////////// +/// + // TODO: Consider adding vortex removal logic here + // Read in the full set of fields required for vortex removal + // Add flag to configure which fields are used for vortex removal void process_fields(const TrackInfoArray &tracks, const unixtime vld_ut, int i_vld, const string &domain, const DomainInfo &di) { int i, j, i_pnt; - DataPlane data_dp; Grid grid_dp; VarInfoFactory vi_factory; VarInfo *vi = (VarInfo *) 0; - StringArray tmp_key_sa; + vector vi_list; + DataPlane dp; + vector dp_list; + StringArray tmp_key_sa, fields_missing; - // TODO: Consider adding vortex removal logic here - // Read in the full set of fields required for vortex removal - // Add flag to configure which fields are used for vortex removal - - // Loop over the VarInfo fields to be processed + // Create vector of VarInfo objects for(i=0; iset_valid(vld_ut); + vi_list.push_back(vi); + } - // Find data for this track point - get_series_entry(i_vld, vi, + // Read all data at the same time if they are all in the same file + if(conf_info.one_time_per_file_flag) { + + // Find all entries for this track point + bool status = get_series_entries(i_vld, vi_list, di.data_files, file_type, - data_dp, grid_dp); + dp_list, grid_dp, + false, false); + + } + // Otherwise, read data one field at a time + else { + + // Loop over the VarInfo fields to be processed + for(i=0; imagic_str()); + + // Store the requested valid time + dp_list[i].set_valid(vld_ut); + + } + } // end for i + + // Loop over the VarInfo fields to be processed + for(i=0; i 0) { + mlog << Warning << "For the " + << domain << " domain, " + << sec_to_hhmmss(vld_ut - init_ut) << " lead time, and " + << unix_to_yyyymmdd_hhmmss(vld_ut) << " valid time, " + << fields_missing.n() << " of " << di.var_info_ptr.size() + << " requested fields missing:\n" + << write_css(fields_missing) << "\n"; + } + // Loop over the current set of temp files for(i=0; i 0) { + dp_out = met_regrid(dp_in, grid_in, grid_out, ri); + } + // Handle empty input fields + else { + dp_out.set_valid(dp_in.valid()); + dp_out.set_size(ra_grid.range_n(), ra_grid.azimuth_n(), + bad_data_double); + } // Logic for pressure level data bool is_prs = (vi->level().type() == LevelType_Pres); diff --git a/src/tools/tc_utils/tc_diag/tc_diag.h b/src/tools/tc_utils/tc_diag/tc_diag.h index 2c79d5f8b2..b768b93864 100644 --- a/src/tools/tc_utils/tc_diag/tc_diag.h +++ b/src/tools/tc_utils/tc_diag/tc_diag.h @@ -91,6 +91,8 @@ static ConcatString out_prefix; // //////////////////////////////////////////////////////////////////////// +static unixtime init_ut = (unixtime) 0; + class OutFileInfo { private: diff --git a/src/tools/tc_utils/tc_diag/tc_diag_conf_info.cc b/src/tools/tc_utils/tc_diag/tc_diag_conf_info.cc index 62c8f374c8..97d14d9e1f 100644 --- a/src/tools/tc_utils/tc_diag/tc_diag_conf_info.cc +++ b/src/tools/tc_utils/tc_diag/tc_diag_conf_info.cc @@ -186,6 +186,7 @@ void TCDiagConfInfo::clear() { domain_info.clear(); vortex_removal_flag = false; + one_time_per_file_flag = true; nc_rng_azi_flag = false; nc_diag_flag = false; @@ -327,6 +328,10 @@ void TCDiagConfInfo::process_config(GrdFileType file_type, vortex_removal_flag = conf.lookup_bool(conf_key_vortex_removal); + // Conf: one_time_per_file_flag + one_time_per_file_flag = + conf.lookup_bool(conf_key_one_time_per_file_flag); + // Conf: nc_rng_azi_flag nc_rng_azi_flag = conf.lookup_bool(conf_key_nc_rng_azi_flag); diff --git a/src/tools/tc_utils/tc_diag/tc_diag_conf_info.h b/src/tools/tc_utils/tc_diag/tc_diag_conf_info.h index 3c2d0b5132..ff0bbc6c6e 100644 --- a/src/tools/tc_utils/tc_diag/tc_diag_conf_info.h +++ b/src/tools/tc_utils/tc_diag/tc_diag_conf_info.h @@ -119,9 +119,12 @@ class TCDiagConfInfo { // Vector of DomainInfo std::vector domain_info; - // Vortext removal settings + // Vortex removal settings bool vortex_removal_flag; + // One time per file settings + bool one_time_per_file_flag; + // Directory for temporary files ConcatString tmp_dir;