From 4c843acabcc3b4949366cbbca14d50697704e871 Mon Sep 17 00:00:00 2001 From: Howard Soh Date: Wed, 24 Apr 2024 21:03:31 +0000 Subject: [PATCH 01/30] #2867 Added compute_adp_qc_flag and adjusted ADP QC flags --- src/tools/other/point2grid/point2grid.cc | 95 ++++++++++++++++-------- 1 file changed, 62 insertions(+), 33 deletions(-) diff --git a/src/tools/other/point2grid/point2grid.cc b/src/tools/other/point2grid/point2grid.cc index d11044dc5f..589f78314a 100644 --- a/src/tools/other/point2grid/point2grid.cc +++ b/src/tools/other/point2grid/point2grid.cc @@ -126,6 +126,7 @@ static NcFile *nc_out = (NcFile *) nullptr; static NcDim lat_dim ; static NcDim lon_dim ; +static bool is_baseline_algorithm = true; /* false for Enterprise algorithm */ //////////////////////////////////////////////////////////////////////// @@ -942,8 +943,6 @@ void process_point_met_data(MetPointData *met_point_obs, MetConfig &config, VarI int valid_count = 0; int absent_count = 0; int censored_count = 0; - int qc_filtered_count = 0; - int adp_qc_filtered_count = 0; float data_value; float from_min_value = 10e10; float from_max_value = -10e10; @@ -1187,7 +1186,6 @@ void process_point_file(NcFile *nc_in, MetConfig &config, VarInfo *vinfo, void process_point_python(string python_command, MetConfig &config, VarInfo *vinfo, const Grid to_grid, bool use_xarray) { - int idx, hdr_idx; ConcatString vname, vname_cnt, vname_mask; DataPlane fr_dp, to_dp; DataPlane cnt_dp, mask_dp; @@ -1195,8 +1193,6 @@ void process_point_python(string python_command, MetConfig &config, VarInfo *vin NcVar var_obs_gc, var_obs_var; clock_t start_clock = clock(); - bool has_prob_thresh = !prob_cat_thresh.check(bad_data_double); - unixtime requested_valid_time, valid_time; static const char *method_name = "process_point_python() -> "; static const char *method_name_s = "process_point_python()"; @@ -1865,6 +1861,30 @@ void check_lat_lon(int data_size, float *latitudes, float *longitudes) { << method_name << "LONG: " << min_lon << " to " << max_lon << "\n"; } +//////////////////////////////////////////////////////////////////////// +// QC flags: 0=high, 1=medium, 2=low +// Enterpise algorithm: 0=high, 1=medium, 2=low +// Baseline algorithm: 3=high, 1=medium, 0=low (high=12/48, medium=4/16) +// returns bad_data_int if it does not belong to high, mediuam, or low. + +int compute_adp_qc_flag(int adp_qc, int shift_bits) { + int particle_qc = ((adp_qc >> shift_bits) & 0x03); + int qc_for_flag = particle_qc; + + if (is_baseline_algorithm) { + switch (particle_qc) { + case 3: qc_for_flag = 0; break; /* high */ + case 1: qc_for_flag = 1; break; /* medium */ + case 0: qc_for_flag = 2; break; /* low */ + default: qc_for_flag = bad_data_int; break; + } + } + else if (3 == particle_qc) qc_for_flag = bad_data_int; /* Enterprise algorithm */ + + return qc_for_flag; +} + + //////////////////////////////////////////////////////////////////////// static unixtime compute_unixtime(NcVar *time_var, unixtime var_value) { @@ -2523,19 +2543,20 @@ void regrid_goes_variable(NcFile *nc_in, VarInfo *vinfo, int non_missing_count = 0; int qc_filtered_count = 0; int adp_qc_filtered_count = 0; - float data_value; float from_min_value = 10e10; float from_max_value = -10e10; float qc_min_value = 10e10; float qc_max_value = -10e10; IntArray cellArray; NumArray dataArray; - int particle_qc; bool has_qc_flags = (qc_flags.n() > 0); missing_count = non_missing_count = 0; to_dp.set_constant(bad_data_double); + int shift_bits = 2; + if (is_dust_only) shift_bits += 2; + for (int xIdx=0; xIdx= 4) { if (from_min_value > data_value) from_min_value = data_value; if (from_max_value < data_value) from_max_value = data_value; } - // Filter by QC flag - qc_value = qc_data[from_index]; - if (!has_qc_var || !has_qc_flags || qc_flags.has(qc_value)) { - for(int i=0; icensor_thresh().n(); i++) { - // Break out after the first match. - if(vinfo->censor_thresh()[i].check(data_value)) { - data_value = vinfo->censor_val()[i]; - censored_count++; - break; - } - } - if (0 == adp_data[from_index]) { - absent_count++; - continue; + // Apply censor threshold + for(int i=0; icensor_thresh().n(); i++) { + // Break out after the first match. + if(vinfo->censor_thresh()[i].check(data_value)) { + data_value = vinfo->censor_val()[i]; + censored_count++; + break; } + } + + // Check the data existance (always 1 if ADP variable does not exist) + if (0 == adp_data[from_index]) { + absent_count++; + continue; + } - if (has_adp_qc_var && has_qc_flags) { - int shift_bits = 2; - if (is_dust_only) shift_bits += 2; - particle_qc = ((adp_qc_data[from_index] >> shift_bits) & 0x03); - int qc_for_flag = 3 - particle_qc; // high = 3, qc_flag for high = 0 - if (!qc_flags.has(qc_for_flag)) { + // Filter by QC flag + if (has_qc_var || has_adp_qc_var) { + qc_value = qc_data[from_index]; + if (has_adp_qc_var) { + int qc_for_flag = compute_adp_qc_flag(adp_qc_data[from_index], shift_bits); + bool filter_out = is_eq(qc_for_flag, bad_data_int); + + if (!filter_out) { + /* Adjust the quality by AOD data QC */ + if (1 == qc_value && 0 == qc_for_flag) qc_for_flag = 1; /* high to medium quality */ + else if (2 == qc_value) qc_for_flag = 2; /* high/medium to low quality */ + + if (has_qc_flags && !qc_flags.has(qc_for_flag)) filter_out = true; + } + if (filter_out) { adp_qc_filtered_count++; continue; } } + else if (has_qc_var && has_qc_flags && !qc_flags.has(qc_value)) { + qc_filtered_count++; + continue; + } dataArray.add(data_value); if (mlog.verbosity_level() >= 4) { @@ -2591,9 +2624,6 @@ void regrid_goes_variable(NcFile *nc_in, VarInfo *vinfo, if (qc_max_value < qc_value) qc_max_value = qc_value; } } - else { - qc_filtered_count++; - } valid_count++; } @@ -2620,7 +2650,6 @@ void regrid_goes_variable(NcFile *nc_in, VarInfo *vinfo, << data_count << " data values.\n"; } } - else {} } } From 438c2eedfcfb25edf77af345915c3783151eb261 Mon Sep 17 00:00:00 2001 From: Howard Soh Date: Tue, 30 Apr 2024 16:46:06 +0000 Subject: [PATCH 02/30] #2867 Added point2grid_GOES_16_ADP_Enterprise_high. Changed AOD QC flags to 0,1,2 (was 1,2,3) --- internal/test_unit/xml/unit_point2grid.xml | 29 ++++++++++++++++++---- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/internal/test_unit/xml/unit_point2grid.xml b/internal/test_unit/xml/unit_point2grid.xml index dd1792ccf0..5221ea4460 100644 --- a/internal/test_unit/xml/unit_point2grid.xml +++ b/internal/test_unit/xml/unit_point2grid.xml @@ -137,7 +137,7 @@ &OUTPUT_DIR;/point2grid/point2grid_GOES_16_AOD_TO_G212_compute.nc - + &MET_BIN;/point2grid @@ -155,7 +155,7 @@ &OUTPUT_DIR;/point2grid/point2grid_GOES_16_AOD_TO_G212_gaussian.nc - + &MET_BIN;/point2grid @@ -174,7 +174,7 @@ &OUTPUT_DIR;/point2grid/point2grid_GOES_16_ADP.nc - + &MET_BIN;/point2grid @@ -185,7 +185,7 @@ G212 \ &OUTPUT_DIR;/point2grid/point2grid_GOES_16_AOD_TO_G212_grid_map.nc \ -field 'name="AOD"; level="(*,*)";' \ - -qc 1,2,3 -method MAX \ + -qc 0,1,2 -method MAX \ -v 1 @@ -205,7 +205,7 @@ G212 \ &OUTPUT_DIR;/point2grid/point2grid_GOES_16_AOD_TO_G212.nc \ -field 'name="AOD"; level="(*,*)";' \ - -qc 1,2,3 -method MAX \ + -qc 0,1,2 -method MAX \ -v 1 @@ -213,6 +213,25 @@ + + &MET_BIN;/point2grid + + MET_TMP_DIR &OUTPUT_DIR;/point2grid + + \ + &DATA_DIR_MODEL;/goes_16/OR_ABI-L2-AODC-M6_G16_s20241100001171_e20241100003544_c20241100006242.nc \ + G212 \ + &OUTPUT_DIR;/point2grid/point2grid_GOES_16_ADP_Enterprise_high.nc \ + -field 'name="AOD_Smoke"; level="(*,*)";' \ + -adp &DATA_DIR_MODEL;/goes_16/OR_ABI-L2-ADPC-M6_G16_s20241100001171_e20241100003544_c20241100006361.nc \ + -qc 0,1 -method MAX \ + -v 1 + + + &OUTPUT_DIR;/point2grid/point2grid_GOES_16_ADP_Enterprise_high.nc + + + &MET_BIN;/point2grid From d9d287cc2603549fff39afb1fb97a7519ab97742 Mon Sep 17 00:00:00 2001 From: Howard Soh Date: Tue, 30 Apr 2024 16:46:43 +0000 Subject: [PATCH 03/30] #2867 Added get_nc_att_values_ --- src/libcode/vx_nc_util/nc_utils.hpp | 29 +++++++++++++++++++++++++---- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/src/libcode/vx_nc_util/nc_utils.hpp b/src/libcode/vx_nc_util/nc_utils.hpp index 4a6c0a1a92..94f8f7a348 100644 --- a/src/libcode/vx_nc_util/nc_utils.hpp +++ b/src/libcode/vx_nc_util/nc_utils.hpp @@ -93,12 +93,33 @@ bool get_nc_att_value_(const netCDF::NcVar *var, const ConcatString &att_name, if (!status) { mlog << Error << "\n" << caller_name << get_log_msg_for_att(att, GET_SAFE_NC_NAME_P(var), att_name); - if (exit_on_error) { - if (att) delete att; - exit(1); - } } if (att) delete att; + if (!status && exit_on_error) exit(1); + + return status; +} + +//////////////////////////////////////////////////////////////////////// + +template +bool get_nc_att_values_(const netCDF::NcVar *var, const ConcatString &att_name, + T *att_vals, bool exit_on_error, + T bad_data, const char *caller_name) { + // caller should initialize att_vals + + // + // Retrieve the NetCDF variable attribute. + // + netCDF::NcVarAtt *att = get_nc_att(var, att_name); + bool status = IS_VALID_NC_P(att); + if (status) att->getValues(att_vals); + else { + mlog << Error << "\n" << caller_name + << get_log_msg_for_att(att, GET_SAFE_NC_NAME_P(var), att_name); + } + if (att) delete att; + if (!status && exit_on_error) exit(1); return status; } From 96fde58b5865811b8bc210d15b4b644bab517a07 Mon Sep 17 00:00:00 2001 From: Howard Soh Date: Tue, 30 Apr 2024 16:48:24 +0000 Subject: [PATCH 04/30] #2867 Added get_nc_att_values. Added the argument allow_conversion to get_nc_data(netCDF::NcVar *, uchar *data) --- src/libcode/vx_nc_util/nc_utils.cc | 22 +++++++++++++++++++--- src/libcode/vx_nc_util/nc_utils.h | 5 +++-- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/src/libcode/vx_nc_util/nc_utils.cc b/src/libcode/vx_nc_util/nc_utils.cc index 87a46d9391..520b1a255f 100644 --- a/src/libcode/vx_nc_util/nc_utils.cc +++ b/src/libcode/vx_nc_util/nc_utils.cc @@ -537,6 +537,16 @@ bool get_nc_att_value(const NcVar *var, const ConcatString &att_name, //////////////////////////////////////////////////////////////////////// +bool get_nc_att_values(const NcVar *var, const ConcatString &att_name, + unsigned short *att_val, bool exit_on_error) { + static const char *method_name = "get_nc_att_value(NcVar,float) -> "; + bool status = get_nc_att_values_(var, att_name, att_val, exit_on_error, + (unsigned short)bad_data_int, method_name); + return status; +} + +//////////////////////////////////////////////////////////////////////// + bool get_nc_att_value(const NcVarAtt *att, ConcatString &att_val) { bool status = false; @@ -1750,15 +1760,21 @@ bool get_nc_data(NcVar *var, char **data) { //////////////////////////////////////////////////////////////////////// -bool get_nc_data(NcVar *var, uchar *data) { +bool get_nc_data(NcVar *var, uchar *data, bool allow_conversion) { bool return_status = false; int cell_count = get_data_size(var); int data_type = GET_NC_TYPE_ID_P(var); static const char *method_name = "get_nc_data(NcVar *, uchar *) -> "; if (NC_UBYTE == data_type) return_status = get_nc_data_t(var, data); - else if (NC_BYTE == data_type && has_unsigned_attribute(var)) { + else if (NC_BYTE == data_type) { + if (!has_unsigned_attribute(var) && !allow_conversion) { + mlog << Debug(1) << "\n" << method_name + << "INFO: Unexpected conversion from 'ncbyte' for variable \"" + << GET_NC_NAME_P(var) << "\".\n\n"; + } ncbyte *signed_data = new ncbyte[cell_count]; - if (return_status = get_nc_data_t(var, signed_data)) { + return_status = get_nc_data_t(var, signed_data); + if (return_status) { for (int idx=0; idx Date: Tue, 30 Apr 2024 17:15:21 +0000 Subject: [PATCH 05/30] #2867 Read the ADP QC flag values and meanings attributes from DQF variable and set the QC high, meduium, low values to support Enterprise algorithm. Adjusted the ADP QC values by using AOD qc values --- src/tools/other/point2grid/point2grid.cc | 133 ++++++++++++++++++----- 1 file changed, 106 insertions(+), 27 deletions(-) diff --git a/src/tools/other/point2grid/point2grid.cc b/src/tools/other/point2grid/point2grid.cc index 589f78314a..503733217d 100644 --- a/src/tools/other/point2grid/point2grid.cc +++ b/src/tools/other/point2grid/point2grid.cc @@ -126,7 +126,9 @@ static NcFile *nc_out = (NcFile *) nullptr; static NcDim lat_dim ; static NcDim lon_dim ; -static bool is_baseline_algorithm = true; /* false for Enterprise algorithm */ +static int adp_qc_high; /* 3 as baseline algorithm, 0 for enterpirse algorithm */ +static int adp_qc_medium; /* 1 as baseline algorithm, 1 for enterpirse algorithm */ +static int adp_qc_low; /* 0 as baseline algorithm, 2 for enterpirse algorithm */ //////////////////////////////////////////////////////////////////////// @@ -173,6 +175,7 @@ static void regrid_nc_variable(NcFile *nc_in, Met2dDataFile *fr_mtddf, static bool keep_message_type(const int mt_index); static bool has_lat_lon_vars(NcFile *nc_in); +static void set_adp_gc_values(NcVar var_adp_qc); //////////////////////////////////////////////////////////////////////// // for GOES 16 @@ -1869,17 +1872,12 @@ void check_lat_lon(int data_size, float *latitudes, float *longitudes) { int compute_adp_qc_flag(int adp_qc, int shift_bits) { int particle_qc = ((adp_qc >> shift_bits) & 0x03); - int qc_for_flag = particle_qc; - - if (is_baseline_algorithm) { - switch (particle_qc) { - case 3: qc_for_flag = 0; break; /* high */ - case 1: qc_for_flag = 1; break; /* medium */ - case 0: qc_for_flag = 2; break; /* low */ - default: qc_for_flag = bad_data_int; break; - } - } - else if (3 == particle_qc) qc_for_flag = bad_data_int; /* Enterprise algorithm */ + int qc_for_flag; + + if (particle_qc == adp_qc_high) qc_for_flag = 0; + else if (particle_qc == adp_qc_medium) qc_for_flag = 1; + else if (particle_qc == adp_qc_low) qc_for_flag = 2; + else qc_for_flag = bad_data_int; return qc_for_flag; } @@ -2364,12 +2362,17 @@ int get_lon_count(NcFile *_nc) { static NcVar get_goes_nc_var(NcFile *nc, const ConcatString var_name, bool exit_if_error) { - NcVar var_data = get_nc_var(nc, var_name.c_str(), false); + NcVar var_data; + static const char *method_name = "get_goes_nc_var() -> "; + if (has_var(nc, var_name.c_str())) var_data = get_nc_var(nc, var_name.c_str(), false); if (IS_INVALID_NC(var_data)) { + mlog << Debug(4) << method_name + << "The variable \"" << var_name << "\" does not exist. Find \"" + << var_name.split("_")[0] << "\" variable\n"; var_data = get_nc_var(nc, var_name.split("_")[0].c_str()); } if (IS_INVALID_NC(var_data)) { - mlog << Error << "\nget_goes_nc_var() -> " + mlog << Error << "\n" << method_name << "The variable \"" << var_name << "\" does not exist\n\n"; if (exit_if_error) exit(1); } @@ -2444,6 +2447,7 @@ void regrid_goes_variable(NcFile *nc_in, VarInfo *vinfo, DataPlane &fr_dp, DataPlane &to_dp, Grid fr_grid, Grid to_grid, IntArray *cellMapping, NcFile *nc_adp) { + const int log_debug_level = 4; bool has_qc_var = false; bool has_adp_qc_var = false; clock_t start_clock = clock(); @@ -2465,6 +2469,10 @@ void regrid_goes_variable(NcFile *nc_in, VarInfo *vinfo, // -99 is arbitrary number as invalid QC value memset(qc_data, -99, from_data_size*sizeof(uchar)); + adp_qc_high = 3; /* 3 as baseline algorithm, 0 for enterpirse algorithm */ + adp_qc_medium = 1; /* 1 as baseline algorithm, 1 for enterpirse algorithm */ + adp_qc_low = 0; /* 0 as baseline algorithm, 2 for enterpirse algorithm */ + NcVar var_qc; NcVar var_adp; NcVar var_adp_qc; @@ -2487,7 +2495,7 @@ void regrid_goes_variable(NcFile *nc_in, VarInfo *vinfo, else if (is_smoke_only) var_adp = get_goes_nc_var(nc_adp, vname_smoke); if (IS_VALID_NC(var_adp)) { - get_nc_data(&var_adp, adp_data); + get_nc_data(&var_adp, adp_data, true); //Smoke:ancillary_variables = "DQF" ; ubyte DQF(y, x) ; if (get_att_value_string(&var_adp, (string)"ancillary_variables", qc_var_name)) { @@ -2497,6 +2505,7 @@ void regrid_goes_variable(NcFile *nc_in, VarInfo *vinfo, has_adp_qc_var = true; mlog << Debug(5) << method_name << "found QC var: " << qc_var_name << " for " << GET_NC_NAME(var_adp) << ".\n"; + set_adp_gc_values(var_adp_qc); } } else { @@ -2557,6 +2566,15 @@ void regrid_goes_variable(NcFile *nc_in, VarInfo *vinfo, int shift_bits = 2; if (is_dust_only) shift_bits += 2; + int cnt_aod_qc_low = 0; + int cnt_aod_qc_high = 0; + int cnt_aod_qc_medium = 0; + int cnt_aod_qc_nr = 0; // no_retrieval_qf + int cnt_adp_qc_low = 0; + int cnt_adp_qc_high = 0; + int cnt_adp_qc_medium = 0; + int cnt_adp_qc_nr = 0; // no_retrieval_qf + int cnt_adp_qc_adjused = 0; for (int xIdx=0; xIdx= log_debug_level) { + if (qc_min_value > qc_value) qc_min_value = qc_value; + if (qc_max_value < qc_value) qc_max_value = qc_value; + switch (qc_value) { + case 0: cnt_aod_qc_high++; break; + case 1: cnt_aod_qc_medium++; break; + case 2: cnt_aod_qc_low++; break; + default: cnt_aod_qc_nr++; break; + } + } if (has_adp_qc_var) { int qc_for_flag = compute_adp_qc_flag(adp_qc_data[from_index], shift_bits); bool filter_out = is_eq(qc_for_flag, bad_data_int); + if (mlog.verbosity_level() >= log_debug_level) { + switch (qc_for_flag) { + case 0: cnt_adp_qc_high++; break; + case 1: cnt_adp_qc_medium++; break; + case 2: cnt_adp_qc_low++; break; + default: cnt_adp_qc_nr++; break; + } + } + if (!filter_out) { /* Adjust the quality by AOD data QC */ - if (1 == qc_value && 0 == qc_for_flag) qc_for_flag = 1; /* high to medium quality */ - else if (2 == qc_value) qc_for_flag = 2; /* high/medium to low quality */ + if (1 == qc_value && 0 == qc_for_flag) { + qc_for_flag = 1; /* high to medium quality */ + cnt_adp_qc_adjused++; + } + else if (2 == qc_value) { + qc_for_flag = 2; /* high/medium to low quality */ + cnt_adp_qc_adjused++; + } if (has_qc_flags && !qc_flags.has(qc_for_flag)) filter_out = true; } @@ -2617,13 +2660,8 @@ void regrid_goes_variable(NcFile *nc_in, VarInfo *vinfo, qc_filtered_count++; continue; } - - dataArray.add(data_value); - if (mlog.verbosity_level() >= 4) { - if (qc_min_value > qc_value) qc_min_value = qc_value; - if (qc_max_value < qc_value) qc_max_value = qc_value; - } } + dataArray.add(data_value); valid_count++; } @@ -2658,14 +2696,20 @@ void regrid_goes_variable(NcFile *nc_in, VarInfo *vinfo, delete [] from_data; delete [] adp_qc_data; - mlog << Debug(4) << method_name << "Count: actual: " << to_cell_count + mlog << Debug(log_debug_level) << method_name << "Count: actual: " << to_cell_count << ", missing: " << missing_count << ", non_missing: " << non_missing_count - << "\n\tFiltered: by QC: " << qc_filtered_count + << "\n Filtered: by QC: " << qc_filtered_count << ", by adp QC: " << adp_qc_filtered_count << ", by absent: " << absent_count << ", total: " << (qc_filtered_count + adp_qc_filtered_count + absent_count) - << "\n\tRange: data: [" << from_min_value << " - " << from_max_value - << "] QC: [" << qc_min_value << " - " << qc_max_value << "]\n"; + << "\n Range: data: [" << from_min_value << " - " << from_max_value + << "] QC: [" << qc_min_value << " - " << qc_max_value << "]" + << "\n AOD QC: high=" << cnt_aod_qc_high + << " medium=" << cnt_aod_qc_medium << ", low=" << cnt_aod_qc_low + << ", no_retrieval=" << cnt_aod_qc_nr + << "\n ADP QC: high=" << cnt_adp_qc_high << " medium=" << cnt_adp_qc_medium + << ", low=" << cnt_adp_qc_low << ", no_retrieval=" << cnt_adp_qc_nr + << ", adjusted=" << cnt_adp_qc_adjused << "\n"; if (to_cell_count == 0) { mlog << Warning << "\n" << method_name @@ -2866,6 +2910,41 @@ void usage() { exit(1); } +//////////////////////////////////////////////////////////////////////// +const static ConcatString att_name_values = "flag_values"; +const static ConcatString att_name_meanings = "flag_meanings"; + +void set_adp_gc_values(NcVar var_adp_qc) { + //NcVarAtt *nc_att_values = get_nc_att(&var_adp_qc, "flag_values"); + //NcVarAtt *nc_att_meanings = get_nc_att(&var_adp_qc, "flag_meanings"); + //unsigned short *flag_values = new unsigned short [256]; + ConcatString att_flag_meanings; + + if (get_nc_att_value(&var_adp_qc, (ConcatString)"flag_meanings", att_flag_meanings)) { + StringArray flag_meanings = to_lower(att_flag_meanings).split(" "); + unsigned short flag_values[256]; + for (int i=0; i> 2) & 0x03; + } + if (flag_meanings.has("medium_confidence_smoke_detection_qf", idx)) { + adp_qc_medium = (flag_values[idx] >> 2) & 0x03; + } + if (flag_meanings.has("high_confidence_smoke_detection_qf", idx)) { + adp_qc_high = (flag_values[idx] >> 2) & 0x03; + } + } + mlog << Debug(4) << "set_adp_gc_values() " + << " high_confidence = " << adp_qc_high + << ", medium_confidence = " << adp_qc_medium + << ", low_confidence = " << adp_qc_low << "\n"; + } +} + //////////////////////////////////////////////////////////////////////// void set_field(const StringArray &a) { From 2ede9d7eec444906a75cdd3f391b928a963d0e6f Mon Sep 17 00:00:00 2001 From: Howard Soh Date: Wed, 8 May 2024 18:53:35 +0000 Subject: [PATCH 06/30] #2867 Cleanup --- src/libcode/vx_nc_util/nc_utils.h | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/libcode/vx_nc_util/nc_utils.h b/src/libcode/vx_nc_util/nc_utils.h index cebd231f6b..c2b4dfc823 100644 --- a/src/libcode/vx_nc_util/nc_utils.h +++ b/src/libcode/vx_nc_util/nc_utils.h @@ -19,7 +19,7 @@ #include #ifndef ncbyte -typedef signed char ncbyte; // from ncvalues.h +typedef signed char ncbyte; /* from ncvalues.h */ #endif /* ncbyte */ #ifndef uchar typedef unsigned char uchar; @@ -262,7 +262,6 @@ extern bool get_nc_data(netCDF::NcVar *, uchar *data, bool allow_conversion=fal extern bool get_nc_data(netCDF::NcVar *, float *data); extern bool get_nc_data(netCDF::NcVar *, double *data); extern bool get_nc_data(netCDF::NcVar *, time_t *data); -//extern bool get_nc_data(netCDF::NcVar *, ncbyte *data); extern bool get_nc_data(netCDF::NcVar *, unsigned short *data); extern bool get_nc_data(netCDF::NcVar *, int *data, const LongArray &curs); From 338cee99bc404f39f564c5bc74d4b8849d8b94e0 Mon Sep 17 00:00:00 2001 From: Howard Soh Date: Wed, 8 May 2024 18:53:49 +0000 Subject: [PATCH 07/30] #2867 Corrected indent --- src/libcode/vx_nc_util/nc_utils.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libcode/vx_nc_util/nc_utils.cc b/src/libcode/vx_nc_util/nc_utils.cc index 520b1a255f..bb110fed72 100644 --- a/src/libcode/vx_nc_util/nc_utils.cc +++ b/src/libcode/vx_nc_util/nc_utils.cc @@ -538,7 +538,7 @@ bool get_nc_att_value(const NcVar *var, const ConcatString &att_name, //////////////////////////////////////////////////////////////////////// bool get_nc_att_values(const NcVar *var, const ConcatString &att_name, - unsigned short *att_val, bool exit_on_error) { + unsigned short *att_val, bool exit_on_error) { static const char *method_name = "get_nc_att_value(NcVar,float) -> "; bool status = get_nc_att_values_(var, att_name, att_val, exit_on_error, (unsigned short)bad_data_int, method_name); From ddd0080678926efdc8050975887262a127d86898 Mon Sep 17 00:00:00 2001 From: Howard Soh Date: Wed, 8 May 2024 18:57:57 +0000 Subject: [PATCH 08/30] #2867 Changed log message --- src/tools/other/point2grid/point2grid.cc | 79 ++++++++++++++---------- 1 file changed, 47 insertions(+), 32 deletions(-) diff --git a/src/tools/other/point2grid/point2grid.cc b/src/tools/other/point2grid/point2grid.cc index 503733217d..d8ca9b03c4 100644 --- a/src/tools/other/point2grid/point2grid.cc +++ b/src/tools/other/point2grid/point2grid.cc @@ -130,6 +130,14 @@ static int adp_qc_high; /* 3 as baseline algorithm, 0 for enterpirse algorit static int adp_qc_medium; /* 1 as baseline algorithm, 1 for enterpirse algorithm */ static int adp_qc_low; /* 0 as baseline algorithm, 2 for enterpirse algorithm */ +static const int MET_ADP_QC_HIGH = 0; +static const int MET_ADP_QC_MEDIUM = 1; +static const int MET_ADP_QC_LOW = 2; +static const int MET_ADP_QC_NA = bad_data_int; + +static const ConcatString att_name_values = "flag_values"; +static const ConcatString att_name_meanings = "flag_meanings"; + //////////////////////////////////////////////////////////////////////// static void process_command_line(int, char **); @@ -1874,10 +1882,10 @@ int compute_adp_qc_flag(int adp_qc, int shift_bits) { int particle_qc = ((adp_qc >> shift_bits) & 0x03); int qc_for_flag; - if (particle_qc == adp_qc_high) qc_for_flag = 0; - else if (particle_qc == adp_qc_medium) qc_for_flag = 1; - else if (particle_qc == adp_qc_low) qc_for_flag = 2; - else qc_for_flag = bad_data_int; + if (particle_qc == adp_qc_high) qc_for_flag = MET_ADP_QC_HIGH; + else if (particle_qc == adp_qc_medium) qc_for_flag = MET_ADP_QC_MEDIUM; + else if (particle_qc == adp_qc_low) qc_for_flag = MET_ADP_QC_LOW; + else qc_for_flag = MET_ADP_QC_NA; return qc_for_flag; } @@ -2369,7 +2377,7 @@ static NcVar get_goes_nc_var(NcFile *nc, const ConcatString var_name, mlog << Debug(4) << method_name << "The variable \"" << var_name << "\" does not exist. Find \"" << var_name.split("_")[0] << "\" variable\n"; - var_data = get_nc_var(nc, var_name.split("_")[0].c_str()); + var_data = get_nc_var(nc, var_name.split("_")[0].c_str()); } if (IS_INVALID_NC(var_data)) { mlog << Error << "\n" << method_name @@ -2447,9 +2455,9 @@ void regrid_goes_variable(NcFile *nc_in, VarInfo *vinfo, DataPlane &fr_dp, DataPlane &to_dp, Grid fr_grid, Grid to_grid, IntArray *cellMapping, NcFile *nc_adp) { - const int log_debug_level = 4; bool has_qc_var = false; bool has_adp_qc_var = false; + const int log_debug_level = 4; clock_t start_clock = clock(); int to_lat_count = to_grid.ny(); int to_lon_count = to_grid.nx(); @@ -2502,10 +2510,10 @@ void regrid_goes_variable(NcFile *nc_in, VarInfo *vinfo, var_adp_qc = get_nc_var(nc_adp, qc_var_name.c_str()); if (IS_VALID_NC(var_adp_qc)) { get_nc_data(&var_adp_qc, adp_qc_data); + set_adp_gc_values(var_adp_qc); has_adp_qc_var = true; mlog << Debug(5) << method_name << "found QC var: " << qc_var_name << " for " << GET_NC_NAME(var_adp) << ".\n"; - set_adp_gc_values(var_adp_qc); } } else { @@ -2574,7 +2582,9 @@ void regrid_goes_variable(NcFile *nc_in, VarInfo *vinfo, int cnt_adp_qc_high = 0; int cnt_adp_qc_medium = 0; int cnt_adp_qc_nr = 0; // no_retrieval_qf - int cnt_adp_qc_adjused = 0; + int cnt_adp_qc_high_to_low = 0; + int cnt_adp_qc_high_to_medium = 0; + int cnt_adp_qc_medium_to_low = 0; for (int xIdx=0; xIdx qc_value) qc_min_value = qc_value; if (qc_max_value < qc_value) qc_max_value = qc_value; switch (qc_value) { - case 0: cnt_aod_qc_high++; break; - case 1: cnt_aod_qc_medium++; break; - case 2: cnt_aod_qc_low++; break; + case MET_ADP_QC_HIGH: cnt_aod_qc_high++; break; + case MET_ADP_QC_MEDIUM: cnt_aod_qc_medium++; break; + case MET_ADP_QC_LOW: cnt_aod_qc_low++; break; default: cnt_aod_qc_nr++; break; } } @@ -2631,24 +2641,26 @@ void regrid_goes_variable(NcFile *nc_in, VarInfo *vinfo, if (mlog.verbosity_level() >= log_debug_level) { switch (qc_for_flag) { - case 0: cnt_adp_qc_high++; break; - case 1: cnt_adp_qc_medium++; break; - case 2: cnt_adp_qc_low++; break; + case MET_ADP_QC_HIGH: cnt_adp_qc_high++; break; + case MET_ADP_QC_MEDIUM: cnt_adp_qc_medium++; break; + case MET_ADP_QC_LOW: cnt_adp_qc_low++; break; default: cnt_adp_qc_nr++; break; } } if (!filter_out) { /* Adjust the quality by AOD data QC */ - if (1 == qc_value && 0 == qc_for_flag) { - qc_for_flag = 1; /* high to medium quality */ - cnt_adp_qc_adjused++; + if (MET_ADP_QC_LOW == qc_value) { + if (MET_ADP_QC_LOW > qc_for_flag) { + if (MET_ADP_QC_HIGH == qc_for_flag) cnt_adp_qc_high_to_low++; + else if (MET_ADP_QC_MEDIUM == qc_for_flag) cnt_adp_qc_medium_to_low++; + qc_for_flag = MET_ADP_QC_LOW; /* high/medium to low quality */ + } } - else if (2 == qc_value) { - qc_for_flag = 2; /* high/medium to low quality */ - cnt_adp_qc_adjused++; + else if (MET_ADP_QC_MEDIUM == qc_value && MET_ADP_QC_HIGH == qc_for_flag) { + qc_for_flag = MET_ADP_QC_MEDIUM; /* high to medium quality */ + cnt_adp_qc_high_to_medium++; } - if (has_qc_flags && !qc_flags.has(qc_for_flag)) filter_out = true; } if (filter_out) { @@ -2696,6 +2708,11 @@ void regrid_goes_variable(NcFile *nc_in, VarInfo *vinfo, delete [] from_data; delete [] adp_qc_data; + int cnt_adjused_low = cnt_adp_qc_low + cnt_adp_qc_high_to_low + cnt_adp_qc_medium_to_low; + int cnt_adjused_high = cnt_adp_qc_high - cnt_adp_qc_high_to_medium - cnt_adp_qc_high_to_low; + int cnt_adjused_medium = cnt_adp_qc_medium + cnt_adp_qc_high_to_medium - cnt_adp_qc_medium_to_low; + int cnt_adjused_total = cnt_adp_qc_high_to_medium + cnt_adp_qc_high_to_low + cnt_adp_qc_medium_to_low; + mlog << Debug(log_debug_level) << method_name << "Count: actual: " << to_cell_count << ", missing: " << missing_count << ", non_missing: " << non_missing_count << "\n Filtered: by QC: " << qc_filtered_count @@ -2704,12 +2721,15 @@ void regrid_goes_variable(NcFile *nc_in, VarInfo *vinfo, << ", total: " << (qc_filtered_count + adp_qc_filtered_count + absent_count) << "\n Range: data: [" << from_min_value << " - " << from_max_value << "] QC: [" << qc_min_value << " - " << qc_max_value << "]" - << "\n AOD QC: high=" << cnt_aod_qc_high - << " medium=" << cnt_aod_qc_medium << ", low=" << cnt_aod_qc_low - << ", no_retrieval=" << cnt_aod_qc_nr - << "\n ADP QC: high=" << cnt_adp_qc_high << " medium=" << cnt_adp_qc_medium - << ", low=" << cnt_adp_qc_low << ", no_retrieval=" << cnt_adp_qc_nr - << ", adjusted=" << cnt_adp_qc_adjused << "\n"; + << "\n ADP QC: high=" << cnt_adjused_high << " (" << cnt_adp_qc_high + << "), medium=" << cnt_adjused_medium << " (" << cnt_adp_qc_medium + << "), low=" << cnt_adjused_low << " (" << cnt_adp_qc_low + << "), no_retrieval=" << cnt_adp_qc_nr + << "\n adjusted: high to medium=" << cnt_adp_qc_high_to_medium + << ", high to low=" << cnt_adp_qc_high_to_low + << ", medium to low=" << cnt_adp_qc_medium_to_low + << ", total=" << cnt_adjused_total + << "\n"; if (to_cell_count == 0) { mlog << Warning << "\n" << method_name @@ -2911,13 +2931,8 @@ void usage() { } //////////////////////////////////////////////////////////////////////// -const static ConcatString att_name_values = "flag_values"; -const static ConcatString att_name_meanings = "flag_meanings"; void set_adp_gc_values(NcVar var_adp_qc) { - //NcVarAtt *nc_att_values = get_nc_att(&var_adp_qc, "flag_values"); - //NcVarAtt *nc_att_meanings = get_nc_att(&var_adp_qc, "flag_meanings"); - //unsigned short *flag_values = new unsigned short [256]; ConcatString att_flag_meanings; if (get_nc_att_value(&var_adp_qc, (ConcatString)"flag_meanings", att_flag_meanings)) { From 6672e8314d73699c5a1567277e61db8cbcdffcae Mon Sep 17 00:00:00 2001 From: Howard Soh Date: Thu, 9 May 2024 23:10:25 +0000 Subject: [PATCH 09/30] #2867 Removed unused argument --- src/libcode/vx_nc_util/nc_utils.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libcode/vx_nc_util/nc_utils.cc b/src/libcode/vx_nc_util/nc_utils.cc index bb110fed72..3698dbe41a 100644 --- a/src/libcode/vx_nc_util/nc_utils.cc +++ b/src/libcode/vx_nc_util/nc_utils.cc @@ -541,7 +541,7 @@ bool get_nc_att_values(const NcVar *var, const ConcatString &att_name, unsigned short *att_val, bool exit_on_error) { static const char *method_name = "get_nc_att_value(NcVar,float) -> "; bool status = get_nc_att_values_(var, att_name, att_val, exit_on_error, - (unsigned short)bad_data_int, method_name); + method_name); return status; } From fd2c0b409b790fa11cab94df622b95e020aca417 Mon Sep 17 00:00:00 2001 From: Howard Soh Date: Thu, 9 May 2024 23:10:41 +0000 Subject: [PATCH 10/30] #2867 Removed unused argument --- src/libcode/vx_nc_util/nc_utils.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libcode/vx_nc_util/nc_utils.hpp b/src/libcode/vx_nc_util/nc_utils.hpp index 94f8f7a348..697a34ceec 100644 --- a/src/libcode/vx_nc_util/nc_utils.hpp +++ b/src/libcode/vx_nc_util/nc_utils.hpp @@ -105,7 +105,7 @@ bool get_nc_att_value_(const netCDF::NcVar *var, const ConcatString &att_name, template bool get_nc_att_values_(const netCDF::NcVar *var, const ConcatString &att_name, T *att_vals, bool exit_on_error, - T bad_data, const char *caller_name) { + const char *caller_name) { // caller should initialize att_vals // From f8e8a83566bd8240007f72208878beb01b3cf00a Mon Sep 17 00:00:00 2001 From: Howard Soh Date: Thu, 9 May 2024 23:11:45 +0000 Subject: [PATCH 11/30] Cleanup --- src/tools/other/point2grid/point2grid_conf_info.cc | 6 +++--- src/tools/other/point2grid/point2grid_conf_info.h | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/tools/other/point2grid/point2grid_conf_info.cc b/src/tools/other/point2grid/point2grid_conf_info.cc index ba2989212f..51e0c0a996 100644 --- a/src/tools/other/point2grid/point2grid_conf_info.cc +++ b/src/tools/other/point2grid/point2grid_conf_info.cc @@ -85,7 +85,7 @@ void PointToGridConfInfo::read_config(const char *default_file_name, void PointToGridConfInfo::process_config() { ConcatString s; StringArray sa; - Dictionary *dict = (Dictionary *) nullptr; + auto dict = (Dictionary *) nullptr; // Dump the contents of the config file if(mlog.verbosity_level() >= 5) conf.dump(cout); @@ -126,7 +126,7 @@ void PointToGridConfInfo::process_config() { //////////////////////////////////////////////////////////////////////// -ConcatString PointToGridConfInfo::get_var_id(const ConcatString var_name) { +ConcatString PointToGridConfInfo::get_var_id(const ConcatString &var_name) { ConcatString var_id; map::iterator ptr; @@ -151,7 +151,7 @@ ConcatString PointToGridConfInfo::get_var_id(const ConcatString var_name) { //////////////////////////////////////////////////////////////////////// -ConcatString PointToGridConfInfo::get_var_name(const ConcatString var_name) { +ConcatString PointToGridConfInfo::get_var_name(const ConcatString &var_name) { ConcatString out_var; ConcatString t_name; diff --git a/src/tools/other/point2grid/point2grid_conf_info.h b/src/tools/other/point2grid/point2grid_conf_info.h index f6dff25b55..0a12d0baa9 100644 --- a/src/tools/other/point2grid/point2grid_conf_info.h +++ b/src/tools/other/point2grid/point2grid_conf_info.h @@ -29,11 +29,11 @@ class PointToGridConfInfo { private: void init_from_scratch(); - - protected: std::map var_name_map; std::map def_var_name_map; + protected: + public: // PointToGrid configuration object @@ -54,8 +54,8 @@ class PointToGridConfInfo { void process_config(); void read_config(const char *, const char *); - ConcatString get_var_name(const ConcatString); - ConcatString get_var_id(const ConcatString); + ConcatString get_var_name(const ConcatString &); + ConcatString get_var_id(const ConcatString &); }; //////////////////////////////////////////////////////////////////////// From b3278818bbdd6cead0b5ca21ba13175a0a35836c Mon Sep 17 00:00:00 2001 From: Howard Soh Date: Thu, 9 May 2024 23:13:53 +0000 Subject: [PATCH 12/30] #2867 Fix SonarQube findings --- src/tools/other/point2grid/point2grid.cc | 1028 +++++++++++----------- 1 file changed, 502 insertions(+), 526 deletions(-) diff --git a/src/tools/other/point2grid/point2grid.cc b/src/tools/other/point2grid/point2grid.cc index d8ca9b03c4..a919f817ea 100644 --- a/src/tools/other/point2grid/point2grid.cc +++ b/src/tools/other/point2grid/point2grid.cc @@ -59,27 +59,24 @@ using namespace netCDF; static ConcatString program_name; // Constants -static const int TYPE_UNKNOWN = 0; // Can not process the input file -static const int TYPE_OBS = 1; // MET Point Obs NetCDF (from xxx2nc) -static const int TYPE_NCCF = 2; // CF NetCDF with time and lat/lon variables -static const int TYPE_GOES = 5; -static const int TYPE_GOES_ADP = 6; -static const int TYPE_PYTHON = 7; // MET Point Obs NetCDF from PYTHON +constexpr int TYPE_UNKNOWN = 0; // Can not process the input file +constexpr int TYPE_OBS = 1; // MET Point Obs NetCDF (from xxx2nc) +constexpr int TYPE_NCCF = 2; // CF NetCDF with time and lat/lon variables +constexpr int TYPE_GOES = 5; +constexpr int TYPE_GOES_ADP = 6; +constexpr int TYPE_PYTHON = 7; // MET Point Obs NetCDF from PYTHON -static const InterpMthd DefaultInterpMthd = InterpMthd::UW_Mean; -static const int DefaultInterpWdth = 2; -static const double DefaultVldThresh = 0.5; +constexpr InterpMthd DefaultInterpMthd = InterpMthd::UW_Mean; +constexpr int DefaultInterpWdth = 2; +constexpr double DefaultVldThresh = 0.5; -static const float MISSING_LATLON = -999.0; -static const int QC_NA_INDEX = -1; -static const int LEVEL_FOR_PERFORMANCE = 6; +constexpr float MISSING_LATLON = -999.0; +constexpr int QC_NA_INDEX = -1; +constexpr int LEVEL_FOR_PERFORMANCE = 6; -static const char * default_config_filename = "MET_BASE/config/Point2GridConfig_default"; +constexpr char default_config_filename[] = "MET_BASE/config/Point2GridConfig_default"; -static const string lat_dim_name_list = "x"; // "lat,latitude"; -static const string lon_dim_name_list = "y"; // "lon,longitude"; - -static const char * GOES_global_attr_names[] = { +std::vector GOES_global_attr_names = { "naming_authority", "project", "production_site", @@ -110,7 +107,7 @@ static IntArray message_type_list; // Variables for command line arguments static ConcatString InputFilename; static ConcatString OutputFilename; -static ConcatString AdpFilename; +static ConcatString adp_filename; static ConcatString config_filename; static PointToGridConfInfo conf_info; static StringArray FieldSA; @@ -130,10 +127,10 @@ static int adp_qc_high; /* 3 as baseline algorithm, 0 for enterpirse algorit static int adp_qc_medium; /* 1 as baseline algorithm, 1 for enterpirse algorithm */ static int adp_qc_low; /* 0 as baseline algorithm, 2 for enterpirse algorithm */ -static const int MET_ADP_QC_HIGH = 0; -static const int MET_ADP_QC_MEDIUM = 1; -static const int MET_ADP_QC_LOW = 2; -static const int MET_ADP_QC_NA = bad_data_int; +constexpr int MET_ADP_QC_HIGH = 0; +constexpr int MET_ADP_QC_MEDIUM = 1; +constexpr int MET_ADP_QC_LOW = 2; +constexpr int MET_ADP_QC_NA = bad_data_int; static const ConcatString att_name_values = "flag_values"; static const ConcatString att_name_meanings = "flag_meanings"; @@ -189,11 +186,11 @@ static void set_adp_gc_values(NcVar var_adp_qc); // for GOES 16 // static const int factor_float_to_int = 1000000; -static const char *key_geostationary_data = "MET_GEOSTATIONARY_DATA"; -static const char *dim_name_lat = "lat"; -static const char *dim_name_lon = "lon"; -static const char *var_name_lat = "latitude"; -static const char *var_name_lon = "longitude"; +constexpr char key_geostationary_data[] = "MET_GEOSTATIONARY_DATA"; +constexpr char dim_name_lat[] = "lat"; +constexpr char dim_name_lon[] = "lon"; +constexpr char var_name_lat[] = "latitude"; +constexpr char var_name_lon[] = "longitude"; static const ConcatString vname_dust("Dust"); static const ConcatString vname_smoke("Smoke"); @@ -210,8 +207,7 @@ static int get_lon_count(NcFile *); static NcVar get_goes_nc_var(NcFile *nc, const ConcatString var_name, bool exit_if_error=true); static bool is_time_mismatch(NcFile *nc_in, NcFile *nc_adp); -static ConcatString make_geostationary_filename(Grid fr_grid, Grid to_grid, - ConcatString regrid_name); +static ConcatString make_geostationary_filename(Grid fr_grid, Grid to_grid); static void regrid_goes_variable(NcFile *nc_in, VarInfo *vinfo, DataPlane &fr_dp, DataPlane &to_dp, Grid fr_grid, Grid to_grid, IntArray *cellMapping, NcFile *nc_adp); @@ -244,7 +240,7 @@ const string get_tool_name() { //////////////////////////////////////////////////////////////////////// -void process_command_line(int argc, char **argv) { +static void process_command_line(int argc, char **argv) { CommandLine cline; static const char *method_name = "process_command_line() -> "; @@ -304,7 +300,7 @@ void process_command_line(int argc, char **argv) { // Check if the input file #ifdef WITH_PYTHON if (use_python) { - int offset = python_command.find("="); + auto offset = python_command.find("="); if (offset == std::string::npos) { mlog << Error << "\n" << method_name << "trouble parsing the python command " << python_command << ".\n\n"; @@ -330,12 +326,10 @@ void process_command_line(int argc, char **argv) { // Check that same variable is required multiple times without -name argument if(VarNameSA.n() == 0) { - VarInfo *vinfo; MetConfig config; - VarInfoFactory v_factory; ConcatString vname; StringArray var_names; - vinfo = v_factory.new_var_info(FileType_NcMet); + VarInfo *vinfo = VarInfoFactory::new_var_info(FileType_NcMet); for(int i=0; iclear(); // Populate the VarInfo object using the config string @@ -377,11 +371,11 @@ void process_command_line(int argc, char **argv) { //////////////////////////////////////////////////////////////////////// -void process_data_file() { +static void process_data_file() { Grid fr_grid, to_grid; GrdFileType ftype; ConcatString run_cs; - NcFile *nc_in = (NcFile *) nullptr; + auto nc_in = (NcFile *) nullptr; static const char *method_name = "process_data_file() -> "; // Initialize configuration object @@ -400,14 +394,13 @@ void process_data_file() { bool goes_data = false; bool use_python = false; int obs_type; - Met2dDataFileFactory m_factory; - Met2dDataFile *fr_mtddf = (Met2dDataFile *) nullptr; + auto fr_mtddf = (Met2dDataFile *) nullptr; #ifdef WITH_PYTHON string python_command = InputFilename; bool use_xarray = (0 == python_command.find(conf_val_python_xarray)); use_python = use_xarray || (0 == python_command.find(conf_val_python_numpy)); if (use_python) { - int offset = python_command.find("="); + auto offset = python_command.find("="); if (offset == std::string::npos) { mlog << Error << "\n" << method_name << "trouble parsing the python command " << python_command << ".\n\n"; @@ -430,7 +423,7 @@ void process_data_file() { if (obs_type == TYPE_NCCF) setenv(nc_att_met_point_nccf, "yes", 1); // Read the input data file - fr_mtddf = m_factory.new_met_2d_data_file(InputFilename.c_str(), ftype); + fr_mtddf = Met2dDataFileFactory::new_met_2d_data_file(InputFilename.c_str(), ftype); } if(!fr_mtddf) { @@ -444,8 +437,7 @@ void process_data_file() { // Setup the VarInfo request object VarInfoFactory v_factory; - VarInfo *vinfo; - vinfo = v_factory.new_var_info(ftype); + VarInfo *vinfo = VarInfoFactory::new_var_info(ftype); if(!vinfo) { mlog << Error << "\n" << method_name @@ -472,11 +464,11 @@ void process_data_file() { // Build the run command string run_cs << "Point obs (" << fr_grid.serialize() << ") to " << to_grid.serialize(); - + if (goes_data) { mlog << Debug(2) << "Input grid: " << fr_grid.serialize() << "\n"; ConcatString grid_string = get_goes_grid_input(config, fr_grid, to_grid); - if (grid_string.length() > 0) run_cs << " with " << grid_string; + if (!grid_string.empty()) run_cs << " with " << grid_string; } mlog << Debug(2) << "Output grid: " << to_grid.serialize() << "\n"; @@ -509,7 +501,7 @@ void process_data_file() { close_nc(); // Clean up - if(nc_in) { delete nc_in; nc_in = 0; } + if(nc_in) { delete nc_in; nc_in = nullptr; } if(fr_mtddf) { delete fr_mtddf; fr_mtddf = (Met2dDataFile *) nullptr; } if(vinfo) { delete vinfo; vinfo = (VarInfo *) nullptr; } @@ -521,7 +513,7 @@ void process_data_file() { bool get_nc_data_int_array(NcFile *nc, char *var_name, int *data_array, bool stop=true) { bool status = false; - NcVar nc_var = get_nc_var(nc, (char *)var_name, stop); + NcVar nc_var = get_nc_var(nc, var_name, stop); if (IS_INVALID_NC(nc_var)) { if (stop) exit(1); } @@ -548,7 +540,7 @@ bool get_nc_data_int_array(NcFile *nc, const char *var_name, int *data_array, bo // returns true if no error bool get_nc_data_float_array(NcFile *nc, char *var_name, float *data_array) { - NcVar nc_var = get_nc_var(nc, (char *)var_name); + NcVar nc_var = get_nc_var(nc, var_name); if (IS_INVALID_NC(nc_var)) exit(1); bool status = get_nc_data(&nc_var, data_array); @@ -597,24 +589,24 @@ bool get_nc_data_string_array(NcFile *nc, const char *var_name, //////////////////////////////////////////////////////////////////////// -int get_obs_type(NcFile *nc) { +static int get_obs_type(NcFile *nc) { int obs_type = TYPE_UNKNOWN; ConcatString att_val_scene_id; ConcatString att_val_project; ConcatString input_type; static const char *method_name = "get_obs_type() -> "; - + bool has_project = get_global_att(nc, (string)"project", att_val_project); bool has_scene_id = get_global_att(nc, (string)"scene_id", att_val_scene_id); if( has_scene_id && has_project && att_val_project == "GOES" ) { obs_type = TYPE_GOES; input_type = "GOES"; - if (0 < AdpFilename.length()) { + if (!adp_filename.empty()) { obs_type = TYPE_GOES_ADP; input_type = "GOES_ADP"; - if (!file_exists(AdpFilename.c_str())) { + if (!file_exists(adp_filename.c_str())) { mlog << Error << "\n" << method_name - << "ADP input \"" << AdpFilename << "\" does not exist!\n\n"; + << "ADP input \"" << adp_filename << "\" does not exist!\n\n"; exit(1); } } @@ -636,7 +628,7 @@ int get_obs_type(NcFile *nc) { //////////////////////////////////////////////////////////////////////// // Check the message types -void prepare_message_types(const StringArray hdr_types) { +void prepare_message_types(const StringArray &hdr_types) { static const char *method_name = "prepare_message_types() -> "; message_type_list.clear(); if (0 < conf_info.message_type.n()) { @@ -665,14 +657,14 @@ void prepare_message_types(const StringArray hdr_types) { //////////////////////////////////////////////////////////////////////// -IntArray prepare_qc_array(const IntArray qc_flags, StringArray qc_tables) { +IntArray prepare_qc_array(const IntArray &_qc_flags, StringArray &qc_tables) { IntArray qc_idx_array; - bool has_qc_flags = (qc_flags.n() > 0); + bool has_qc_flags = (_qc_flags.n() > 0); if (has_qc_flags) { for(int idx=0; idxget_header_data(); + const MetPointHeader *header_data = met_point_obs->get_header_data(); MetPointObsData *obs_data = met_point_obs->get_point_obs_data(); nhdr = met_point_obs->get_hdr_cnt(); @@ -718,8 +707,8 @@ void process_point_met_data(MetPointData *met_point_obs, MetConfig &config, VarI bool empty_input = (nhdr == 0 && nobs == 0); bool use_var_id = met_point_obs->is_using_var_id(); - float *hdr_lats = new float[nhdr]; - float *hdr_lons = new float[nhdr]; + auto hdr_lats = new float[nhdr]; + auto hdr_lons = new float[nhdr]; IntArray var_index_array; IntArray valid_time_array; StringArray qc_tables = met_point_obs->get_qty_data(); @@ -734,416 +723,410 @@ void process_point_met_data(MetPointData *met_point_obs, MetConfig &config, VarI prepare_message_types(header_data->typ_array); // Check and read obs_vid and obs_var if exists - bool success_to_read = true; - if (success_to_read) { - bool has_qc_flags = (qc_flags.n() > 0); - IntArray qc_idx_array = prepare_qc_array(qc_flags, qc_tables); + bool has_qc_flags = (qc_flags.n() > 0); + IntArray qc_idx_array = prepare_qc_array(qc_flags, qc_tables); - // Initialize size and values of output fields - nx = to_grid.nx(); - ny = to_grid.ny(); - to_dp.set_size(nx, ny); - to_dp.set_constant(bad_data_double); - cnt_dp.set_size(nx, ny); - cnt_dp.set_constant(0); - mask_dp.set_size(nx, ny); - mask_dp.set_constant(0); - if (has_prob_thresh || do_gaussian_filter) { - prob_dp.set_size(nx, ny); - prob_dp.set_constant(0); - prob_mask_dp.set_size(nx, ny); - prob_mask_dp.set_constant(0); - } - - // Loop through the requested fields - int obs_count_zero_to, obs_count_non_zero_to; - int obs_count_zero_from, obs_count_non_zero_from; - IntArray *cellMapping = (IntArray *) nullptr; + // Initialize size and values of output fields + nx = to_grid.nx(); + ny = to_grid.ny(); + to_dp.set_size(nx, ny); + to_dp.set_constant(bad_data_double); + cnt_dp.set_size(nx, ny); + cnt_dp.set_constant(0); + mask_dp.set_size(nx, ny); + mask_dp.set_constant(0); + if (has_prob_thresh || do_gaussian_filter) { + prob_dp.set_size(nx, ny); + prob_dp.set_constant(0); + prob_mask_dp.set_size(nx, ny); + prob_mask_dp.set_constant(0); + } - obs_count_zero_to = obs_count_non_zero_to = 0; - obs_count_zero_from = obs_count_non_zero_from = 0; - for(int i=0; iclear(); + obs_count_zero_to = obs_count_non_zero_to = 0; + obs_count_zero_from = obs_count_non_zero_from = 0; + for(int i=0; iset_dict(config); + var_idx_or_gc = -1; - // Check the variable name - ConcatString error_msg; - vname = vinfo->name(); - bool exit_by_field_name_error = false; - if (vname == "obs_val" || vname == "obs_lvl" || vname == "obs_hgt") { - exit_by_field_name_error = true; - error_msg << "The variable \"" << vname - << "\" exists but is not a valid field name.\n"; + // Initialize + vinfo->clear(); + + // Populate the VarInfo object using the config string + config.read_string(FieldSA[i].c_str()); + vinfo->set_dict(config); + + // Check the variable name + ConcatString error_msg; + vname = vinfo->name(); + bool exit_by_field_name_error = false; + if (vname == "obs_val" || vname == "obs_lvl" || vname == "obs_hgt") { + exit_by_field_name_error = true; + error_msg << "The variable \"" << vname + << "\" exists but is not a valid field name.\n"; + } + else { + if (use_var_id) { + if (!var_names.has(vname, var_idx_or_gc)) { + exit_by_field_name_error = true; + error_msg << "The variable \"" << vname << "\" is not available.\n"; + } } else { - if (use_var_id) { - if (!var_names.has(vname, var_idx_or_gc)) { + const int TMP_BUF_LEN = 128; + char grib_code[TMP_BUF_LEN + 1]; + var_idx_or_gc = atoi(vname.c_str()); + snprintf(grib_code, TMP_BUF_LEN, "%d", var_idx_or_gc); + if (vname != grib_code) { + ConcatString var_id = conf_info.get_var_id(vname); + if( var_id.nonempty() ) { + var_idx_or_gc = atoi(var_id.c_str()); + snprintf(grib_code, TMP_BUF_LEN, "%d", var_idx_or_gc); + } + else { exit_by_field_name_error = true; - error_msg << "The variable \"" << vname << "\" is not available.\n"; + error_msg << "Invalid GRIB code [" << vname << "]\n"; } } else { - const int TMP_BUF_LEN = 128; - char grib_code[TMP_BUF_LEN + 1]; - var_idx_or_gc = atoi(vname.c_str()); - snprintf(grib_code, TMP_BUF_LEN, "%d", var_idx_or_gc); - if (vname != grib_code) { - ConcatString var_id = conf_info.get_var_id(vname); - if( var_id.nonempty() ) { - var_idx_or_gc = atoi(var_id.c_str()); - snprintf(grib_code, TMP_BUF_LEN, "%d", var_idx_or_gc); - } - else { - exit_by_field_name_error = true; - error_msg << "Invalid GRIB code [" << vname << "]\n"; + bool not_found_grib_code = true; + for (idx=0; idxobs_ids[idx]) { + not_found_grib_code = false; + break; } } - else { - bool not_found_grib_code = true; - for (idx=0; idxobs_ids[idx]) { - not_found_grib_code = false; - break; - } - } - if (not_found_grib_code) { - exit_by_field_name_error = true; - error_msg << "No data for the GRIB code [" << vname << "]\n"; - } + if (not_found_grib_code) { + exit_by_field_name_error = true; + error_msg << "No data for the GRIB code [" << vname << "]\n"; } } } + } - if (exit_by_field_name_error) { - ConcatString log_msg; - if (use_var_id) { - for (idx=0; idxobs_ids[idx])) { - grib_codes.add(obs_data->obs_ids[idx]); - if (0 < idx) log_msg << ", "; - log_msg << obs_data->obs_ids[idx]; - } - } - } - if (empty_input) { - mlog << Warning << "\n" << method_name - << error_msg << "\tBut ignored because of empty input\n\n"; - } - else { - mlog << Error << "\n" << method_name - << error_msg - << "Try setting the \"name\" in the \"-field\" command line option to one of the available names:\n" - << "\t" << log_msg << "\n\n"; - exit(1); + if (exit_by_field_name_error) { + ConcatString log_msg; + if (use_var_id) { + for (idx=0; idxvalid(); - if (valid_time == 0) valid_time = conf_info.valid_time; - requested_valid_time = valid_time; - if (0 < valid_time) { - valid_beg_ut = valid_end_ut = valid_time; - if (!is_bad_data(conf_info.beg_ds)) valid_beg_ut += conf_info.beg_ds; - if (!is_bad_data(conf_info.end_ds)) valid_end_ut += conf_info.end_ds; - for(idx=0; idxobs_ids[idx])) { + grib_codes.add(obs_data->obs_ids[idx]); + if (0 < idx) log_msg << ", "; + log_msg << obs_data->obs_ids[idx]; } } - valid_time_array.add(bad_data_int); // added dummy entry + } + if (empty_input) { + mlog << Warning << "\n" << method_name + << error_msg << "\tBut ignored because of empty input\n\n"; } else { - valid_time_from_config = false; - // Set the latest available valid time - valid_time = 0; - for(idx=0; idx valid_time) valid_time = obs_time; + mlog << Error << "\n" << method_name + << error_msg + << "Try setting the \"name\" in the \"-field\" command line option to one of the available names:\n" + << "\t" << log_msg << "\n\n"; + exit(1); + } + } + + // Check the time range. Apply the time window + bool valid_time_from_config = true; + unixtime valid_beg_ut, valid_end_ut, obs_time; + + valid_time_array.clear(); + valid_time = vinfo->valid(); + if (valid_time == 0) valid_time = conf_info.valid_time; + requested_valid_time = valid_time; + if (0 < valid_time) { + valid_beg_ut = valid_end_ut = valid_time; + if (!is_bad_data(conf_info.beg_ds)) valid_beg_ut += conf_info.beg_ds; + if (!is_bad_data(conf_info.end_ds)) valid_end_ut += conf_info.end_ds; + for(idx=0; idx valid_time) valid_time = obs_time; } + } + mlog << Debug(3) << method_name << "valid_time from " + << (valid_time_from_config ? "config" : "input data") << ": " + << unix_to_yyyymmdd_hhmmss(valid_time) << "\n"; - var_index_array.clear(); - // Select output variable name - vname = (VarNameSA.n() == 0) - ? conf_info.get_var_name(vinfo->name()) - : conf_info.get_var_name(VarNameSA[i]); - mlog << Debug(4) << method_name - << "var: " << vname << ", index: " << var_idx_or_gc << ".\n"; - - var_count = var_count2 = to_count = 0; - filtered_by_time = filtered_by_msg_type = filtered_by_qc = 0; - for (idx=0; idx < nobs; idx++) { - if (var_idx_or_gc == obs_data->obs_ids[idx]) { - var_count2++; - hdr_idx = obs_data->obs_hids[idx]; - if (0 < valid_time_array.n() && - !valid_time_array.has(header_data->vld_idx_array[hdr_idx])) { - filtered_by_time++; - continue; - } + to_dp.set_init(valid_time); + to_dp.set_valid(valid_time); + cnt_dp.set_init(valid_time); + cnt_dp.set_valid(valid_time); + mask_dp.set_init(valid_time); + mask_dp.set_valid(valid_time); + if (has_prob_thresh || do_gaussian_filter) { + prob_dp.set_init(valid_time); + prob_dp.set_valid(valid_time); + prob_mask_dp.set_init(valid_time); + prob_mask_dp.set_valid(valid_time); + } - if(!keep_message_type(header_data->typ_idx_array[hdr_idx])) { - filtered_by_msg_type++; - continue; - } + var_index_array.clear(); + // Select output variable name + vname = (VarNameSA.n() == 0) + ? conf_info.get_var_name(vinfo->name()) + : conf_info.get_var_name(VarNameSA[i]); + mlog << Debug(4) << method_name + << "var: " << vname << ", index: " << var_idx_or_gc << ".\n"; + + var_count = var_count2 = to_count = 0; + filtered_by_time = filtered_by_msg_type = filtered_by_qc = 0; + for (idx=0; idx < nobs; idx++) { + if (var_idx_or_gc == obs_data->obs_ids[idx]) { + var_count2++; + hdr_idx = obs_data->obs_hids[idx]; + if (0 < valid_time_array.n() && + !valid_time_array.has(header_data->vld_idx_array[hdr_idx])) { + filtered_by_time++; + continue; + } - // Filter by QC flag - if (has_qc_flags && !qc_idx_array.has(obs_data->obs_qids[idx])) { - filtered_by_qc++; - continue; - } + if(!keep_message_type(header_data->typ_idx_array[hdr_idx])) { + filtered_by_msg_type++; + continue; + } - var_index_array.add(idx); - var_count++; - if (is_eq(obs_data->obs_vals[idx], 0.)) obs_count_zero_from++; - else obs_count_non_zero_from++; + // Filter by QC flag + if (has_qc_flags && !qc_idx_array.has(obs_data->obs_qids[idx])) { + filtered_by_qc++; + continue; } + + var_index_array.add(idx); + var_count++; + if (is_eq(obs_data->obs_vals[idx], 0.)) obs_count_zero_from++; + else obs_count_non_zero_from++; } + } - if (cellMapping) { - for (idx=0; idx<(nx*ny); idx++) cellMapping[idx].clear(); - delete [] cellMapping; + if (cellMapping) { + for (idx=0; idx<(nx*ny); idx++) cellMapping[idx].clear(); + delete [] cellMapping; + } + cellMapping = new IntArray[nx * ny]; + if( get_grid_mapping(to_grid, cellMapping, var_index_array, + obs_data->obs_hids, hdr_lats, hdr_lons) ) { + int from_index; + IntArray cellArray; + NumArray dataArray; + int offset = 0; + int valid_count = 0; + int censored_count = 0; + double data_value; + double from_min_value = 10e10; + double from_max_value = -10e10; + + // Initialize counter and output fields + to_count = 0; + to_dp.set_constant(bad_data_double); + cnt_dp.set_constant(0); + mask_dp.set_constant(0); + if (has_prob_thresh || do_gaussian_filter) { + prob_dp.set_constant(0); + prob_mask_dp.set_constant(0); } - cellMapping = new IntArray[nx * ny]; - if( get_grid_mapping(to_grid, cellMapping, var_index_array, - obs_data->obs_hids, hdr_lats, hdr_lons) ) { - int from_index; - IntArray cellArray; - NumArray dataArray; - int offset = 0; - int valid_count = 0; - int absent_count = 0; - int censored_count = 0; - float data_value; - float from_min_value = 10e10; - float from_max_value = -10e10; - - // Initialize counter and output fields - to_count = 0; - to_dp.set_constant(bad_data_double); - cnt_dp.set_constant(0); - mask_dp.set_constant(0); - if (has_prob_thresh || do_gaussian_filter) { - prob_dp.set_constant(0); - prob_mask_dp.set_constant(0); - } - for (int x_idx = 0; x_idxget_obs_val(from_index); - if (is_bad_data(data_value)) continue; - - if(mlog.verbosity_level() >= 4) { - if (from_min_value > data_value) from_min_value = data_value; - if (from_max_value < data_value) from_max_value = data_value; - } + for (int x_idx = 0; x_idxget_obs_val(from_index); + if (is_bad_data(data_value)) continue; + + if(mlog.verbosity_level() >= 4) { + if (from_min_value > data_value) from_min_value = data_value; + if (from_max_value < data_value) from_max_value = data_value; + } - for(int ic=0; iccensor_thresh().n(); ic++) { - // Break out after the first match. - if(vinfo->censor_thresh()[ic].check(data_value)) { - data_value = vinfo->censor_val()[ic]; - censored_count++; - break; - } + for(int ic=0; iccensor_thresh().n(); ic++) { + // Break out after the first match. + if(vinfo->censor_thresh()[ic].check(data_value)) { + data_value = vinfo->censor_val()[ic]; + censored_count++; + break; } + } - dataArray.add(data_value); - valid_count++; + dataArray.add(data_value); + valid_count++; + } + if (0 < valid_count) to_count++; + + int data_count = dataArray.n(); + if (0 < data_count) { + double to_value; + if (RGInfo.method == InterpMthd::Min) to_value = dataArray.min(); + else if (RGInfo.method == InterpMthd::Max) to_value = dataArray.max(); + else if (RGInfo.method == InterpMthd::Median) { + dataArray.sort_array(); + to_value = dataArray[data_count/2]; + if (0 == data_count % 2) + to_value = (float)(to_value + dataArray[(data_count/2)+1])/2; + } + else to_value = dataArray.sum() / data_count; + + if (is_eq(to_value, 0.)) obs_count_zero_to++; + else obs_count_non_zero_to++; + + cnt_dp.set(data_count, x_idx, y_idx); + mask_dp.set(1, x_idx, y_idx); + to_dp.set(to_value, x_idx, y_idx); + if ((has_prob_thresh && prob_cat_thresh.check(to_value)) + || (do_gaussian_filter && !has_prob_thresh)) { + prob_dp.set(1, x_idx, y_idx); + prob_mask_dp.set(1, x_idx, y_idx); } - if (0 < valid_count) to_count++; - - int data_count = dataArray.n(); - if (0 < data_count) { - float to_value; - if (RGInfo.method == InterpMthd::Min) to_value = dataArray.min(); - else if (RGInfo.method == InterpMthd::Max) to_value = dataArray.max(); - else if (RGInfo.method == InterpMthd::Median) { - dataArray.sort_array(); - to_value = dataArray[data_count/2]; - if (0 == data_count % 2) - to_value = (to_value + dataArray[(data_count/2)+1])/2; - } - else to_value = dataArray.sum() / data_count; - - if (is_eq(to_value, 0.)) obs_count_zero_to++; - else obs_count_non_zero_to++; - - cnt_dp.set(data_count, x_idx, y_idx); - mask_dp.set(1, x_idx, y_idx); - to_dp.set(to_value, x_idx, y_idx); - if ((has_prob_thresh && prob_cat_thresh.check(to_value)) - || (do_gaussian_filter && !has_prob_thresh)) { - prob_dp.set(1, x_idx, y_idx); - prob_mask_dp.set(1, x_idx, y_idx); - } - if (1 < data_count) { - mlog << Debug(9) << method_name - << " to_value:" << to_value - << " at " << x_idx << "," << y_idx - << ", max: " << dataArray.max() - << ", min: " << dataArray.min() - << ", mean: " << dataArray.sum()/data_count - << " from " << data_count << " data values.\n"; - } - mlog << Debug(8) << method_name << "data at " << x_idx << "," << y_idx - << ", value: " << to_value << "\n"; + if (1 < data_count) { + mlog << Debug(9) << method_name + << " to_value:" << to_value + << " at " << x_idx << "," << y_idx + << ", max: " << dataArray.max() + << ", min: " << dataArray.min() + << ", mean: " << dataArray.sum()/data_count + << " from " << data_count << " data values.\n"; } + mlog << Debug(8) << method_name << "data at " << x_idx << "," << y_idx + << ", value: " << to_value << "\n"; } } } } + } - // Write the regridded data - write_nc(to_dp, to_grid, vinfo, vname.c_str()); + // Write the regridded data + write_nc(to_dp, to_grid, vinfo, vname.c_str()); - vname_cnt = vname; - vname_cnt << "_cnt"; - vname_mask = vname; - vname_mask << "_mask"; + vname_cnt = vname; + vname_cnt << "_cnt"; + vname_mask = vname; + vname_mask << "_mask"; - ConcatString tmp_long_name; - ConcatString var_long_name = vinfo->long_name(); - ConcatString dim_string = "(*,*)"; + ConcatString tmp_long_name; + ConcatString var_long_name = vinfo->long_name(); + ConcatString dim_string = "(*,*)"; - tmp_long_name = vname_cnt; - tmp_long_name << dim_string; - vinfo->set_long_name(tmp_long_name.c_str()); - write_nc_int(cnt_dp, to_grid, vinfo, vname_cnt.c_str()); + tmp_long_name = vname_cnt; + tmp_long_name << dim_string; + vinfo->set_long_name(tmp_long_name.c_str()); + write_nc_int(cnt_dp, to_grid, vinfo, vname_cnt.c_str()); - tmp_long_name = vname_mask; - tmp_long_name << dim_string; - vinfo->set_long_name(tmp_long_name.c_str()); - write_nc_int(mask_dp, to_grid, vinfo, vname_mask.c_str()); + tmp_long_name = vname_mask; + tmp_long_name << dim_string; + vinfo->set_long_name(tmp_long_name.c_str()); + write_nc_int(mask_dp, to_grid, vinfo, vname_mask.c_str()); - if (has_prob_thresh || do_gaussian_filter) { - ConcatString vname_prob = vname; - vname_prob << "_prob_" << prob_cat_thresh.get_abbr_str(); - ConcatString vname_prob_mask = vname_prob; - vname_prob_mask << "_mask"; + if (has_prob_thresh || do_gaussian_filter) { + ConcatString vname_prob = vname; + vname_prob << "_prob_" << prob_cat_thresh.get_abbr_str(); + ConcatString vname_prob_mask = vname_prob; + vname_prob_mask << "_mask"; - if (do_gaussian_filter) interp_gaussian_dp(prob_dp, RGInfo.gaussian, RGInfo.vld_thresh); + if (do_gaussian_filter) interp_gaussian_dp(prob_dp, RGInfo.gaussian, RGInfo.vld_thresh); - tmp_long_name = vname_prob; - tmp_long_name << dim_string; - vinfo->set_long_name(tmp_long_name.c_str()); - write_nc(prob_dp, to_grid, vinfo, vname_prob.c_str()); - if (do_gaussian_filter) { - NcVar prob_var = get_var(nc_out, vname_prob.c_str()); - if (IS_VALID_NC(prob_var)) { - add_att(&prob_var, "gaussian_radius", RGInfo.gaussian.radius); - add_att(&prob_var, "gaussian_dx", RGInfo.gaussian.dx); - add_att(&prob_var, "trunc_factor", RGInfo.gaussian.trunc_factor); - } + tmp_long_name = vname_prob; + tmp_long_name << dim_string; + vinfo->set_long_name(tmp_long_name.c_str()); + write_nc(prob_dp, to_grid, vinfo, vname_prob.c_str()); + if (do_gaussian_filter) { + NcVar prob_var = get_var(nc_out, vname_prob.c_str()); + if (IS_VALID_NC(prob_var)) { + add_att(&prob_var, "gaussian_radius", RGInfo.gaussian.radius); + add_att(&prob_var, "gaussian_dx", RGInfo.gaussian.dx); + add_att(&prob_var, "trunc_factor", RGInfo.gaussian.trunc_factor); } - - tmp_long_name = vname_prob_mask; - tmp_long_name << dim_string; - vinfo->set_long_name(tmp_long_name.c_str()); - write_nc_int(prob_mask_dp, to_grid, vinfo, vname_prob_mask.c_str()); } - vinfo->set_long_name(var_long_name.c_str()); - mlog << Debug(7) << method_name << "obs_count_zero_to: " << obs_count_zero_to - << ", obs_count_non_zero_to: " << obs_count_non_zero_to << "\n"; + tmp_long_name = vname_prob_mask; + tmp_long_name << dim_string; + vinfo->set_long_name(tmp_long_name.c_str()); + write_nc_int(prob_mask_dp, to_grid, vinfo, vname_prob_mask.c_str()); + } + vinfo->set_long_name(var_long_name.c_str()); + + mlog << Debug(7) << method_name << "obs_count_zero_to: " << obs_count_zero_to + << ", obs_count_non_zero_to: " << obs_count_non_zero_to << "\n"; - ConcatString log_msg; - log_msg << "Filtered by time: " << filtered_by_time; - if (0 < requested_valid_time) { - log_msg << " [" << unix_to_yyyymmdd_hhmmss(requested_valid_time) << "]"; - } - log_msg << ", by msg_type: " << filtered_by_msg_type; - if (0 < filtered_by_msg_type) { - log_msg << " ["; - for(idx=0; idx 0) log_msg << ","; - log_msg << conf_info.message_type[idx]; - } - log_msg << "]"; + ConcatString log_msg; + log_msg << "Filtered by time: " << filtered_by_time; + if (0 < requested_valid_time) { + log_msg << " [" << unix_to_yyyymmdd_hhmmss(requested_valid_time) << "]"; + } + log_msg << ", by msg_type: " << filtered_by_msg_type; + if (0 < filtered_by_msg_type) { + log_msg << " ["; + for(idx=0; idx 0) log_msg << ","; + log_msg << conf_info.message_type[idx]; } - log_msg << ", by QC: " << filtered_by_qc; - if (0 < filtered_by_qc) { - log_msg << " ["; - for(idx=0; idx 0) log_msg << ","; - log_msg << qc_flags[idx]; - } - log_msg << "]"; + log_msg << "]"; + } + log_msg << ", by QC: " << filtered_by_qc; + if (0 < filtered_by_qc) { + log_msg << " ["; + for(idx=0; idx 0) log_msg << ","; + log_msg << qc_flags[idx]; } - log_msg << ", out of " << var_count2; - int filtered_count = filtered_by_msg_type + filtered_by_qc + requested_valid_time; - if (0 == var_count) { - if (0 == filtered_count) { - mlog << Warning << "\n" << method_name - << "No valid data for the variable [" - << vinfo->name() << "]\n\n"; - } - else { - mlog << Warning << "\n" << method_name - << "No valid data after filtering.\n\t" - << log_msg << ".\n\n"; - } + log_msg << "]"; + } + log_msg << ", out of " << var_count2; + int filtered_count = filtered_by_msg_type + filtered_by_qc + (int)requested_valid_time; + if (0 == var_count) { + if (0 == filtered_count) { + mlog << Warning << "\n" << method_name + << "No valid data for the variable [" + << vinfo->name() << "]\n\n"; } else { - mlog << Debug(2) << method_name << "var_count=" << var_count - << ", grid: " << to_count << " out of " << (nx * ny) << " " - << (0 < filtered_count ? log_msg.c_str() : " ") << "\n"; + mlog << Warning << "\n" << method_name + << "No valid data after filtering.\n\t" + << log_msg << ".\n\n"; } - } // end for i - - if (cellMapping) { - delete [] cellMapping; cellMapping = (IntArray *) nullptr; } + else { + mlog << Debug(2) << method_name << "var_count=" << var_count + << ", grid: " << to_count << " out of " << (nx * ny) << " " + << (0 < filtered_count ? log_msg.c_str() : " ") << "\n"; + } + } // end for i + + if (cellMapping) { + delete [] cellMapping; cellMapping = (IntArray *) nullptr; } delete [] hdr_lats; @@ -1154,7 +1137,7 @@ void process_point_met_data(MetPointData *met_point_obs, MetConfig &config, VarI //////////////////////////////////////////////////////////////////////// -void process_point_file(NcFile *nc_in, MetConfig &config, VarInfo *vinfo, +static void process_point_file(NcFile *nc_in, MetConfig &config, VarInfo *vinfo, const Grid to_grid) { ConcatString vname, vname_cnt, vname_mask; DataPlane fr_dp, to_dp; @@ -1164,7 +1147,6 @@ void process_point_file(NcFile *nc_in, MetConfig &config, VarInfo *vinfo, clock_t start_clock = clock(); - unixtime requested_valid_time, valid_time; static const char *method_name = "process_point_file() -> "; static const char *method_name_s = "process_point_file()"; @@ -1195,8 +1177,8 @@ void process_point_file(NcFile *nc_in, MetConfig &config, VarInfo *vinfo, #ifdef WITH_PYTHON -void process_point_python(string python_command, MetConfig &config, VarInfo *vinfo, - const Grid to_grid, bool use_xarray) { +static void process_point_python(string python_command, MetConfig &config, VarInfo *vinfo, + const Grid to_grid, bool use_xarray) { ConcatString vname, vname_cnt, vname_mask; DataPlane fr_dp, to_dp; DataPlane cnt_dp, mask_dp; @@ -1204,9 +1186,7 @@ void process_point_python(string python_command, MetConfig &config, VarInfo *vin NcVar var_obs_gc, var_obs_var; clock_t start_clock = clock(); - unixtime requested_valid_time, valid_time; static const char *method_name = "process_point_python() -> "; - static const char *method_name_s = "process_point_python()"; // Check for at least one configuration string if(FieldSA.n() < 1) { @@ -1240,15 +1220,15 @@ void process_point_python(string python_command, MetConfig &config, VarInfo *vin //////////////////////////////////////////////////////////////////////// -void process_point_nccf_file(NcFile *nc_in, MetConfig &config, - VarInfo *vinfo, Met2dDataFile *fr_mtddf, - const Grid to_grid) { +static void process_point_nccf_file(NcFile *nc_in, MetConfig &config, + VarInfo *vinfo, Met2dDataFile *fr_mtddf, + const Grid to_grid) { ConcatString vname, vname_cnt, vname_mask; DataPlane fr_dp, to_dp; DataPlane cnt_dp, mask_dp; unixtime valid_beg_ut, valid_end_ut; - bool *skip_times = 0; - double *valid_times = 0; + bool *skip_times = nullptr; + double *valid_times = nullptr; int filtered_by_time = 0; clock_t start_clock = clock(); bool opt_all_attrs = false; @@ -1268,7 +1248,7 @@ void process_point_nccf_file(NcFile *nc_in, MetConfig &config, << "can not find the longitude variable.\n\n"; exit(1); } - + // Check for at least one configuration string if(FieldSA.n() < 1) { mlog << Error << "\n" << method_name @@ -1278,7 +1258,7 @@ void process_point_nccf_file(NcFile *nc_in, MetConfig &config, unixtime valid_time = bad_data_int; valid_beg_ut = valid_end_ut = conf_info.valid_time; - + NcVar time_var = get_nc_var_time(nc_in); if( IS_VALID_NC(time_var) ) { if( 1 < get_dim_count(&time_var) ) { @@ -1288,7 +1268,7 @@ void process_point_nccf_file(NcFile *nc_in, MetConfig &config, if (get_nc_data(&time_var, valid_times)) { int sec_per_unit = 0; bool no_leap_year = false; - unixtime ref_ut = (unixtime) 0; + auto ref_ut = (unixtime) 0; unixtime tmp_time; if( conf_info.valid_time > 0 ) { if (!is_bad_data(conf_info.beg_ds)) valid_beg_ut += conf_info.beg_ds; @@ -1311,7 +1291,7 @@ void process_point_nccf_file(NcFile *nc_in, MetConfig &config, else valid_time = find_valid_time(time_var); } to_dp.set_size(to_grid.nx(), to_grid.ny()); - IntArray *cellMapping = new IntArray[to_grid.nx() * to_grid.ny()]; + auto cellMapping = new IntArray[to_grid.nx() * to_grid.ny()]; get_grid_mapping(fr_grid, to_grid, cellMapping, var_lat, var_lon, skip_times); if( skip_times ) delete [] skip_times; if( valid_times ) delete [] valid_times; @@ -1411,9 +1391,9 @@ void process_point_nccf_file(NcFile *nc_in, MetConfig &config, //////////////////////////////////////////////////////////////////////// -void regrid_nc_variable(NcFile *nc_in, Met2dDataFile *fr_mtddf, - VarInfo *vinfo, DataPlane &fr_dp, DataPlane &to_dp, - Grid to_grid, IntArray *cellMapping) { +static void regrid_nc_variable(NcFile *nc_in, Met2dDataFile *fr_mtddf, + VarInfo *vinfo, DataPlane &fr_dp, DataPlane &to_dp, + Grid to_grid, IntArray *cellMapping) { int to_cell_cnt = 0; clock_t start_clock = clock(); @@ -1427,7 +1407,7 @@ void regrid_nc_variable(NcFile *nc_in, Met2dDataFile *fr_mtddf, << InputFilename << "\"\n\n"; exit(1); } - + int from_lat_cnt = fr_grid.ny(); int from_lon_cnt = fr_grid.nx(); int from_data_size = from_lat_cnt * from_lon_cnt; @@ -1440,7 +1420,7 @@ void regrid_nc_variable(NcFile *nc_in, Met2dDataFile *fr_mtddf, } else { bool is_to_north = !fr_grid.get_swap_to_north(); - float *from_data = new float[from_data_size]; + auto from_data = new float[from_data_size]; for (int xIdx=0; xIdx= 4) { @@ -1490,7 +1469,7 @@ void regrid_nc_variable(NcFile *nc_in, Met2dDataFile *fr_mtddf, } if (0 < dataArray.n()) { - float to_value; + double to_value; int data_cnt = dataArray.n(); if (1 == data_cnt) to_value = dataArray[0]; else if (RGInfo.method == InterpMthd::Min) to_value = dataArray.min(); @@ -1502,7 +1481,7 @@ void regrid_nc_variable(NcFile *nc_in, Met2dDataFile *fr_mtddf, to_value = (to_value + dataArray[(data_cnt/2)+1])/2; } else to_value = dataArray.sum() / data_cnt; // UW_Mean - + to_dp.set(to_value, xIdx, yIdx); to_cell_cnt++; if(mlog.verbosity_level() >= 9) { @@ -1529,9 +1508,9 @@ void regrid_nc_variable(NcFile *nc_in, Met2dDataFile *fr_mtddf, } } } - + delete [] from_data; - + mlog << Debug(4) << method_name << "[Count] data cells: " << to_cell_cnt << ", missing: " << missing_cnt << ", non_missing: " << non_missing_cnt << ", non mapped cells: " << no_map_cnt @@ -1541,7 +1520,7 @@ void regrid_nc_variable(NcFile *nc_in, Met2dDataFile *fr_mtddf, } if (to_cell_cnt == 0) { - mlog << Warning << "\n" << method_name + mlog << Warning << "\n" << method_name << " There are no matching cells between input and the target grid.\n\n"; } @@ -1551,7 +1530,7 @@ void regrid_nc_variable(NcFile *nc_in, Met2dDataFile *fr_mtddf, //////////////////////////////////////////////////////////////////////// -void open_nc(const Grid &grid, ConcatString run_cs) { +static void open_nc(const Grid &grid, ConcatString run_cs) { // Create output file nc_out = open_ncfile(OutputFilename.c_str(), true); @@ -1583,7 +1562,7 @@ void open_nc(const Grid &grid, ConcatString run_cs) { void write_nc_data(const DataPlane &dp, const Grid &grid, NcVar *data_var) { // Allocate memory to store data values for each grid point - float *data = new float [grid.nx()*grid.ny()]; + auto data = new float [grid.nx()*grid.ny()]; // Store the data int grid_nx = grid.nx(); @@ -1613,7 +1592,7 @@ void write_nc_data(const DataPlane &dp, const Grid &grid, NcVar *data_var) { void write_nc_data_int(const DataPlane &dp, const Grid &grid, NcVar *data_var) { // Allocate memory to store data values for each grid point - int *data = new int [grid.nx()*grid.ny()]; + auto data = new int [grid.nx()*grid.ny()]; // Store the data int grid_nx = grid.nx(); @@ -1640,8 +1619,8 @@ void write_nc_data_int(const DataPlane &dp, const Grid &grid, NcVar *data_var) { //////////////////////////////////////////////////////////////////////// -void write_nc(const DataPlane &dp, const Grid &grid, - const VarInfo *vinfo, const char *vname) { +static void write_nc(const DataPlane &dp, const Grid &grid, + const VarInfo *vinfo, const char *vname) { int deflate_level = compress_level; if (deflate_level < 0) deflate_level = 0; @@ -1662,8 +1641,8 @@ void write_nc(const DataPlane &dp, const Grid &grid, //////////////////////////////////////////////////////////////////////// -void write_nc_int(const DataPlane &dp, const Grid &grid, - const VarInfo *vinfo, const char *vname) { +static void write_nc_int(const DataPlane &dp, const Grid &grid, + const VarInfo *vinfo, const char *vname) { int deflate_level = compress_level; if (deflate_level < 0) deflate_level = 0; @@ -1689,27 +1668,27 @@ void write_nc_int(const DataPlane &dp, const Grid &grid, //////////////////////////////////////////////////////////////////////// -void process_goes_file(NcFile *nc_in, MetConfig &config, VarInfo *vinfo, - const Grid fr_grid, const Grid to_grid) { +static void process_goes_file(NcFile *nc_in, MetConfig &config, VarInfo *vinfo, + const Grid fr_grid, const Grid to_grid) { DataPlane fr_dp, to_dp; ConcatString vname; - int global_attr_count; + const int global_attr_count = GOES_global_attr_names.size(); bool opt_all_attrs = false; clock_t start_clock = clock(); - NcFile *nc_adp = (NcFile *) nullptr; + auto nc_adp = (NcFile *) nullptr; static const char *method_name = "process_goes_file() -> "; ConcatString tmp_dir = config.get_tmp_dir(); ConcatString geostationary_file(tmp_dir); geostationary_file.add("/"); - geostationary_file.add(make_geostationary_filename(fr_grid, to_grid, RGInfo.name)); + geostationary_file.add(make_geostationary_filename(fr_grid, to_grid)); // Open ADP file if exists - if (0 < AdpFilename.length() && file_exists(AdpFilename.c_str())) { - nc_adp = open_ncfile(AdpFilename.c_str()); + if (!adp_filename.empty() && file_exists(adp_filename.c_str())) { + nc_adp = open_ncfile(adp_filename.c_str()); if (IS_INVALID_NC_P(nc_adp)) { mlog << Error << "\n" << method_name - << "Can't open the ADP input \"" << AdpFilename << "\"\n\n"; + << "Can't open the ADP input \"" << adp_filename << "\"\n\n"; exit(1); } else if (is_time_mismatch(nc_in, nc_adp)) { @@ -1720,8 +1699,7 @@ void process_goes_file(NcFile *nc_in, MetConfig &config, VarInfo *vinfo, NcVar time_var = get_nc_var_time(nc_in); unixtime valid_time = find_valid_time(time_var); to_dp.set_size(to_grid.nx(), to_grid.ny()); - global_attr_count = sizeof(GOES_global_attr_names)/sizeof(*GOES_global_attr_names); - IntArray *cellMapping = new IntArray[to_grid.nx() * to_grid.ny()]; + auto cellMapping = new IntArray[to_grid.nx() * to_grid.ny()]; get_grid_mapping(fr_grid, to_grid, cellMapping, geostationary_file); // Loop through the requested fields @@ -1783,7 +1761,7 @@ void process_goes_file(NcFile *nc_in, MetConfig &config, VarInfo *vinfo, prob_dp.set_constant(0); for (int x=0; x mapVar = GET_NC_VARS_P(nc_in); - for (multimap::iterator itVar = mapVar.begin(); + for (auto itVar = mapVar.begin(); itVar != mapVar.end(); ++itVar) { if ((*itVar).first == "t" || string::npos != (*itVar).first.find("time")) { @@ -1822,9 +1800,8 @@ void process_goes_file(NcFile *nc_in, MetConfig &config, VarInfo *vinfo, copy_nc_var(nc_out, &from_var); } } - //copy_nc_atts(_nc_in, nc_out, opt_all_attrs); - delete nc_adp; nc_adp = 0; + delete nc_adp; nc_adp = nullptr; delete [] cellMapping; cellMapping = (IntArray *) nullptr; mlog << Debug(LEVEL_FOR_PERFORMANCE) << method_name << "took " << (clock()-start_clock)/double(CLOCKS_PER_SEC) << " seconds\n"; @@ -1985,20 +1962,20 @@ static void get_grid_mapping_latlon( int data_size = from_lat_count * from_lon_count; static const char *method_name = "get_grid_mapping(lats, lons) -> "; - int *to_cell_counts = new int[to_size]; - int *mapping_indices = new int[data_size]; + auto to_cell_counts = new int[to_size]; + auto mapping_indices = new int[data_size]; for (int xIdx=0; xIdx 0 ) { @@ -2104,8 +2081,8 @@ static bool get_grid_mapping(Grid fr_grid, Grid to_grid, IntArray *cellMapping, } else if (data_size > 0) { int last_idx = data_size - 1; - float *latitudes = new float[data_size]; - float *longitudes = new float[data_size]; + auto latitudes = new float[data_size]; + auto longitudes = new float[data_size]; status = get_nc_data(&var_lat, latitudes); if( status ) status = get_nc_data(&var_lon, longitudes); if( status ) { @@ -2138,7 +2115,7 @@ static unixtime find_valid_time(NcVar time_var) { if( IS_VALID_NC(time_var) || get_dim_count(&time_var) < 2) { int time_count = get_dim_size(&time_var, 0); - + double time_values [time_count + 1]; if (get_nc_data(&time_var, time_values)) { valid_time = compute_unixtime(&time_var, time_values[0]); @@ -2162,13 +2139,13 @@ static unixtime find_valid_time(NcVar time_var) { //////////////////////////////////////////////////////////////////////// -ConcatString get_goes_grid_input(MetConfig config, Grid fr_grid, Grid to_grid) { +static ConcatString get_goes_grid_input(MetConfig config, Grid fr_grid, Grid to_grid) { ConcatString run_string; ConcatString env_coord_name; ConcatString tmp_dir = config.get_tmp_dir(); ConcatString geostationary_file(tmp_dir); geostationary_file.add("/"); - geostationary_file.add(make_geostationary_filename(fr_grid, to_grid, RGInfo.name)); + geostationary_file.add(make_geostationary_filename(fr_grid, to_grid)); if (get_env(key_geostationary_data, env_coord_name) && env_coord_name.nonempty() && file_exists(env_coord_name.c_str())) { @@ -2183,7 +2160,7 @@ ConcatString get_goes_grid_input(MetConfig config, Grid fr_grid, Grid to_grid) { //////////////////////////////////////////////////////////////////////// -void get_grid_mapping(Grid fr_grid, Grid to_grid, IntArray *cellMapping, +static void get_grid_mapping(Grid fr_grid, Grid to_grid, IntArray *cellMapping, ConcatString geostationary_file) { static const char *method_name = "get_grid_mapping() -> "; DataPlane from_dp, to_dp; @@ -2212,7 +2189,7 @@ void get_grid_mapping(Grid fr_grid, Grid to_grid, IntArray *cellMapping, } // Override the from nx & ny from NetCDF if exists - NcFile *coord_nc_in = (NcFile *) nullptr; + auto coord_nc_in = (NcFile *) nullptr; if (has_coord_input) { mlog << Debug(2) << method_name << "Reading coord file: " << cur_coord_name << "\n"; coord_nc_in = open_ncfile(cur_coord_name.c_str()); @@ -2231,10 +2208,10 @@ void get_grid_mapping(Grid fr_grid, Grid to_grid, IntArray *cellMapping, to_dp.set_size(to_lon_count, to_lat_count); if (data_size > 0) { - float *latitudes = (float *)nullptr; - float *longitudes = (float *)nullptr; - float *latitudes_buf = (float *)nullptr; - float *longitudes_buf = (float *)nullptr; + auto latitudes = (float *)nullptr; + auto longitudes = (float *)nullptr; + auto latitudes_buf = (float *)nullptr; + auto longitudes_buf = (float *)nullptr; int buff_size = data_size*sizeof(float); GoesImagerData grid_data; grid_data.reset(); @@ -2272,8 +2249,8 @@ void get_grid_mapping(Grid fr_grid, Grid to_grid, IntArray *cellMapping, int lat_mis_matching_count = 0; int lon_matching_count = 0; int lon_mis_matching_count = 0; - float *tmp_lats = grid_data.lat_values; - float *tmp_lons = grid_data.lon_values; + const float *tmp_lats = grid_data.lat_values; + const float *tmp_lons = grid_data.lon_values; for (int idx=0; idx MISSING_LATLON) && (tmp_lats[idx] > MISSING_LATLON)) { @@ -2320,18 +2297,18 @@ void get_grid_mapping(Grid fr_grid, Grid to_grid, IntArray *cellMapping, } } } - if (0 == latitudes) { + if (nullptr == latitudes) { mlog << Error << "\n" << method_name << "Fail to get latitudes!\n\n"; } - else if (0 == longitudes) { + else if (nullptr == longitudes) { mlog << Error << "\n" << method_name << "Fail to get longitudes!\n\n"; } else { check_lat_lon(data_size, latitudes, longitudes); get_grid_mapping_latlon(from_dp, to_dp, to_grid, cellMapping, latitudes, - longitudes, from_lat_count, from_lon_count, 0, + longitudes, from_lat_count, from_lon_count, nullptr, !fr_grid.get_swap_to_north()); } @@ -2350,7 +2327,7 @@ void get_grid_mapping(Grid fr_grid, Grid to_grid, IntArray *cellMapping, //////////////////////////////////////////////////////////////////////// -int get_lat_count(NcFile *_nc) { +static int get_lat_count(NcFile *_nc) { int lat_count = 0; NcDim dim_lat = get_nc_dim(_nc, dim_name_lat); if(IS_INVALID_NC(dim_lat)) dim_lat = get_nc_dim(_nc, "y"); @@ -2358,7 +2335,9 @@ int get_lat_count(NcFile *_nc) { return lat_count; } -int get_lon_count(NcFile *_nc) { +//////////////////////////////////////////////////////////////////////// + +static int get_lon_count(NcFile *_nc) { int lon_count = 0; NcDim dim_lon = get_nc_dim(_nc, dim_name_lon); if(IS_INVALID_NC(dim_lon)) dim_lon = get_nc_dim(_nc, "x"); @@ -2390,8 +2369,7 @@ static NcVar get_goes_nc_var(NcFile *nc, const ConcatString var_name, //////////////////////////////////////////////////////////////////////// -static ConcatString make_geostationary_filename(Grid fr_grid, Grid to_grid, - ConcatString regrid_name) { +static ConcatString make_geostationary_filename(Grid fr_grid, Grid to_grid) { ConcatString geo_data_filename; GridInfo info = fr_grid.info(); @@ -2451,7 +2429,7 @@ static bool is_time_mismatch(NcFile *nc_in, NcFile *nc_adp) { //////////////////////////////////////////////////////////////////////// -void regrid_goes_variable(NcFile *nc_in, VarInfo *vinfo, +static void regrid_goes_variable(NcFile *nc_in, VarInfo *vinfo, DataPlane &fr_dp, DataPlane &to_dp, Grid fr_grid, Grid to_grid, IntArray *cellMapping, NcFile *nc_adp) { @@ -2468,10 +2446,10 @@ void regrid_goes_variable(NcFile *nc_in, VarInfo *vinfo, ConcatString goes_var_sub_name; ConcatString qc_var_name; uchar qc_value; - uchar *qc_data = new uchar[from_data_size]; - uchar *adp_data = new uchar[from_data_size]; - float *from_data = new float[from_data_size]; - unsigned short *adp_qc_data = new unsigned short[from_data_size]; + auto qc_data = new uchar[from_data_size]; + auto adp_data = new uchar[from_data_size]; + auto from_data = new float[from_data_size]; + auto adp_qc_data = new unsigned short[from_data_size]; static const char *method_name = "regrid_goes_variable() -> "; // -99 is arbitrary number as invalid QC value @@ -2488,7 +2466,7 @@ void regrid_goes_variable(NcFile *nc_in, VarInfo *vinfo, bool is_dust_only = false; bool is_smoke_only = false; string actual_var_name = GET_NC_NAME(var_data); - int actual_var_len = actual_var_name.length(); + auto actual_var_len = actual_var_name.length(); bool is_adp_variable = (0 != actual_var_name.compare(vinfo->name().c_str())); memset(adp_data, 1, from_data_size*sizeof(uchar)); // Default: 1 = data present @@ -2505,7 +2483,7 @@ void regrid_goes_variable(NcFile *nc_in, VarInfo *vinfo, if (IS_VALID_NC(var_adp)) { get_nc_data(&var_adp, adp_data, true); - //Smoke:ancillary_variables = "DQF" ; ubyte DQF(y, x) ; + //ADP::Smoke:ancillary_variables = ubyte DQF(y, x) if (get_att_value_string(&var_adp, (string)"ancillary_variables", qc_var_name)) { var_adp_qc = get_nc_var(nc_adp, qc_var_name.c_str()); if (IS_VALID_NC(var_adp_qc)) { @@ -2527,7 +2505,7 @@ void regrid_goes_variable(NcFile *nc_in, VarInfo *vinfo, mlog << Debug(5) << method_name << "is_dust: " << is_dust_only << ", is_smoke: " << is_smoke_only << "\n"; - //AOD:ancillary_variables = "DQF" ; byte DQF(y, x) ; + //AOD:ancillary_variables = byte DQF(y, x) if (get_att_value_string(&var_data, (string)"ancillary_variables", qc_var_name)) { var_qc = get_nc_var(nc_in, qc_var_name.c_str()); if (IS_VALID_NC(var_qc)) { @@ -2542,7 +2520,7 @@ void regrid_goes_variable(NcFile *nc_in, VarInfo *vinfo, } } - get_nc_data(&var_data, (float *)from_data); + get_nc_data(&var_data, from_data); fr_dp.set_size(from_lon_count, from_lat_count); for (int xIdx=0; xIdx= log_debug_level) { switch (qc_for_flag) { case MET_ADP_QC_HIGH: cnt_adp_qc_high++; break; case MET_ADP_QC_MEDIUM: cnt_adp_qc_medium++; break; case MET_ADP_QC_LOW: cnt_adp_qc_low++; break; - default: cnt_adp_qc_nr++; break; + default: cnt_adp_qc_nr++; break; } } @@ -2679,7 +2657,7 @@ void regrid_goes_variable(NcFile *nc_in, VarInfo *vinfo, if (0 < dataArray.n()) { int data_count = dataArray.n(); - float to_value; + double to_value; if (RGInfo.method == InterpMthd::Min) to_value = dataArray.min(); else if (RGInfo.method == InterpMthd::Max) to_value = dataArray.max(); else if (RGInfo.method == InterpMthd::Median) { @@ -2767,7 +2745,7 @@ static void save_geostationary_data(const ConcatString geostationary_file, add_att(&lat_var, "units","degrees_south"); } add_att(&lat_var, "dy_rad", grid_data.dy_rad); - if(!put_nc_data((NcVar *)&lat_var, latitudes)) { + if(!put_nc_data(&lat_var, latitudes)) { has_error = true; mlog << Warning << "\nsave_geostationary_data() -> " << "Cannot save latitudes!\n\n"; @@ -2783,7 +2761,7 @@ static void save_geostationary_data(const ConcatString geostationary_file, add_att(&lon_var, "units","degrees_west"); } add_att(&lon_var, "dx_rad", grid_data.dx_rad); - if(!put_nc_data((NcVar *)&lon_var, longitudes)) { + if(!put_nc_data(&lon_var, longitudes)) { has_error = true; mlog << Warning << "\nsave_geostationary_data() -> " << "Cannot save longitudes!\n\n"; @@ -2803,7 +2781,7 @@ static void save_geostationary_data(const ConcatString geostationary_file, << geostationary_file << ") was saved\n"; } - delete nc_file; nc_file = 0; + delete nc_file; nc_file = nullptr; mlog << Debug(LEVEL_FOR_PERFORMANCE) << method_name << "took " << (clock()-start_clock)/double(CLOCKS_PER_SEC) << " seconds\n"; @@ -2811,7 +2789,7 @@ static void save_geostationary_data(const ConcatString geostationary_file, //////////////////////////////////////////////////////////////////////// -void close_nc() { +static void close_nc() { // Clean up if(nc_out) { @@ -2827,8 +2805,7 @@ void close_nc() { //////////////////////////////////////////////////////////////////////// -//bool keep_message_type(const char *mt_str) { -bool keep_message_type(const int mt_index) { +static bool keep_message_type(const int mt_index) { bool keep = false; @@ -2840,7 +2817,7 @@ bool keep_message_type(const int mt_index) { //////////////////////////////////////////////////////////////////////// -bool has_lat_lon_vars(NcFile *nc) { +static bool has_lat_lon_vars(NcFile *nc) { bool has_lat_var = IS_VALID_NC(get_nc_var_lat(nc)); bool has_lon_var = IS_VALID_NC(get_nc_var_lon(nc)); @@ -2856,7 +2833,7 @@ bool has_lat_lon_vars(NcFile *nc) { //////////////////////////////////////////////////////////////////////// -void usage() { +static void usage() { cout << "\n*** Model Evaluation Tools (MET" << met_version << ") ***\n\n" @@ -2932,17 +2909,16 @@ void usage() { //////////////////////////////////////////////////////////////////////// -void set_adp_gc_values(NcVar var_adp_qc) { +static void set_adp_gc_values(NcVar var_adp_qc) { ConcatString att_flag_meanings; if (get_nc_att_value(&var_adp_qc, (ConcatString)"flag_meanings", att_flag_meanings)) { StringArray flag_meanings = to_lower(att_flag_meanings).split(" "); - unsigned short flag_values[256]; + unsigned short flag_values[flag_meanings.n()+128]; /* reserve enough storage */ for (int i=0; i> 2) & 0x03; } @@ -2962,13 +2938,13 @@ void set_adp_gc_values(NcVar var_adp_qc) { //////////////////////////////////////////////////////////////////////// -void set_field(const StringArray &a) { +static void set_field(const StringArray &a) { FieldSA.add(a[0]); } //////////////////////////////////////////////////////////////////////// -void set_method(const StringArray &a) { +static void set_method(const StringArray &a) { InterpMthd method_id = string_to_interpmthd(a[0].c_str()); if (method_id == InterpMthd::Gaussian || method_id == InterpMthd::MaxGauss ) { do_gaussian_filter = true; @@ -2980,13 +2956,13 @@ void set_method(const StringArray &a) { //////////////////////////////////////////////////////////////////////// -void set_prob_cat_thresh(const StringArray &a) { +static void set_prob_cat_thresh(const StringArray &a) { prob_cat_thresh.set(a[0].c_str()); } //////////////////////////////////////////////////////////////////////// -void set_vld_thresh(const StringArray &a) { +static void set_vld_thresh(const StringArray &a) { RGInfo.vld_thresh = atof(a[0].c_str()); if(RGInfo.vld_thresh > 1 || RGInfo.vld_thresh < 0) { mlog << Error << "\nset_vld_thresh() -> " @@ -2998,25 +2974,25 @@ void set_vld_thresh(const StringArray &a) { //////////////////////////////////////////////////////////////////////// -void set_name(const StringArray & a) { +static void set_name(const StringArray & a) { VarNameSA.add_css(a[0]); } //////////////////////////////////////////////////////////////////////// -void set_config(const StringArray & a) { +static void set_config(const StringArray & a) { config_filename = a[0]; } //////////////////////////////////////////////////////////////////////// -void set_compress(const StringArray & a) { +static void set_compress(const StringArray & a) { compress_level = atoi(a[0].c_str()); } //////////////////////////////////////////////////////////////////////// -void set_qc_flags(const StringArray & a) { +static void set_qc_flags(const StringArray & a) { int qc_flag; StringArray sa; @@ -3029,24 +3005,24 @@ void set_qc_flags(const StringArray & a) { //////////////////////////////////////////////////////////////////////// -void set_adp(const StringArray & a) { - AdpFilename = a[0]; - if (!file_exists(AdpFilename.c_str())) { +static void set_adp(const StringArray & a) { + adp_filename = a[0]; + if (!file_exists(adp_filename.c_str())) { mlog << Error << "\nset_adp() -> " - << "\"" << AdpFilename << "\" does not exist\n\n"; + << "\"" << adp_filename << "\" does not exist\n\n"; exit(1); } } //////////////////////////////////////////////////////////////////////// -void set_gaussian_dx(const StringArray &a) { +static void set_gaussian_dx(const StringArray &a) { RGInfo.gaussian.dx = atof(a[0].c_str()); } //////////////////////////////////////////////////////////////////////// -void set_gaussian_radius(const StringArray &a) { +static void set_gaussian_radius(const StringArray &a) { RGInfo.gaussian.radius = atof(a[0].c_str()); } From 6fd77219fc8eb74789faaf8ab6dfb50263e1b55b Mon Sep 17 00:00:00 2001 From: Howard Soh Date: Fri, 10 May 2024 05:02:35 +0000 Subject: [PATCH 13/30] #2867 Deleted protected section with no members --- src/tools/other/point2grid/point2grid_conf_info.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/tools/other/point2grid/point2grid_conf_info.h b/src/tools/other/point2grid/point2grid_conf_info.h index 0a12d0baa9..5b12b6166e 100644 --- a/src/tools/other/point2grid/point2grid_conf_info.h +++ b/src/tools/other/point2grid/point2grid_conf_info.h @@ -32,8 +32,6 @@ class PointToGridConfInfo { std::map var_name_map; std::map def_var_name_map; - protected: - public: // PointToGrid configuration object From d2084b6cf45f0ef8ade6f85655938728121631a5 Mon Sep 17 00:00:00 2001 From: Howard Soh Date: Fri, 10 May 2024 05:03:22 +0000 Subject: [PATCH 14/30] #2867 Cleanup --- src/tools/other/point2grid/point2grid_conf_info.cc | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/src/tools/other/point2grid/point2grid_conf_info.cc b/src/tools/other/point2grid/point2grid_conf_info.cc index 51e0c0a996..ad661ebfe8 100644 --- a/src/tools/other/point2grid/point2grid_conf_info.cc +++ b/src/tools/other/point2grid/point2grid_conf_info.cc @@ -136,7 +136,7 @@ ConcatString PointToGridConfInfo::get_var_id(const ConcatString &var_name) { break; } } - + if( var_id.empty() ) { for (ptr=def_var_name_map.begin(); ptr != def_var_name_map.end(); ptr++) { if( ptr->second == var_name ) { @@ -153,15 +153,12 @@ ConcatString PointToGridConfInfo::get_var_id(const ConcatString &var_name) { ConcatString PointToGridConfInfo::get_var_name(const ConcatString &var_name) { ConcatString out_var; - ConcatString t_name; - - t_name = var_name_map[var_name]; + ConcatString t_name = var_name_map[var_name]; if (t_name.empty()) t_name = def_var_name_map[var_name]; if (t_name.empty()) { - ConcatString tmp_key, tmp_value; - tmp_key = "grib_code_"; + ConcatString tmp_key = "grib_code_"; tmp_key << atoi(var_name.c_str()); - tmp_value = var_name_map[tmp_key]; + ConcatString tmp_value = var_name_map[tmp_key]; if (tmp_value.empty()) tmp_value = def_var_name_map[tmp_key]; if (!tmp_value.empty()) t_name = tmp_value; } From 68b6f7001e1d886afc791801ec1fc85d1ba8f23b Mon Sep 17 00:00:00 2001 From: Howard Soh Date: Fri, 10 May 2024 05:04:31 +0000 Subject: [PATCH 15/30] #2867 FIxed SonarQube findings; unused local variables, decalare as const, etc --- src/tools/other/point2grid/point2grid.cc | 134 ++++++++++------------- 1 file changed, 55 insertions(+), 79 deletions(-) diff --git a/src/tools/other/point2grid/point2grid.cc b/src/tools/other/point2grid/point2grid.cc index a919f817ea..d64bd73d7c 100644 --- a/src/tools/other/point2grid/point2grid.cc +++ b/src/tools/other/point2grid/point2grid.cc @@ -76,7 +76,7 @@ constexpr int LEVEL_FOR_PERFORMANCE = 6; constexpr char default_config_filename[] = "MET_BASE/config/Point2GridConfig_default"; -std::vector GOES_global_attr_names = { +static const vector GOES_global_attr_names = { "naming_authority", "project", "production_site", @@ -167,9 +167,9 @@ static void set_gaussian_dx(const StringArray &); static void set_gaussian_radius(const StringArray &); static unixtime compute_unixtime(NcVar *time_var, unixtime var_value); -static bool get_grid_mapping(Grid fr_grid, Grid to_grid, IntArray *cellMapping, +static bool get_grid_mapping(const Grid fr_grid, const Grid to_grid, IntArray *cellMapping, NcVar var_lat, NcVar var_lon, bool *skip_times); -static bool get_grid_mapping(Grid to_grid, IntArray *cellMapping, +static bool get_grid_mapping(const Grid to_grid, IntArray *cellMapping, const IntArray obs_index_array, const int *obs_hids, const float *hdr_lats, const float *hdr_lons); static int get_obs_type(NcFile *nc_in); @@ -200,11 +200,11 @@ static void process_goes_file(NcFile *nc_in, MetConfig &config, VarInfo *, const Grid fr_grid, const Grid to_grid); static unixtime find_valid_time(NcVar time_var); static ConcatString get_goes_grid_input(MetConfig config, Grid fr_grid, Grid to_grid); -static void get_grid_mapping(Grid fr_grid, Grid to_grid, - IntArray *cellMapping, ConcatString geostationary_file); +static void get_grid_mapping(const Grid fr_grid, const Grid to_grid, + IntArray *cellMapping, ConcatString &geostationary_file); static int get_lat_count(NcFile *); static int get_lon_count(NcFile *); -static NcVar get_goes_nc_var(NcFile *nc, const ConcatString var_name, +static NcVar get_goes_nc_var(NcFile *nc, const ConcatString &var_name, bool exit_if_error=true); static bool is_time_mismatch(NcFile *nc_in, NcFile *nc_adp); static ConcatString make_geostationary_filename(Grid fr_grid, Grid to_grid); @@ -213,7 +213,7 @@ static void regrid_goes_variable(NcFile *nc_in, VarInfo *vinfo, Grid fr_grid, Grid to_grid, IntArray *cellMapping, NcFile *nc_adp); static void save_geostationary_data(const ConcatString geostationary_file, const float *latitudes, const float *longitudes, - const GoesImagerData grid_data); + const GoesImagerData &grid_data); static void set_qc_flags(const StringArray &); //////////////////////////////////////////////////////////////////////// @@ -234,7 +234,7 @@ int met_main(int argc, char *argv[]) { //////////////////////////////////////////////////////////////////////// -const string get_tool_name() { +string get_tool_name() { return "point2grid"; } @@ -372,7 +372,7 @@ static void process_command_line(int argc, char **argv) { //////////////////////////////////////////////////////////////////////// static void process_data_file() { - Grid fr_grid, to_grid; + Grid fr_grid; GrdFileType ftype; ConcatString run_cs; auto nc_in = (NcFile *) nullptr; @@ -436,7 +436,6 @@ static void process_data_file() { ftype = fr_mtddf->file_type(); // Setup the VarInfo request object - VarInfoFactory v_factory; VarInfo *vinfo = VarInfoFactory::new_var_info(ftype); if(!vinfo) { @@ -456,7 +455,7 @@ static void process_data_file() { #endif // Determine the "to" grid - to_grid = parse_vx_grid(RGInfo, &fr_grid, &fr_grid); + Grid to_grid = parse_vx_grid(RGInfo, &fr_grid, &fr_grid); mlog << Debug(2) << "Interpolation options: " << "method = " << interpmthd_to_string(RGInfo.method) @@ -657,7 +656,7 @@ void prepare_message_types(const StringArray &hdr_types) { //////////////////////////////////////////////////////////////////////// -IntArray prepare_qc_array(const IntArray &_qc_flags, StringArray &qc_tables) { +IntArray prepare_qc_array(const IntArray &_qc_flags, const StringArray &qc_tables) { IntArray qc_idx_array; bool has_qc_flags = (_qc_flags.n() > 0); if (has_qc_flags) { @@ -676,16 +675,14 @@ IntArray prepare_qc_array(const IntArray &_qc_flags, StringArray &qc_tables) { void process_point_met_data(MetPointData *met_point_obs, MetConfig &config, VarInfo *vinfo, const Grid &to_grid) { - int nhdr, nobs; - int nx, ny, var_count, to_count, var_count2; + int var_count, to_count, var_count2; int idx, hdr_idx; int var_idx_or_gc; int filtered_by_time, filtered_by_msg_type, filtered_by_qc; - ConcatString vname, vname_cnt, vname_mask; + ConcatString vname; DataPlane fr_dp, to_dp; DataPlane cnt_dp, mask_dp; DataPlane prob_dp, prob_mask_dp; - NcVar var_obs_gc, var_obs_var; bool has_prob_thresh = !prob_cat_thresh.check(bad_data_double); @@ -702,8 +699,8 @@ void process_point_met_data(MetPointData *met_point_obs, MetConfig &config, VarI const MetPointHeader *header_data = met_point_obs->get_header_data(); MetPointObsData *obs_data = met_point_obs->get_point_obs_data(); - nhdr = met_point_obs->get_hdr_cnt(); - nobs = met_point_obs->get_obs_cnt(); + int nhdr = met_point_obs->get_hdr_cnt(); + int nobs = met_point_obs->get_obs_cnt(); bool empty_input = (nhdr == 0 && nobs == 0); bool use_var_id = met_point_obs->is_using_var_id(); @@ -728,8 +725,8 @@ void process_point_met_data(MetPointData *met_point_obs, MetConfig &config, VarI IntArray qc_idx_array = prepare_qc_array(qc_flags, qc_tables); // Initialize size and values of output fields - nx = to_grid.nx(); - ny = to_grid.ny(); + int nx = to_grid.nx(); + int ny = to_grid.ny(); to_dp.set_size(nx, ny); to_dp.set_constant(bad_data_double); cnt_dp.set_size(nx, ny); @@ -744,12 +741,11 @@ void process_point_met_data(MetPointData *met_point_obs, MetConfig &config, VarI } // Loop through the requested fields - int obs_count_zero_to, obs_count_non_zero_to; - int obs_count_zero_from, obs_count_non_zero_from; - IntArray *cellMapping = (IntArray *) nullptr; - - obs_count_zero_to = obs_count_non_zero_to = 0; - obs_count_zero_from = obs_count_non_zero_from = 0; + int obs_count_zero_to = 0; + int obs_count_zero_from = 0; + int obs_count_non_zero_to = 0; + int obs_count_non_zero_from = 0; + auto cellMapping = (IntArray *) nullptr; for(int i=0; ilong_name(); ConcatString dim_string = "(*,*)"; - tmp_long_name = vname_cnt; + ConcatString tmp_long_name = vname_cnt; tmp_long_name << dim_string; vinfo->set_long_name(tmp_long_name.c_str()); write_nc_int(cnt_dp, to_grid, vinfo, vname_cnt.c_str()); @@ -1138,15 +1133,9 @@ void process_point_met_data(MetPointData *met_point_obs, MetConfig &config, VarI //////////////////////////////////////////////////////////////////////// static void process_point_file(NcFile *nc_in, MetConfig &config, VarInfo *vinfo, - const Grid to_grid) { - ConcatString vname, vname_cnt, vname_mask; - DataPlane fr_dp, to_dp; - DataPlane cnt_dp, mask_dp; - DataPlane prob_dp, prob_mask_dp; - NcVar var_obs_gc, var_obs_var; - + const Grid to_grid) { + ConcatString vname; clock_t start_clock = clock(); - static const char *method_name = "process_point_file() -> "; static const char *method_name_s = "process_point_file()"; @@ -1179,12 +1168,6 @@ static void process_point_file(NcFile *nc_in, MetConfig &config, VarInfo *vinfo, static void process_point_python(string python_command, MetConfig &config, VarInfo *vinfo, const Grid to_grid, bool use_xarray) { - ConcatString vname, vname_cnt, vname_mask; - DataPlane fr_dp, to_dp; - DataPlane cnt_dp, mask_dp; - DataPlane prob_dp, prob_mask_dp; - NcVar var_obs_gc, var_obs_var; - clock_t start_clock = clock(); static const char *method_name = "process_point_python() -> "; @@ -1223,7 +1206,7 @@ static void process_point_python(string python_command, MetConfig &config, VarIn static void process_point_nccf_file(NcFile *nc_in, MetConfig &config, VarInfo *vinfo, Met2dDataFile *fr_mtddf, const Grid to_grid) { - ConcatString vname, vname_cnt, vname_mask; + ConcatString vname; DataPlane fr_dp, to_dp; DataPlane cnt_dp, mask_dp; unixtime valid_beg_ut, valid_end_ut; @@ -1348,7 +1331,7 @@ static void process_point_nccf_file(NcFile *nc_in, MetConfig &config, prob_dp.set_constant(0); for (int x=0; xname()); if(IS_VALID_NC(var_data)) { - for (int idx=0; idx= to_size ) { mlog << Error << "\n" << method_name @@ -2046,7 +2022,7 @@ static void get_grid_mapping_latlon( //////////////////////////////////////////////////////////////////////// -static bool get_grid_mapping(Grid fr_grid, Grid to_grid, IntArray *cellMapping, +static bool get_grid_mapping(const Grid fr_grid, const Grid to_grid, IntArray *cellMapping, NcVar var_lat, NcVar var_lon, bool *skip_times) { bool status = false; DataPlane from_dp, to_dp; @@ -2160,8 +2136,8 @@ static ConcatString get_goes_grid_input(MetConfig config, Grid fr_grid, Grid to_ //////////////////////////////////////////////////////////////////////// -static void get_grid_mapping(Grid fr_grid, Grid to_grid, IntArray *cellMapping, - ConcatString geostationary_file) { +static void get_grid_mapping(const Grid fr_grid, const Grid to_grid, IntArray *cellMapping, + ConcatString &geostationary_file) { static const char *method_name = "get_grid_mapping() -> "; DataPlane from_dp, to_dp; ConcatString cur_coord_name; @@ -2347,7 +2323,7 @@ static int get_lon_count(NcFile *_nc) { //////////////////////////////////////////////////////////////////////// -static NcVar get_goes_nc_var(NcFile *nc, const ConcatString var_name, +static NcVar get_goes_nc_var(NcFile *nc, const ConcatString &var_name, bool exit_if_error) { NcVar var_data; static const char *method_name = "get_goes_nc_var() -> "; @@ -2483,7 +2459,7 @@ static void regrid_goes_variable(NcFile *nc_in, VarInfo *vinfo, if (IS_VALID_NC(var_adp)) { get_nc_data(&var_adp, adp_data, true); - //ADP::Smoke:ancillary_variables = ubyte DQF(y, x) + //ADP Smoke:ancillary_variables: ubyte DQF(y, x) if (get_att_value_string(&var_adp, (string)"ancillary_variables", qc_var_name)) { var_adp_qc = get_nc_var(nc_adp, qc_var_name.c_str()); if (IS_VALID_NC(var_adp_qc)) { @@ -2505,7 +2481,7 @@ static void regrid_goes_variable(NcFile *nc_in, VarInfo *vinfo, mlog << Debug(5) << method_name << "is_dust: " << is_dust_only << ", is_smoke: " << is_smoke_only << "\n"; - //AOD:ancillary_variables = byte DQF(y, x) + //AOD ancillary_variables: byte DQF(y, x) if (get_att_value_string(&var_data, (string)"ancillary_variables", qc_var_name)) { var_qc = get_nc_var(nc_in, qc_var_name.c_str()); if (IS_VALID_NC(var_qc)) { @@ -2722,7 +2698,7 @@ static void regrid_goes_variable(NcFile *nc_in, VarInfo *vinfo, static void save_geostationary_data(const ConcatString geostationary_file, const float *latitudes, const float *longitudes, - const GoesImagerData grid_data) { + const GoesImagerData &grid_data) { bool has_error = false; int deflate_level = 0; clock_t start_clock = clock(); From 7200b8b8f2b4f19366e5cf53535e902e773eabaf Mon Sep 17 00:00:00 2001 From: Howard Soh Date: Fri, 10 May 2024 05:34:51 +0000 Subject: [PATCH 16/30] #2867 MOved include directives to top --- src/libcode/vx_nc_util/nc_utils.h | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/libcode/vx_nc_util/nc_utils.h b/src/libcode/vx_nc_util/nc_utils.h index c2b4dfc823..212771152a 100644 --- a/src/libcode/vx_nc_util/nc_utils.h +++ b/src/libcode/vx_nc_util/nc_utils.h @@ -18,6 +18,14 @@ #include +#include "concat_string.h" +#include "int_array.h" +#include "long_array.h" +#include "num_array.h" +#include "nc_var_info.h" + +//////////////////////////////////////////////////////////////////////// + #ifndef ncbyte typedef signed char ncbyte; /* from ncvalues.h */ #endif /* ncbyte */ @@ -25,12 +33,6 @@ typedef signed char ncbyte; /* from ncvalues.h */ typedef unsigned char uchar; #endif /* uchar */ -#include "concat_string.h" -#include "int_array.h" -#include "long_array.h" -#include "num_array.h" -#include "nc_var_info.h" - //////////////////////////////////////////////////////////////////////// static const std::string C_unknown_str = std::string("unknown"); From 2f982d82f7ce5d68b5b366b7fa426656e96d86fb Mon Sep 17 00:00:00 2001 From: Howard Soh Date: Fri, 10 May 2024 05:47:00 +0000 Subject: [PATCH 17/30] #2867 Changed some argumenmt with references to avoid copying objects --- src/tools/other/point2grid/point2grid.cc | 27 ++++++++++++------------ 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/src/tools/other/point2grid/point2grid.cc b/src/tools/other/point2grid/point2grid.cc index d64bd73d7c..31ea9f2f24 100644 --- a/src/tools/other/point2grid/point2grid.cc +++ b/src/tools/other/point2grid/point2grid.cc @@ -167,9 +167,9 @@ static void set_gaussian_dx(const StringArray &); static void set_gaussian_radius(const StringArray &); static unixtime compute_unixtime(NcVar *time_var, unixtime var_value); -static bool get_grid_mapping(const Grid fr_grid, const Grid to_grid, IntArray *cellMapping, +static bool get_grid_mapping(const Grid &fr_grid, const Grid &to_grid, IntArray *cellMapping, NcVar var_lat, NcVar var_lon, bool *skip_times); -static bool get_grid_mapping(const Grid to_grid, IntArray *cellMapping, +static bool get_grid_mapping(const Grid &to_grid, IntArray *cellMapping, const IntArray obs_index_array, const int *obs_hids, const float *hdr_lats, const float *hdr_lons); static int get_obs_type(NcFile *nc_in); @@ -200,8 +200,8 @@ static void process_goes_file(NcFile *nc_in, MetConfig &config, VarInfo *, const Grid fr_grid, const Grid to_grid); static unixtime find_valid_time(NcVar time_var); static ConcatString get_goes_grid_input(MetConfig config, Grid fr_grid, Grid to_grid); -static void get_grid_mapping(const Grid fr_grid, const Grid to_grid, - IntArray *cellMapping, ConcatString &geostationary_file); +static void get_grid_mapping(const Grid &fr_grid, const Grid &to_grid, + IntArray *cellMapping, const ConcatString &geostationary_file); static int get_lat_count(NcFile *); static int get_lon_count(NcFile *); static NcVar get_goes_nc_var(NcFile *nc, const ConcatString &var_name, @@ -675,10 +675,8 @@ IntArray prepare_qc_array(const IntArray &_qc_flags, const StringArray &qc_table void process_point_met_data(MetPointData *met_point_obs, MetConfig &config, VarInfo *vinfo, const Grid &to_grid) { - int var_count, to_count, var_count2; int idx, hdr_idx; int var_idx_or_gc; - int filtered_by_time, filtered_by_msg_type, filtered_by_qc; ConcatString vname; DataPlane fr_dp, to_dp; DataPlane cnt_dp, mask_dp; @@ -893,8 +891,12 @@ void process_point_met_data(MetPointData *met_point_obs, MetConfig &config, VarI mlog << Debug(4) << method_name << "var: " << vname << ", index: " << var_idx_or_gc << ".\n"; - var_count = var_count2 = to_count = 0; - filtered_by_time = filtered_by_msg_type = filtered_by_qc = 0; + int var_count = 0; + int var_count2 = 0; + int to_count = 0; + int filtered_by_time = 0; + int filtered_by_msg_type = 0; + int filtered_by_qc = 0; for (idx=0; idx < nobs; idx++) { if (var_idx_or_gc == obs_data->obs_ids[idx]) { var_count2++; @@ -941,7 +943,6 @@ void process_point_met_data(MetPointData *met_point_obs, MetConfig &config, VarI double from_max_value = -10e10; // Initialize counter and output fields - to_count = 0; to_dp.set_constant(bad_data_double); cnt_dp.set_constant(0); mask_dp.set_constant(0); @@ -1871,7 +1872,7 @@ static unixtime compute_unixtime(NcVar *time_var, unixtime var_value) { //////////////////////////////////////////////////////////////////////// -static bool get_grid_mapping(const Grid to_grid, IntArray *cellMapping, +static bool get_grid_mapping(const Grid &to_grid, IntArray *cellMapping, const IntArray obs_index_array, const int *obs_hids, const float *hdr_lats, const float *hdr_lons) { bool status = false; @@ -2022,7 +2023,7 @@ static void get_grid_mapping_latlon( //////////////////////////////////////////////////////////////////////// -static bool get_grid_mapping(const Grid fr_grid, const Grid to_grid, IntArray *cellMapping, +static bool get_grid_mapping(const Grid &fr_grid, const Grid &to_grid, IntArray *cellMapping, NcVar var_lat, NcVar var_lon, bool *skip_times) { bool status = false; DataPlane from_dp, to_dp; @@ -2136,8 +2137,8 @@ static ConcatString get_goes_grid_input(MetConfig config, Grid fr_grid, Grid to_ //////////////////////////////////////////////////////////////////////// -static void get_grid_mapping(const Grid fr_grid, const Grid to_grid, IntArray *cellMapping, - ConcatString &geostationary_file) { +static void get_grid_mapping(const Grid &fr_grid, const Grid &to_grid, IntArray *cellMapping, + const ConcatString &geostationary_file) { static const char *method_name = "get_grid_mapping() -> "; DataPlane from_dp, to_dp; ConcatString cur_coord_name; From 11d75333755628237009d16de3f9f5eade2d62ff Mon Sep 17 00:00:00 2001 From: Howard Soh Date: Fri, 10 May 2024 17:05:30 +0000 Subject: [PATCH 18/30] #2867 Do not filter by QC flag if -qc is not given --- src/tools/other/point2grid/point2grid.cc | 86 ++++++++++++------------ 1 file changed, 44 insertions(+), 42 deletions(-) diff --git a/src/tools/other/point2grid/point2grid.cc b/src/tools/other/point2grid/point2grid.cc index 31ea9f2f24..db5fea47a2 100644 --- a/src/tools/other/point2grid/point2grid.cc +++ b/src/tools/other/point2grid/point2grid.cc @@ -127,10 +127,10 @@ static int adp_qc_high; /* 3 as baseline algorithm, 0 for enterpirse algorit static int adp_qc_medium; /* 1 as baseline algorithm, 1 for enterpirse algorithm */ static int adp_qc_low; /* 0 as baseline algorithm, 2 for enterpirse algorithm */ -constexpr int MET_ADP_QC_HIGH = 0; -constexpr int MET_ADP_QC_MEDIUM = 1; -constexpr int MET_ADP_QC_LOW = 2; -constexpr int MET_ADP_QC_NA = bad_data_int; +constexpr int MET_QC_HIGH = 0; +constexpr int MET_QC_MEDIUM = 1; +constexpr int MET_QC_LOW = 2; +constexpr int MET_QC_NA = 3; static const ConcatString att_name_values = "flag_values"; static const ConcatString att_name_meanings = "flag_meanings"; @@ -1841,14 +1841,14 @@ void check_lat_lon(int data_size, float *latitudes, float *longitudes) { int compute_adp_qc_flag(int adp_qc, int shift_bits) { int particle_qc = ((adp_qc >> shift_bits) & 0x03); - int qc_for_flag; + int adp_qc_flag; - if (particle_qc == adp_qc_high) qc_for_flag = MET_ADP_QC_HIGH; - else if (particle_qc == adp_qc_medium) qc_for_flag = MET_ADP_QC_MEDIUM; - else if (particle_qc == adp_qc_low) qc_for_flag = MET_ADP_QC_LOW; - else qc_for_flag = MET_ADP_QC_NA; + if (particle_qc == adp_qc_high) adp_qc_flag = MET_QC_HIGH; + else if (particle_qc == adp_qc_medium) adp_qc_flag = MET_QC_MEDIUM; + else if (particle_qc == adp_qc_low) adp_qc_flag = MET_QC_LOW; + else adp_qc_flag = MET_QC_NA; - return qc_for_flag; + return adp_qc_flag; } @@ -2578,52 +2578,51 @@ static void regrid_goes_variable(NcFile *nc_in, VarInfo *vinfo, } // Filter by QC flag - if (has_qc_var || has_adp_qc_var) { + if (has_qc_flags && (has_qc_var || has_adp_qc_var)) { qc_value = qc_data[from_index]; if (mlog.verbosity_level() >= log_debug_level) { if (qc_min_value > qc_value) qc_min_value = qc_value; if (qc_max_value < qc_value) qc_max_value = qc_value; switch (qc_value) { - case MET_ADP_QC_HIGH: cnt_aod_qc_high++; break; - case MET_ADP_QC_MEDIUM: cnt_aod_qc_medium++; break; - case MET_ADP_QC_LOW: cnt_aod_qc_low++; break; - default: cnt_aod_qc_nr++; break; + case MET_QC_HIGH: cnt_aod_qc_high++; break; + case MET_QC_MEDIUM: cnt_aod_qc_medium++; break; + case MET_QC_LOW: cnt_aod_qc_low++; break; + default: cnt_aod_qc_nr++; break; } } if (has_adp_qc_var) { - int qc_for_flag = compute_adp_qc_flag(adp_qc_data[from_index], shift_bits); - bool filter_out = is_eq(qc_for_flag, MET_ADP_QC_NA); - + int adp_qc_flag = compute_adp_qc_flag(adp_qc_data[from_index], shift_bits); if (mlog.verbosity_level() >= log_debug_level) { - switch (qc_for_flag) { - case MET_ADP_QC_HIGH: cnt_adp_qc_high++; break; - case MET_ADP_QC_MEDIUM: cnt_adp_qc_medium++; break; - case MET_ADP_QC_LOW: cnt_adp_qc_low++; break; - default: cnt_adp_qc_nr++; break; + switch (adp_qc_flag) { + case MET_QC_HIGH: cnt_adp_qc_high++; break; + case MET_QC_MEDIUM: cnt_adp_qc_medium++; break; + case MET_QC_LOW: cnt_adp_qc_low++; break; + default: cnt_adp_qc_nr++; break; } } + bool filter_out = is_eq(adp_qc_flag, MET_QC_NA); if (!filter_out) { /* Adjust the quality by AOD data QC */ - if (MET_ADP_QC_LOW == qc_value) { - if (MET_ADP_QC_LOW > qc_for_flag) { - if (MET_ADP_QC_HIGH == qc_for_flag) cnt_adp_qc_high_to_low++; - else if (MET_ADP_QC_MEDIUM == qc_for_flag) cnt_adp_qc_medium_to_low++; - qc_for_flag = MET_ADP_QC_LOW; /* high/medium to low quality */ + if (MET_QC_LOW == qc_value) { + if (MET_QC_LOW != adp_qc_flag) { + if (MET_QC_HIGH == adp_qc_flag) cnt_adp_qc_high_to_low++; + else if (MET_QC_MEDIUM == adp_qc_flag) cnt_adp_qc_medium_to_low++; + adp_qc_flag = MET_QC_LOW; /* high/medium to low quality */ } } - else if (MET_ADP_QC_MEDIUM == qc_value && MET_ADP_QC_HIGH == qc_for_flag) { - qc_for_flag = MET_ADP_QC_MEDIUM; /* high to medium quality */ + else if (MET_QC_MEDIUM == qc_value && MET_QC_HIGH == adp_qc_flag) { + adp_qc_flag = MET_QC_MEDIUM; /* high to medium quality */ cnt_adp_qc_high_to_medium++; } - if (has_qc_flags && !qc_flags.has(qc_for_flag)) filter_out = true; + if (!qc_flags.has(adp_qc_flag)) filter_out = true; } if (filter_out) { adp_qc_filtered_count++; continue; } } - else if (has_qc_var && has_qc_flags && !qc_flags.has(qc_value)) { + else if (has_qc_var && !qc_flags.has(qc_value)) { qc_filtered_count++; continue; } @@ -2675,16 +2674,19 @@ static void regrid_goes_variable(NcFile *nc_in, VarInfo *vinfo, << ", by absent: " << absent_count << ", total: " << (qc_filtered_count + adp_qc_filtered_count + absent_count) << "\n Range: data: [" << from_min_value << " - " << from_max_value - << "] QC: [" << qc_min_value << " - " << qc_max_value << "]" - << "\n ADP QC: high=" << cnt_adjused_high << " (" << cnt_adp_qc_high - << "), medium=" << cnt_adjused_medium << " (" << cnt_adp_qc_medium - << "), low=" << cnt_adjused_low << " (" << cnt_adp_qc_low - << "), no_retrieval=" << cnt_adp_qc_nr - << "\n adjusted: high to medium=" << cnt_adp_qc_high_to_medium - << ", high to low=" << cnt_adp_qc_high_to_low - << ", medium to low=" << cnt_adp_qc_medium_to_low - << ", total=" << cnt_adjused_total - << "\n"; + << "] QC: [" << qc_min_value << " - " << qc_max_value << "]\n"; + if (has_qc_flags) { + mlog << Debug(log_debug_level) + << "\n ADP QC: high=" << cnt_adjused_high << " (" << cnt_adp_qc_high + << "), medium=" << cnt_adjused_medium << " (" << cnt_adp_qc_medium + << "), low=" << cnt_adjused_low << " (" << cnt_adp_qc_low + << "), no_retrieval=" << cnt_adp_qc_nr + << "\n adjusted: high to medium=" << cnt_adp_qc_high_to_medium + << ", high to low=" << cnt_adp_qc_high_to_low + << ", medium to low=" << cnt_adp_qc_medium_to_low + << ", total=" << cnt_adjused_total + << "\n"; + } if (to_cell_count == 0) { mlog << Warning << "\n" << method_name From dc395603b6850e73c5c5904d266df7c643a64ad0 Mon Sep 17 00:00:00 2001 From: Howard Soh Date: Fri, 10 May 2024 19:44:33 +0000 Subject: [PATCH 19/30] #2867 Use enumj class for GOES QC: HIGH, MEDIUM, and LOW --- src/tools/other/point2grid/point2grid.cc | 99 +++++++++++++++--------- 1 file changed, 64 insertions(+), 35 deletions(-) diff --git a/src/tools/other/point2grid/point2grid.cc b/src/tools/other/point2grid/point2grid.cc index db5fea47a2..18d1456924 100644 --- a/src/tools/other/point2grid/point2grid.cc +++ b/src/tools/other/point2grid/point2grid.cc @@ -127,10 +127,12 @@ static int adp_qc_high; /* 3 as baseline algorithm, 0 for enterpirse algorit static int adp_qc_medium; /* 1 as baseline algorithm, 1 for enterpirse algorithm */ static int adp_qc_low; /* 0 as baseline algorithm, 2 for enterpirse algorithm */ -constexpr int MET_QC_HIGH = 0; -constexpr int MET_QC_MEDIUM = 1; -constexpr int MET_QC_LOW = 2; -constexpr int MET_QC_NA = 3; +enum class GOES_QC { + HIGH = 0, + MEDIUM, + LOW, + NA +}; static const ConcatString att_name_values = "flag_values"; static const ConcatString att_name_meanings = "flag_meanings"; @@ -656,14 +658,13 @@ void prepare_message_types(const StringArray &hdr_types) { //////////////////////////////////////////////////////////////////////// -IntArray prepare_qc_array(const IntArray &_qc_flags, const StringArray &qc_tables) { +IntArray prepare_qc_array(const StringArray &qc_tables) { IntArray qc_idx_array; - bool has_qc_flags = (_qc_flags.n() > 0); - if (has_qc_flags) { + if (qc_flags.n() > 0) { for(int idx=0; idx prepare_qoes_qc_array() { + std::set qc_flags_set; + for(int idx=0; idx 0); - IntArray qc_idx_array = prepare_qc_array(qc_flags, qc_tables); + IntArray qc_idx_array = prepare_qc_array(qc_tables); // Initialize size and values of output fields int nx = to_grid.nx(); @@ -1839,14 +1855,13 @@ void check_lat_lon(int data_size, float *latitudes, float *longitudes) { // Baseline algorithm: 3=high, 1=medium, 0=low (high=12/48, medium=4/16) // returns bad_data_int if it does not belong to high, mediuam, or low. -int compute_adp_qc_flag(int adp_qc, int shift_bits) { +GOES_QC compute_adp_qc_flag(int adp_qc, int shift_bits) { + GOES_QC adp_qc_flag; int particle_qc = ((adp_qc >> shift_bits) & 0x03); - int adp_qc_flag; - - if (particle_qc == adp_qc_high) adp_qc_flag = MET_QC_HIGH; - else if (particle_qc == adp_qc_medium) adp_qc_flag = MET_QC_MEDIUM; - else if (particle_qc == adp_qc_low) adp_qc_flag = MET_QC_LOW; - else adp_qc_flag = MET_QC_NA; + if (particle_qc == adp_qc_high) adp_qc_flag = GOES_QC::HIGH; + else if (particle_qc == adp_qc_medium) adp_qc_flag = GOES_QC::MEDIUM; + else if (particle_qc == adp_qc_low) adp_qc_flag = GOES_QC::LOW; + else adp_qc_flag = GOES_QC::NA; return adp_qc_flag; } @@ -1872,6 +1887,18 @@ static unixtime compute_unixtime(NcVar *time_var, unixtime var_value) { //////////////////////////////////////////////////////////////////////// +GOES_QC convert_aod_qc_flag(int aod_qc) { + GOES_QC aod_qc_flag; + if (0 == aod_qc) aod_qc_flag = GOES_QC::HIGH; + else if (1 == aod_qc) aod_qc_flag = GOES_QC::MEDIUM; + else if (2 == aod_qc) aod_qc_flag = GOES_QC::LOW; + else aod_qc_flag = GOES_QC::NA; + + return aod_qc_flag; +} + +//////////////////////////////////////////////////////////////////////// + static bool get_grid_mapping(const Grid &to_grid, IntArray *cellMapping, const IntArray obs_index_array, const int *obs_hids, const float *hdr_lats, const float *hdr_lons) { @@ -2522,6 +2549,7 @@ static void regrid_goes_variable(NcFile *nc_in, VarInfo *vinfo, IntArray cellArray; NumArray dataArray; bool has_qc_flags = (qc_flags.n() > 0); + std::set aod_qc_flags = prepare_qoes_qc_array(); missing_count = non_missing_count = 0; to_dp.set_constant(bad_data_double); @@ -2580,49 +2608,50 @@ static void regrid_goes_variable(NcFile *nc_in, VarInfo *vinfo, // Filter by QC flag if (has_qc_flags && (has_qc_var || has_adp_qc_var)) { qc_value = qc_data[from_index]; + GOES_QC aod_qc_flag = convert_aod_qc_flag(qc_value); if (mlog.verbosity_level() >= log_debug_level) { if (qc_min_value > qc_value) qc_min_value = qc_value; if (qc_max_value < qc_value) qc_max_value = qc_value; - switch (qc_value) { - case MET_QC_HIGH: cnt_aod_qc_high++; break; - case MET_QC_MEDIUM: cnt_aod_qc_medium++; break; - case MET_QC_LOW: cnt_aod_qc_low++; break; - default: cnt_aod_qc_nr++; break; + switch (aod_qc_flag) { + case GOES_QC::HIGH: cnt_aod_qc_high++; break; + case GOES_QC::MEDIUM: cnt_aod_qc_medium++; break; + case GOES_QC::LOW: cnt_aod_qc_low++; break; + default: cnt_aod_qc_nr++; break; } } if (has_adp_qc_var) { - int adp_qc_flag = compute_adp_qc_flag(adp_qc_data[from_index], shift_bits); + GOES_QC adp_qc_flag = compute_adp_qc_flag(adp_qc_data[from_index], shift_bits); if (mlog.verbosity_level() >= log_debug_level) { switch (adp_qc_flag) { - case MET_QC_HIGH: cnt_adp_qc_high++; break; - case MET_QC_MEDIUM: cnt_adp_qc_medium++; break; - case MET_QC_LOW: cnt_adp_qc_low++; break; + case GOES_QC::HIGH: cnt_adp_qc_high++; break; + case GOES_QC::MEDIUM: cnt_adp_qc_medium++; break; + case GOES_QC::LOW: cnt_adp_qc_low++; break; default: cnt_adp_qc_nr++; break; } } - bool filter_out = is_eq(adp_qc_flag, MET_QC_NA); + bool filter_out = GOES_QC::NA == adp_qc_flag; if (!filter_out) { /* Adjust the quality by AOD data QC */ - if (MET_QC_LOW == qc_value) { - if (MET_QC_LOW != adp_qc_flag) { - if (MET_QC_HIGH == adp_qc_flag) cnt_adp_qc_high_to_low++; - else if (MET_QC_MEDIUM == adp_qc_flag) cnt_adp_qc_medium_to_low++; - adp_qc_flag = MET_QC_LOW; /* high/medium to low quality */ + if (GOES_QC::LOW == aod_qc_flag) { + if (GOES_QC::LOW != adp_qc_flag) { + if (GOES_QC::HIGH == adp_qc_flag) cnt_adp_qc_high_to_low++; + else if (GOES_QC::MEDIUM == adp_qc_flag) cnt_adp_qc_medium_to_low++; + adp_qc_flag = GOES_QC::LOW; /* high/medium to low quality */ } } - else if (MET_QC_MEDIUM == qc_value && MET_QC_HIGH == adp_qc_flag) { - adp_qc_flag = MET_QC_MEDIUM; /* high to medium quality */ + else if (GOES_QC::MEDIUM == aod_qc_flag && GOES_QC::HIGH == adp_qc_flag) { + adp_qc_flag = GOES_QC::MEDIUM; /* high to medium quality */ cnt_adp_qc_high_to_medium++; } - if (!qc_flags.has(adp_qc_flag)) filter_out = true; + if (0 == aod_qc_flags.count(adp_qc_flag)) filter_out = true; } if (filter_out) { adp_qc_filtered_count++; continue; } } - else if (has_qc_var && !qc_flags.has(qc_value)) { + else if (has_qc_var && 0 == aod_qc_flags.count(aod_qc_flag)) { qc_filtered_count++; continue; } From 5aaa3830ce9bc6747602e1636ad067e7f1f69286 Mon Sep 17 00:00:00 2001 From: Howard Soh Date: Wed, 15 May 2024 20:50:54 +0000 Subject: [PATCH 20/30] #2867 Added log message back which were deleted accidently --- src/tools/other/point2grid/point2grid.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/tools/other/point2grid/point2grid.cc b/src/tools/other/point2grid/point2grid.cc index 18d1456924..567a184f32 100644 --- a/src/tools/other/point2grid/point2grid.cc +++ b/src/tools/other/point2grid/point2grid.cc @@ -2706,6 +2706,9 @@ static void regrid_goes_variable(NcFile *nc_in, VarInfo *vinfo, << "] QC: [" << qc_min_value << " - " << qc_max_value << "]\n"; if (has_qc_flags) { mlog << Debug(log_debug_level) + << " AOD QC: high=" << cnt_aod_qc_high + << " medium=" << cnt_aod_qc_medium << ", low=" << cnt_aod_qc_low + << ", no_retrieval=" << cnt_aod_qc_nr << "\n ADP QC: high=" << cnt_adjused_high << " (" << cnt_adp_qc_high << "), medium=" << cnt_adjused_medium << " (" << cnt_adp_qc_medium << "), low=" << cnt_adjused_low << " (" << cnt_adp_qc_low From 2b86dbf9d6306f803f505a8e96db372bb3846cdf Mon Sep 17 00:00:00 2001 From: Howard Soh Date: Thu, 16 May 2024 22:35:53 +0000 Subject: [PATCH 21/30] #2867 Chaned statci const to constexpr --- src/libcode/vx_nc_util/nc_constants.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libcode/vx_nc_util/nc_constants.h b/src/libcode/vx_nc_util/nc_constants.h index c308d0c484..28d8909321 100644 --- a/src/libcode/vx_nc_util/nc_constants.h +++ b/src/libcode/vx_nc_util/nc_constants.h @@ -29,10 +29,10 @@ // NetCDF keywords -static const char * const CONFIG_NetCDF_Dimension = "NetCDF_Dimension"; +constexpr char CONFIG_NetCDF_Dimension[] = "NetCDF_Dimension"; // Flag value used to indicate a range of values within a dimension -static const int range_flag = bad_data_int; +constexpr int range_flag = bad_data_int; //////////////////////////////////////////////////////////////////////// From ec22c1adca068542279a92a3de9bcad9fb218f75 Mon Sep 17 00:00:00 2001 From: Howard Soh Date: Thu, 16 May 2024 22:42:31 +0000 Subject: [PATCH 22/30] #2867 Initial release. Separated from nc_utils.h --- src/libcode/vx_nc_util/nc_utils_core.h | 80 ++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 src/libcode/vx_nc_util/nc_utils_core.h diff --git a/src/libcode/vx_nc_util/nc_utils_core.h b/src/libcode/vx_nc_util/nc_utils_core.h new file mode 100644 index 0000000000..93838a2dbe --- /dev/null +++ b/src/libcode/vx_nc_util/nc_utils_core.h @@ -0,0 +1,80 @@ +// *=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=* +// ** Copyright UCAR (c) 1992 - 2024 +// ** University Corporation for Atmospheric Research (UCAR) +// ** National Center for Atmospheric Research (NCAR) +// ** Research Applications Lab (RAL) +// ** P.O.Box 3000, Boulder, Colorado, 80307-3000, USA +// *=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=* + +//////////////////////////////////////////////////////////////////////// + +#ifndef __NC_UTILS_CORE_H__ +#define __NC_UTILS_CORE_H__ + +//////////////////////////////////////////////////////////////////////// + + +//////////////////////////////////////////////////////////////////////// + +#ifndef ncbyte +typedef signed char ncbyte; /* from ncvalues.h */ +#endif /* ncbyte */ + +//////////////////////////////////////////////////////////////////////// + +#define IS_VALID_NC(ncObj) (!ncObj.isNull()) +#define IS_VALID_NC_P(ncObjPtr) ((ncObjPtr != nullptr && !ncObjPtr->isNull())) + +#define IS_INVALID_NC(ncObj) ncObj.isNull() +#define IS_INVALID_NC_P(ncObjPtr) (ncObjPtr == nullptr || ncObjPtr->isNull()) + +#define GET_NC_NAME(ncObj) ncObj.getName() +#define GET_NC_NAME_P(ncObjPtr) ncObjPtr->getName() + +#define GET_NC_SIZE(ncObj) ncObj.getSize() +#define GET_NC_SIZE_P(ncObjPtr) ncObjPtr->getSize() + +#define GET_SAFE_NC_NAME(ncObj) (ncObj.isNull() ? C_unknown_str : ncObj.getName()) +#define GET_SAFE_NC_NAME_P(ncObjPtr) (IS_INVALID_NC_P(ncObjPtr) ? C_unknown_str : ncObjPtr->getName()) + +#define GET_NC_TYPE_ID(ncObj) ncObj.getType().getId() +#define GET_NC_TYPE_ID_P(ncObjPtr) ncObjPtr->getType().getId() +#define GET_NC_TYPE_NAME(ncObj) ncObj.getType().getName() +#define GET_NC_TYPE_NAME_P(ncObjPtr) ncObjPtr->getType().getName() + +#define GET_NC_DIM_COUNT(ncObj) ncObj.getDimCount() +#define GET_NC_DIM_COUNT_P(ncObjPtr) ncObjPtr->getDimCount() + +#define GET_NC_VAR_COUNT(ncObj) ncObj.getVarCount() +#define GET_NC_VAR_COUNT_P(ncObjPtr) ncObjPtr->getVarCount() + +#define GET_NC_VARS(ncObj) ncObj.getVars() +#define GET_NC_VARS_P(ncObjPtr) ncObjPtr->getVars() + +//////////////////////////////////////////////////////////////////////// + +static const std::string C_unknown_str = std::string("unknown"); + +static const std::string fill_value_att_name = "_FillValue"; +static const std::string missing_value_att_name = "missing_value"; + +//////////////////////////////////////////////////////////////////////// + + +extern int get_data_size(netCDF::NcVar *); +extern int get_dim_count(const netCDF::NcVar *); +extern int get_dim_size(const netCDF::NcDim *); +extern int get_dim_size(const netCDF::NcVar *, const int dim_offset); + +extern netCDF::NcVarAtt *get_nc_att(const netCDF::NcVar *, const ConcatString &, bool exit_on_error = false); +extern netCDF::NcGroupAtt *get_nc_att(const netCDF::NcFile *, const ConcatString &, bool exit_on_error = false); + +extern netCDF::NcDim get_nc_dim(const netCDF::NcFile *, const std::string &dim_name); +extern netCDF::NcDim get_nc_dim(const netCDF::NcVar *, const std::string &dim_name); +extern netCDF::NcDim get_nc_dim(const netCDF::NcVar *, const int dim_offset); + +//////////////////////////////////////////////////////////////////////// + +#endif /* __NC_UTILS_CORE_H__ */ + +//////////////////////////////////////////////////////////////////////// From 579fa5285b84e5299485ca213b80a67dc8f7de05 Mon Sep 17 00:00:00 2001 From: Howard Soh Date: Thu, 16 May 2024 22:42:59 +0000 Subject: [PATCH 23/30] @2867 Added nc_utils_core.h --- src/libcode/vx_nc_util/Makefile.am | 2 +- src/libcode/vx_nc_util/Makefile.in | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libcode/vx_nc_util/Makefile.am b/src/libcode/vx_nc_util/Makefile.am index a4e8499352..108ce97639 100644 --- a/src/libcode/vx_nc_util/Makefile.am +++ b/src/libcode/vx_nc_util/Makefile.am @@ -13,7 +13,7 @@ include ${top_srcdir}/Make-include noinst_LIBRARIES = libvx_nc_util.a libvx_nc_util_a_SOURCES = \ nc_var_info.cc nc_var_info.h \ - nc_utils.cc nc_utils.h nc_utils.hpp \ + nc_utils.cc nc_utils.h nc_utils_core.h nc_utils.hpp \ write_netcdf.cc write_netcdf.h \ grid_output.cc grid_output.h \ load_tc_data.cc load_tc_data.h \ diff --git a/src/libcode/vx_nc_util/Makefile.in b/src/libcode/vx_nc_util/Makefile.in index 9843b5647b..f2f564ff93 100644 --- a/src/libcode/vx_nc_util/Makefile.in +++ b/src/libcode/vx_nc_util/Makefile.in @@ -353,7 +353,7 @@ MAINTAINERCLEANFILES = Makefile.in noinst_LIBRARIES = libvx_nc_util.a libvx_nc_util_a_SOURCES = \ nc_var_info.cc nc_var_info.h \ - nc_utils.cc nc_utils.h nc_utils.hpp \ + nc_utils.cc nc_utils.h nc_utils_core.h nc_utils.hpp \ write_netcdf.cc write_netcdf.h \ grid_output.cc grid_output.h \ load_tc_data.cc load_tc_data.h \ From 4f0ca12e03167b8221e151fc1dad1371a2c0fbe8 Mon Sep 17 00:00:00 2001 From: Howard Soh Date: Thu, 16 May 2024 22:44:01 +0000 Subject: [PATCH 24/30] #2867 Moved some blocks to nc_utils_core.h --- src/libcode/vx_nc_util/nc_utils.h | 54 ++----------------------------- 1 file changed, 2 insertions(+), 52 deletions(-) diff --git a/src/libcode/vx_nc_util/nc_utils.h b/src/libcode/vx_nc_util/nc_utils.h index 212771152a..c879ba3c1d 100644 --- a/src/libcode/vx_nc_util/nc_utils.h +++ b/src/libcode/vx_nc_util/nc_utils.h @@ -24,50 +24,16 @@ #include "num_array.h" #include "nc_var_info.h" +#include "nc_utils.hpp" + //////////////////////////////////////////////////////////////////////// -#ifndef ncbyte -typedef signed char ncbyte; /* from ncvalues.h */ -#endif /* ncbyte */ #ifndef uchar typedef unsigned char uchar; #endif /* uchar */ //////////////////////////////////////////////////////////////////////// -static const std::string C_unknown_str = std::string("unknown"); - -#define IS_VALID_NC(ncObj) (!ncObj.isNull()) -#define IS_VALID_NC_P(ncObjPtr) ((ncObjPtr != nullptr && !ncObjPtr->isNull())) - -#define IS_INVALID_NC(ncObj) ncObj.isNull() -#define IS_INVALID_NC_P(ncObjPtr) (ncObjPtr == nullptr || ncObjPtr->isNull()) - -#define GET_NC_NAME(ncObj) ncObj.getName() -#define GET_NC_NAME_P(ncObjPtr) ncObjPtr->getName() - -#define GET_NC_SIZE(ncObj) ncObj.getSize() -#define GET_NC_SIZE_P(ncObjPtr) ncObjPtr->getSize() - -#define GET_SAFE_NC_NAME(ncObj) (ncObj.isNull() ? C_unknown_str : ncObj.getName()) -#define GET_SAFE_NC_NAME_P(ncObjPtr) (IS_INVALID_NC_P(ncObjPtr) ? C_unknown_str : ncObjPtr->getName()) - -#define GET_NC_TYPE_ID(ncObj) ncObj.getType().getId() -#define GET_NC_TYPE_ID_P(ncObjPtr) ncObjPtr->getType().getId() -#define GET_NC_TYPE_NAME(ncObj) ncObj.getType().getName() -#define GET_NC_TYPE_NAME_P(ncObjPtr) ncObjPtr->getType().getName() - -#define GET_NC_DIM_COUNT(ncObj) ncObj.getDimCount() -#define GET_NC_DIM_COUNT_P(ncObjPtr) ncObjPtr->getDimCount() - -#define GET_NC_VAR_COUNT(ncObj) ncObj.getVarCount() -#define GET_NC_VAR_COUNT_P(ncObjPtr) ncObjPtr->getVarCount() - -#define GET_NC_VARS(ncObj) ncObj.getVars() -#define GET_NC_VARS_P(ncObjPtr) ncObjPtr->getVars() - -//////////////////////////////////////////////////////////////////////// - #define DEF_DEFLATE_LEVEL (0) #define GET_NC_ATT_OBJ(nc_or_var, att_name) nc_or_var.getAtt(att_name) @@ -137,11 +103,9 @@ static const std::string coordinates_att_name = "coordinates"; static const std::string coordinate_axis_type_att_name = "_CoordinateAxisType"; static const std::string cf_att_name = "Conventions"; static const std::string description_att_name = "description"; -static const std::string fill_value_att_name = "_FillValue"; static const std::string grid_mapping_att_name = "grid_mapping"; static const std::string grid_mapping_name_att_name = "grid_mapping_name"; static const std::string long_name_att_name = "long_name"; -static const std::string missing_value_att_name = "missing_value"; static const std::string projection_att_name = "Projection"; static const std::string scale_factor_att_name = "scale_factor"; static const std::string standard_name_att_name = "standard_name"; @@ -185,9 +149,6 @@ extern bool get_att_no_leap_year(const netCDF::NcVar *); extern bool get_cf_conventions(const netCDF::NcFile *, ConcatString&); -extern netCDF::NcVarAtt *get_nc_att(const netCDF::NcVar *, const ConcatString &, bool exit_on_error = false); -extern netCDF::NcGroupAtt *get_nc_att(const netCDF::NcFile *, const ConcatString &, bool exit_on_error = false); - extern bool get_nc_att_value(const netCDF::NcVarAtt *, std::string &); extern bool get_nc_att_value(const netCDF::NcVarAtt *, int &, bool exit_on_error = true); extern bool get_nc_att_value(const netCDF::NcVarAtt *, float &, bool exit_on_error = true); @@ -361,14 +322,8 @@ extern netCDF::NcDim add_dim(netCDF::NcFile *, const std::string &); extern netCDF::NcDim add_dim(netCDF::NcFile *, const std::string &, const size_t); extern bool has_dim(netCDF::NcFile *, const char *dim_name); extern bool get_dim(const netCDF::NcFile *, const ConcatString &, int &, bool error_out = false); -extern int get_dim_count(const netCDF::NcVar *); extern int get_dim_count(const netCDF::NcFile *); -extern int get_dim_size(const netCDF::NcDim *); -extern int get_dim_size(const netCDF::NcVar *, const int dim_offset); extern int get_dim_value(const netCDF::NcFile *, const std::string &, const bool error_out = false); -extern netCDF::NcDim get_nc_dim(const netCDF::NcFile *, const std::string &dim_name); -extern netCDF::NcDim get_nc_dim(const netCDF::NcVar *, const std::string &dim_name); -extern netCDF::NcDim get_nc_dim(const netCDF::NcVar *, const int dim_offset); extern bool get_dim_names(const netCDF::NcVar *var, StringArray *dimNames); extern bool get_dim_names(const netCDF::NcFile *nc, StringArray *dimNames); @@ -378,7 +333,6 @@ extern netCDF::NcVar get_nc_var_time(const netCDF::NcFile *nc); extern int get_index_at_nc_data(netCDF::NcVar *var, double value, const std::string dim_name, bool is_time=false); extern netCDF::NcFile* open_ncfile(const char * nc_name, bool write = false); -extern int get_data_size(netCDF::NcVar *); extern unixtime get_reference_unixtime(netCDF::NcVar *time_var, int &sec_per_unit, bool &no_leap_year); @@ -392,10 +346,6 @@ extern void parse_time_string(const char *str, unixtime &ut); //////////////////////////////////////////////////////////////////////// -#include "nc_utils.hpp" - -//////////////////////////////////////////////////////////////////////// - #endif /* __NC_UTILS_H__ */ //////////////////////////////////////////////////////////////////////// From 9271d636abc61924bb4b6c2ae3058ca3387f5374 Mon Sep 17 00:00:00 2001 From: Howard Soh Date: Thu, 16 May 2024 22:44:23 +0000 Subject: [PATCH 25/30] #2867 Include nc_utils_core.h --- src/libcode/vx_nc_util/nc_utils.hpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/libcode/vx_nc_util/nc_utils.hpp b/src/libcode/vx_nc_util/nc_utils.hpp index 697a34ceec..f86cb472e9 100644 --- a/src/libcode/vx_nc_util/nc_utils.hpp +++ b/src/libcode/vx_nc_util/nc_utils.hpp @@ -13,6 +13,10 @@ //////////////////////////////////////////////////////////////////////// +#include "nc_utils_core.h" + +//////////////////////////////////////////////////////////////////////// + extern bool get_att_value(const netCDF::NcAtt *att, int &att_val); extern bool get_att_value(const netCDF::NcAtt *att, ConcatString &value); extern bool get_att_value(const netCDF::NcAtt *att, ncbyte &att_val); From 547b26bf6804f41f517bf2eec6849e4db54c30ed Mon Sep 17 00:00:00 2001 From: Howard Soh Date: Thu, 16 May 2024 22:53:35 +0000 Subject: [PATCH 26/30] #2867 Added const references --- src/tools/other/point2grid/point2grid.cc | 46 ++++++++++++------------ 1 file changed, 22 insertions(+), 24 deletions(-) diff --git a/src/tools/other/point2grid/point2grid.cc b/src/tools/other/point2grid/point2grid.cc index 567a184f32..839ede8894 100644 --- a/src/tools/other/point2grid/point2grid.cc +++ b/src/tools/other/point2grid/point2grid.cc @@ -144,7 +144,7 @@ static void process_data_file(); static void process_point_file(NcFile *nc_in, MetConfig &config, VarInfo *, const Grid to_grid); #ifdef WITH_PYTHON -static void process_point_python(string python_command, MetConfig &config, +static void process_point_python(const string python_command, MetConfig &config, VarInfo *vinfo, const Grid to_grid, bool use_xarray); #endif static void process_point_nccf_file(NcFile *nc_in, MetConfig &config, @@ -181,7 +181,7 @@ static void regrid_nc_variable(NcFile *nc_in, Met2dDataFile *fr_mtddf, static bool keep_message_type(const int mt_index); -static bool has_lat_lon_vars(NcFile *nc_in); +static bool has_lat_lon_vars(const NcFile *nc_in); static void set_adp_gc_values(NcVar var_adp_qc); //////////////////////////////////////////////////////////////////////// @@ -201,16 +201,16 @@ static IntArray qc_flags; static void process_goes_file(NcFile *nc_in, MetConfig &config, VarInfo *, const Grid fr_grid, const Grid to_grid); static unixtime find_valid_time(NcVar time_var); -static ConcatString get_goes_grid_input(MetConfig config, Grid fr_grid, Grid to_grid); +static ConcatString get_goes_grid_input(MetConfig config, const Grid fr_grid); static void get_grid_mapping(const Grid &fr_grid, const Grid &to_grid, IntArray *cellMapping, const ConcatString &geostationary_file); -static int get_lat_count(NcFile *); -static int get_lon_count(NcFile *); +static int get_lat_count(const NcFile *); +static int get_lon_count(const NcFile *); static NcVar get_goes_nc_var(NcFile *nc, const ConcatString &var_name, bool exit_if_error=true); static bool is_time_mismatch(NcFile *nc_in, NcFile *nc_adp); -static ConcatString make_geostationary_filename(Grid fr_grid, Grid to_grid); -static void regrid_goes_variable(NcFile *nc_in, VarInfo *vinfo, +static ConcatString make_geostationary_filename(Grid fr_grid); +static void regrid_goes_variable(NcFile *nc_in, const VarInfo *vinfo, DataPlane &fr_dp, DataPlane &to_dp, Grid fr_grid, Grid to_grid, IntArray *cellMapping, NcFile *nc_adp); static void save_geostationary_data(const ConcatString geostationary_file, @@ -468,7 +468,7 @@ static void process_data_file() { if (goes_data) { mlog << Debug(2) << "Input grid: " << fr_grid.serialize() << "\n"; - ConcatString grid_string = get_goes_grid_input(config, fr_grid, to_grid); + ConcatString grid_string = get_goes_grid_input(config, fr_grid); if (!grid_string.empty()) run_cs << " with " << grid_string; } mlog << Debug(2) << "Output grid: " << to_grid.serialize() << "\n"; @@ -1183,8 +1183,8 @@ static void process_point_file(NcFile *nc_in, MetConfig &config, VarInfo *vinfo, #ifdef WITH_PYTHON -static void process_point_python(string python_command, MetConfig &config, VarInfo *vinfo, - const Grid to_grid, bool use_xarray) { +static void process_point_python(const string python_command, MetConfig &config, + VarInfo *vinfo, const Grid to_grid, bool use_xarray) { clock_t start_clock = clock(); static const char *method_name = "process_point_python() -> "; @@ -1681,7 +1681,7 @@ static void process_goes_file(NcFile *nc_in, MetConfig &config, VarInfo *vinfo, ConcatString tmp_dir = config.get_tmp_dir(); ConcatString geostationary_file(tmp_dir); geostationary_file.add("/"); - geostationary_file.add(make_geostationary_filename(fr_grid, to_grid)); + geostationary_file.add(make_geostationary_filename(fr_grid)); // Open ADP file if exists if (!adp_filename.empty() && file_exists(adp_filename.c_str())) { @@ -1792,11 +1792,9 @@ static void process_goes_file(NcFile *nc_in, MetConfig &config, VarInfo *vinfo, } // end for i multimap mapVar = GET_NC_VARS_P(nc_in); - for (auto itVar = mapVar.begin(); - itVar != mapVar.end(); ++itVar) { - if ((*itVar).first == "t" - || string::npos != (*itVar).first.find("time")) { - NcVar from_var = (*itVar).second; + for (const auto &kv : mapVar) { + if (kv.first == "t" || string::npos != kv.first.find("time")) { + NcVar from_var = kv.second; copy_nc_var(nc_out, &from_var); } } @@ -2143,13 +2141,13 @@ static unixtime find_valid_time(NcVar time_var) { //////////////////////////////////////////////////////////////////////// -static ConcatString get_goes_grid_input(MetConfig config, Grid fr_grid, Grid to_grid) { +static ConcatString get_goes_grid_input(MetConfig config, const Grid fr_grid) { ConcatString run_string; ConcatString env_coord_name; ConcatString tmp_dir = config.get_tmp_dir(); ConcatString geostationary_file(tmp_dir); geostationary_file.add("/"); - geostationary_file.add(make_geostationary_filename(fr_grid, to_grid)); + geostationary_file.add(make_geostationary_filename(fr_grid)); if (get_env(key_geostationary_data, env_coord_name) && env_coord_name.nonempty() && file_exists(env_coord_name.c_str())) { @@ -2331,7 +2329,7 @@ static void get_grid_mapping(const Grid &fr_grid, const Grid &to_grid, IntArray //////////////////////////////////////////////////////////////////////// -static int get_lat_count(NcFile *_nc) { +static int get_lat_count(const NcFile *_nc) { int lat_count = 0; NcDim dim_lat = get_nc_dim(_nc, dim_name_lat); if(IS_INVALID_NC(dim_lat)) dim_lat = get_nc_dim(_nc, "y"); @@ -2341,7 +2339,7 @@ static int get_lat_count(NcFile *_nc) { //////////////////////////////////////////////////////////////////////// -static int get_lon_count(NcFile *_nc) { +static int get_lon_count(const NcFile *_nc) { int lon_count = 0; NcDim dim_lon = get_nc_dim(_nc, dim_name_lon); if(IS_INVALID_NC(dim_lon)) dim_lon = get_nc_dim(_nc, "x"); @@ -2373,7 +2371,7 @@ static NcVar get_goes_nc_var(NcFile *nc, const ConcatString &var_name, //////////////////////////////////////////////////////////////////////// -static ConcatString make_geostationary_filename(Grid fr_grid, Grid to_grid) { +static ConcatString make_geostationary_filename(Grid fr_grid) { ConcatString geo_data_filename; GridInfo info = fr_grid.info(); @@ -2433,7 +2431,7 @@ static bool is_time_mismatch(NcFile *nc_in, NcFile *nc_adp) { //////////////////////////////////////////////////////////////////////// -static void regrid_goes_variable(NcFile *nc_in, VarInfo *vinfo, +static void regrid_goes_variable(NcFile *nc_in, const VarInfo *vinfo, DataPlane &fr_dp, DataPlane &to_dp, Grid fr_grid, Grid to_grid, IntArray *cellMapping, NcFile *nc_adp) { @@ -2828,7 +2826,7 @@ static bool keep_message_type(const int mt_index) { //////////////////////////////////////////////////////////////////////// -static bool has_lat_lon_vars(NcFile *nc) { +static bool has_lat_lon_vars(const NcFile *nc) { bool has_lat_var = IS_VALID_NC(get_nc_var_lat(nc)); bool has_lon_var = IS_VALID_NC(get_nc_var_lon(nc)); @@ -2844,7 +2842,7 @@ static bool has_lat_lon_vars(NcFile *nc) { //////////////////////////////////////////////////////////////////////// -static void usage() { +__attribute__((noreturn)) static void usage() { cout << "\n*** Model Evaluation Tools (MET" << met_version << ") ***\n\n" From a1b08caed4515417a244de358496de72ccb62a00 Mon Sep 17 00:00:00 2001 From: Howard Soh Date: Mon, 10 Jun 2024 18:55:12 +0000 Subject: [PATCH 27/30] #2867 Some 'static const' were chnaged to constexpr --- src/libcode/vx_nc_util/nc_utils.h | 104 +++++++++++++++--------------- 1 file changed, 52 insertions(+), 52 deletions(-) diff --git a/src/libcode/vx_nc_util/nc_utils.h b/src/libcode/vx_nc_util/nc_utils.h index c879ba3c1d..671b3bf609 100644 --- a/src/libcode/vx_nc_util/nc_utils.h +++ b/src/libcode/vx_nc_util/nc_utils.h @@ -52,49 +52,49 @@ typedef unsigned char uchar; #define OBS_BUFFER_SIZE (128 * 1024) -static const char nc_dim_nhdr[] = "nhdr"; -static const char nc_dim_nhdr_typ[] = "nhdr_typ"; -static const char nc_dim_nhdr_sid[] = "nhdr_sid"; -static const char nc_dim_nhdr_vld[] = "nhdr_vld"; -static const char nc_dim_npbhdr[] = "npbhdr"; -static const char nc_dim_nobs[] = "nobs"; -static const char nc_dim_nqty[] = "nobs_qty"; -static const char nc_dim_hdr_arr[] = "hdr_arr_len"; -static const char nc_dim_obs_arr[] = "obs_arr_len"; -static const char nc_dim_mxstr[] = "mxstr"; -static const char nc_dim_mxstr2[] = "mxstr2"; -static const char nc_dim_mxstr3[] = "mxstr3"; -static const char nc_dim_nvar[] = "obs_var_num"; -static const char nc_dim_unit[] = "unit_len"; -static const char nc_dim_desc[] = "desc_len"; -static const char nc_var_desc[] = "obs_desc"; -static const char nc_var_hdr_arr[] = "hdr_arr"; -static const char nc_var_hdr_lat[] = "hdr_lat"; -static const char nc_var_hdr_lon[] = "hdr_lon"; -static const char nc_var_hdr_elv[] = "hdr_elv"; -static const char nc_var_hdr_typ[] = "hdr_typ"; -static const char nc_var_hdr_sid[] = "hdr_sid"; -static const char nc_var_hdr_vld[] = "hdr_vld"; -static const char nc_var_hdr_prpt_typ[] = "hdr_prpt_typ"; -static const char nc_var_hdr_irpt_typ[] = "hdr_irpt_typ"; -static const char nc_var_hdr_inst_typ[] = "hdr_inst_typ"; -static const char nc_var_hdr_typ_tbl[] = "hdr_typ_table"; -static const char nc_var_hdr_sid_tbl[] = "hdr_sid_table"; -static const char nc_var_hdr_vld_tbl[] = "hdr_vld_table"; -static const char nc_var_obs_arr[] = "obs_arr"; -static const char nc_var_obs_hid[] = "obs_hid"; -static const char nc_var_obs_gc[] = "obs_gc"; -static const char nc_var_obs_vid[] = "obs_vid"; -static const char nc_var_obs_lvl[] = "obs_lvl"; -static const char nc_var_obs_hgt[] = "obs_hgt"; -static const char nc_var_obs_val[] = "obs_val"; -static const char nc_var_obs_qty[] = "obs_qty"; -static const char nc_var_obs_qty_tbl[] = "obs_qty_table"; -static const char nc_var_obs_var[] = "obs_var"; -static const char nc_var_unit[] = "obs_unit"; -static const std::string nc_att_use_var_id = "use_var_id"; -static const char nc_att_obs_version[] = "MET_Obs_version"; -static const char nc_att_met_point_nccf[] = "MET_point_NCCF"; +constexpr char nc_dim_nhdr[] = "nhdr"; +constexpr char nc_dim_nhdr_typ[] = "nhdr_typ"; +constexpr char nc_dim_nhdr_sid[] = "nhdr_sid"; +constexpr char nc_dim_nhdr_vld[] = "nhdr_vld"; +constexpr char nc_dim_npbhdr[] = "npbhdr"; +constexpr char nc_dim_nobs[] = "nobs"; +constexpr char nc_dim_nqty[] = "nobs_qty"; +constexpr char nc_dim_hdr_arr[] = "hdr_arr_len"; +constexpr char nc_dim_obs_arr[] = "obs_arr_len"; +constexpr char nc_dim_mxstr[] = "mxstr"; +constexpr char nc_dim_mxstr2[] = "mxstr2"; +constexpr char nc_dim_mxstr3[] = "mxstr3"; +constexpr char nc_dim_nvar[] = "obs_var_num"; +constexpr char nc_dim_unit[] = "unit_len"; +constexpr char nc_dim_desc[] = "desc_len"; +constexpr char nc_var_desc[] = "obs_desc"; +constexpr char nc_var_hdr_arr[] = "hdr_arr"; +constexpr char nc_var_hdr_lat[] = "hdr_lat"; +constexpr char nc_var_hdr_lon[] = "hdr_lon"; +constexpr char nc_var_hdr_elv[] = "hdr_elv"; +constexpr char nc_var_hdr_typ[] = "hdr_typ"; +constexpr char nc_var_hdr_sid[] = "hdr_sid"; +constexpr char nc_var_hdr_vld[] = "hdr_vld"; +constexpr char nc_var_hdr_prpt_typ[] = "hdr_prpt_typ"; +constexpr char nc_var_hdr_irpt_typ[] = "hdr_irpt_typ"; +constexpr char nc_var_hdr_inst_typ[] = "hdr_inst_typ"; +constexpr char nc_var_hdr_typ_tbl[] = "hdr_typ_table"; +constexpr char nc_var_hdr_sid_tbl[] = "hdr_sid_table"; +constexpr char nc_var_hdr_vld_tbl[] = "hdr_vld_table"; +constexpr char nc_var_obs_arr[] = "obs_arr"; +constexpr char nc_var_obs_hid[] = "obs_hid"; +constexpr char nc_var_obs_gc[] = "obs_gc"; +constexpr char nc_var_obs_vid[] = "obs_vid"; +constexpr char nc_var_obs_lvl[] = "obs_lvl"; +constexpr char nc_var_obs_hgt[] = "obs_hgt"; +constexpr char nc_var_obs_val[] = "obs_val"; +constexpr char nc_var_obs_qty[] = "obs_qty"; +constexpr char nc_var_obs_qty_tbl[] = "obs_qty_table"; +constexpr char nc_var_obs_var[] = "obs_var"; +constexpr char nc_var_unit[] = "obs_unit"; +constexpr char nc_att_use_var_id[] = "use_var_id"; +constexpr char nc_att_obs_version[] = "MET_Obs_version"; +constexpr char nc_att_met_point_nccf[] = "MET_point_NCCF"; static const std::string add_offset_att_name = "add_offset"; static const std::string axis_att_name = "axis"; @@ -111,17 +111,17 @@ static const std::string scale_factor_att_name = "scale_factor"; static const std::string standard_name_att_name = "standard_name"; static const std::string units_att_name = "units"; -static const char nc_time_unit_exp[] = "^[a-z|A-Z]* *since *[0-9]\\{1,4\\}-[0-9]\\{1,2\\}-[0-9]\\{1,2\\}"; -static const char nc_time_unit_ymd_exp[] = "[0-9]\\{1,4\\}-[0-9]\\{1,2\\}-[0-9]\\{1,2\\}"; +constexpr char nc_time_unit_exp[] = "^[a-z|A-Z]* *since *[0-9]\\{1,4\\}-[0-9]\\{1,2\\}-[0-9]\\{1,2\\}"; +constexpr char nc_time_unit_ymd_exp[] = "[0-9]\\{1,4\\}-[0-9]\\{1,2\\}-[0-9]\\{1,2\\}"; -static const char MET_NC_Obs_ver_1_2[] = "1.02"; -static const char MET_NC_Obs_version[] = "1.02"; +constexpr char MET_NC_Obs_ver_1_2[] = "1.02"; +constexpr char MET_NC_Obs_version[] = "1.02"; -static const int exit_code_no_error = 0; -static const int exit_code_no_dim = 1; -static const int exit_code_no_hdr_vars = 2; -static const int exit_code_no_loc_vars = 3; -static const int exit_code_no_obs_vars = 4; +constexpr int exit_code_no_error = 0; +constexpr int exit_code_no_dim = 1; +constexpr int exit_code_no_hdr_vars = 2; +constexpr int exit_code_no_loc_vars = 3; +constexpr int exit_code_no_obs_vars = 4; //////////////////////////////////////////////////////////////////////// From 47fa422eb31a0058b9c7437a1961482e78d489c6 Mon Sep 17 00:00:00 2001 From: Howard Soh Date: Mon, 10 Jun 2024 18:56:33 +0000 Subject: [PATCH 28/30] #2867 Changed -qc options (1,2,3 to 0,1 - high & medium) for AOD --- internal/test_unit/xml/unit_point2grid.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/test_unit/xml/unit_point2grid.xml b/internal/test_unit/xml/unit_point2grid.xml index 5221ea4460..e2fe85bff3 100644 --- a/internal/test_unit/xml/unit_point2grid.xml +++ b/internal/test_unit/xml/unit_point2grid.xml @@ -130,7 +130,7 @@ G212 \ &OUTPUT_DIR;/point2grid/point2grid_GOES_16_AOD_TO_G212_compute.nc \ -field 'name="AOD"; level="(*,*)";' \ - -qc 1,2,3 -method MAX \ + -qc 0,1 -method MAX \ -v 1 @@ -167,7 +167,7 @@ &OUTPUT_DIR;/point2grid/point2grid_GOES_16_ADP.nc \ -field 'name="AOD_Smoke"; level="(*,*)";' \ -adp &DATA_DIR_MODEL;/goes_16/OR_ABI-L2-ADPC-M6_G16_s20192662141196_e20192662143569_c20192662144526.nc \ - -qc 1,2 -method MAX \ + -qc 0,1 -method MAX \ -v 1 @@ -259,7 +259,7 @@ G212 \ &OUTPUT_DIR;/point2grid/point2grid_GOES_16_AOD_TO_G212_unsigned.nc \ -field 'name="AOD"; level="(*,*)";' \ - -qc 1,2,3 -method MAX \ + -qc 0,1,2 -method MAX \ -v 1 From 17c72d5c81498d553cc9167569056fccf2dbfd06 Mon Sep 17 00:00:00 2001 From: Howard Soh Date: Tue, 11 Jun 2024 17:27:37 +0000 Subject: [PATCH 29/30] #2867 Merged develop branch --- src/tools/other/point2grid/point2grid.cc | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/tools/other/point2grid/point2grid.cc b/src/tools/other/point2grid/point2grid.cc index e324006cde..2b067ffdf4 100644 --- a/src/tools/other/point2grid/point2grid.cc +++ b/src/tools/other/point2grid/point2grid.cc @@ -124,9 +124,9 @@ static NcFile *nc_out = (NcFile *) nullptr; static NcDim lat_dim ; static NcDim lon_dim ; -static int adp_qc_high; /* 3 as baseline algorithm, 0 for enterpirse algorithm */ -static int adp_qc_medium; /* 1 as baseline algorithm, 1 for enterpirse algorithm */ -static int adp_qc_low; /* 0 as baseline algorithm, 2 for enterpirse algorithm */ +static int adp_qc_high; /* 3 as baseline algorithm, 0 for enterprise algorithm */ +static int adp_qc_medium; /* 1 as baseline algorithm, 1 for enterprise algorithm */ +static int adp_qc_low; /* 0 as baseline algorithm, 2 for enterprise algorithm */ enum class GOES_QC { HIGH = 0, @@ -2458,9 +2458,9 @@ static void regrid_goes_variable(NcFile *nc_in, const VarInfo *vinfo, // -99 is arbitrary number as invalid QC value memset(qc_data, -99, from_data_size*sizeof(uchar)); - adp_qc_high = 3; /* 3 as baseline algorithm, 0 for enterpirse algorithm */ - adp_qc_medium = 1; /* 1 as baseline algorithm, 1 for enterpirse algorithm */ - adp_qc_low = 0; /* 0 as baseline algorithm, 2 for enterpirse algorithm */ + adp_qc_high = 3; /* 3 as baseline algorithm, 0 for enterprise algorithm */ + adp_qc_medium = 1; /* 1 as baseline algorithm, 1 for enterprise algorithm */ + adp_qc_low = 0; /* 0 as baseline algorithm, 2 for enterprise algorithm */ NcVar var_qc; NcVar var_adp; From 6e592d4fd89373739da126f1b2c270151fd2f981 Mon Sep 17 00:00:00 2001 From: Howard Soh Date: Tue, 11 Jun 2024 17:31:27 +0000 Subject: [PATCH 30/30] #2867 Corrected the unit test name --- internal/test_unit/xml/unit_point2grid.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/test_unit/xml/unit_point2grid.xml b/internal/test_unit/xml/unit_point2grid.xml index e2fe85bff3..213ffcf756 100644 --- a/internal/test_unit/xml/unit_point2grid.xml +++ b/internal/test_unit/xml/unit_point2grid.xml @@ -213,7 +213,7 @@ - + &MET_BIN;/point2grid MET_TMP_DIR &OUTPUT_DIR;/point2grid