From 91c111a87ec28ae505796b4ae1ec989caebdfdca Mon Sep 17 00:00:00 2001 From: Howard Soh Date: Sat, 8 Oct 2022 01:50:58 -0600 Subject: [PATCH] #2068 Support IODA v2 --- src/tools/other/ioda2nc/ioda2nc.cc | 289 +++++++++++++++++++---------- 1 file changed, 196 insertions(+), 93 deletions(-) diff --git a/src/tools/other/ioda2nc/ioda2nc.cc b/src/tools/other/ioda2nc/ioda2nc.cc index 1aaef67ba1..ab39192ca3 100644 --- a/src/tools/other/ioda2nc/ioda2nc.cc +++ b/src/tools/other/ioda2nc/ioda2nc.cc @@ -64,6 +64,14 @@ static const char *program_name = "ioda2nc"; static const int REJECT_DEBUG_LEVEL = 9; +static const char *metadata_group_name = "MetaData"; +static const char *qc_group_name = "QCFlags"; +static const char *qc_postfix = "PreQC"; +static const char *obs_group_name = "ObsValue"; +static const char *derived_obs_group_name = "DerivedObsValue"; + +enum e_ioda_format { ioda_v1, ioda_v2 }; + //////////////////////////////////////////////////////////////////////// // @@ -74,7 +82,8 @@ static const int REJECT_DEBUG_LEVEL = 9; static StringArray ioda_files; static StringArray core_dims; -static StringArray core_vars; +static StringArray core_dims_v1; +static StringArray core_meta_vars; // Output NetCDF file name static ConcatString ncfile; @@ -128,6 +137,7 @@ static void initialize(); static void process_command_line(int, char **); static void open_netcdf(); static void process_ioda_file(int); +static void process_ioda_file(NcFile *nc); static void write_netcdf_hdr_data(); static void clean_up(); @@ -154,13 +164,14 @@ static void set_valid_beg_time(const StringArray &); static void set_valid_end_time(const StringArray &); static void set_verbosity(const StringArray &); -static bool check_core_data(const bool, const bool, StringArray &, StringArray &); +static bool check_core_data(const bool, const bool, StringArray &, StringArray &, e_ioda_format); static bool check_missing_thresh(float value); static ConcatString find_meta_name(StringArray, StringArray); -static bool get_meta_data_float(NcFile *, StringArray &, const char *, float *, const int); -static bool get_meta_data_strings(NcFile *, const ConcatString, char *); -static bool get_obs_data_float(NcFile *, const ConcatString, - NcVar *, float *, int *, const int); +static bool get_meta_data_float(NcFile *, StringArray &, const char *, float *, + const int, e_ioda_format); +static bool get_meta_data_strings(NcFile *, const ConcatString, char *, e_ioda_format); +static bool get_obs_data_float(NcFile *, const ConcatString, NcVar *, + float *, int *, const int, const e_ioda_format); static bool has_postfix(std::string const &, std::string const &); //////////////////////////////////////////////////////////////////////// @@ -214,15 +225,18 @@ void initialize() { nc_point_obs.init_buffer(); core_dims.clear(); - core_dims.add("nvars"); core_dims.add("nlocs"); - //core_dims.add("nstring"); - //core_dims.add("ndatetime"); + + core_dims_v1.clear(); + core_dims_v1.add("nvars"); + core_dims_v1.add("nlocs"); + core_dims_v1.add("nstring"); + core_dims_v1.add("ndatetime"); - core_vars.clear(); - core_vars.add("datetime"); - core_vars.add("latitude"); - core_vars.add("longitude"); + core_meta_vars.clear(); + core_meta_vars.add("datetime"); + core_meta_vars.add("latitude"); + core_meta_vars.add("longitude"); summary_obs = new SummaryObs(); return; @@ -349,6 +363,8 @@ void process_ioda_file(int i_pb) { int rej_elv, rej_nobs; double x, y; + bool status; + bool number_time = false; unixtime file_ut; unixtime adjusted_file_ut; unixtime msg_ut, beg_ut, end_ut; @@ -411,28 +427,62 @@ void process_ioda_file(int i_pb) { file_name << ioda_files[i_pb]; int nrecs = 0; - StringArray var_names, dim_names; + StringArray dim_names; StringArray metadata_vars; StringArray obs_value_vars; - get_var_names(f_in, &var_names); + e_ioda_format ioda_format = ioda_v2; + int nstring, nvars; + bool error_out = true; + int nlocs = get_dim_value(f_in, "nlocs", error_out); // number of locations + + nvars = nstring = bad_data_int ; + if (! has_nc_group(f_in, obs_group_name)) ioda_format = ioda_v1; + get_dim_names(f_in, &dim_names); - for(idx=0; idx= 8) { + for(idx=0; idx= 6) { - for(idx=0; idxgetId()) && (0 < units.length())) { + parse_cf_time_string(units.c_str(), base_ut, sec_per_unit); + } + else { + mlog << Error << "\n" << method_name << "Fail to get time units from " + << GET_NC_NAME(in_hdr_vld_var) << "\n\n"; + sec_per_unit = 1; + base_ut = 300000; + } } - NcVar in_hdr_vld_var = get_var(f_in, "datetime@MetaData"); - NcVar in_hdr_lat_var = get_var(f_in, "latitude@MetaData"); - NcVar in_hdr_lon_var = get_var(f_in, "longitude@MetaData"); - - int ndatetime; - if(dim_names.has("ndatetime")) ndatetime = get_dim_value(f_in, "ndatetime", error_out); else { - NcDim datetime_dim = get_nc_dim(&in_hdr_vld_var, 1); - ndatetime = IS_VALID_NC(datetime_dim) ? get_dim_size(&datetime_dim) : nstring; - mlog << Debug(3) << "\n" << method_name - << "ndatetime dimension does not exist!\n"; + in_hdr_vld_var = get_nc_var(f_in, "datetime", metadata_group_name); + if(dim_names.has("ndatetime")) ndatetime = get_dim_value(f_in, "ndatetime", error_out); + else { + NcDim datetime_dim = get_nc_dim(&in_hdr_vld_var, 1); + ndatetime = IS_VALID_NC(datetime_dim) ? get_dim_size(&datetime_dim) : nstring; + mlog << Debug(3) << "\n" << method_name + << "ndatetime dimension does not exist!\n"; + } + mlog << Debug(5) << method_name << "dimensions: nvars=" << nvars << ", nlocs=" << nlocs + << ", nrecs=" << nrecs << ", nstring=" << nstring << ", ndatetime=" << ndatetime << "\n"; } - mlog << Debug(5) << method_name << "dimensions: nvars=" << nvars << ", nlocs=" << nlocs - << ", nrecs=" << nrecs << ", nstring=" << nstring << ", ndatetime=" << ndatetime << "\n"; npbmsg_total = npbmsg = nlocs; @@ -489,6 +549,7 @@ void process_ioda_file(int i_pb) { ? (npbmsg * nmsg_percent / 100) : nmsg; } +cout << " DEBUG " << method_name << "nlocs=" << nlocs << ", ndatetime=" << ndatetime<< "\n"; long lengths[2] = { nlocs, ndatetime }; long offsets[2] = { 0, 0 }; float *hdr_lat_arr = new float[nlocs]; @@ -496,19 +557,20 @@ void process_ioda_file(int i_pb) { float *hdr_elv_arr = new float[nlocs]; float *obs_pres_arr = new float[nlocs]; float *obs_hght_arr = new float[nlocs]; + float *hdr_time_arr = new float[nlocs]; char *hdr_vld_block = new char[nlocs*ndatetime]; char *hdr_msg_types = 0; char *hdr_station_ids = 0; vector v_qc_data; vector v_obs_data; - get_meta_data_float(f_in, metadata_vars, "pressure", obs_pres_arr, nlocs); - get_meta_data_float(f_in, metadata_vars, "height", obs_hght_arr, nlocs); - get_meta_data_float(f_in, metadata_vars, "elevation", hdr_elv_arr, nlocs); + get_meta_data_float(f_in, metadata_vars, "pressure", obs_pres_arr, nlocs, ioda_format); + get_meta_data_float(f_in, metadata_vars, "height", obs_hght_arr, nlocs, ioda_format); + get_meta_data_float(f_in, metadata_vars, "elevation", hdr_elv_arr, nlocs, ioda_format); if(has_msg_type) { hdr_msg_types = new char[nlocs*nstring]; - get_meta_data_strings(f_in, msg_type_name, hdr_msg_types); + get_meta_data_strings(f_in, msg_type_name, hdr_msg_types, ioda_format); } else { mlog << Debug(1) << method_name @@ -517,7 +579,7 @@ void process_ioda_file(int i_pb) { if(has_station_id) { hdr_station_ids = new char[nlocs*nstring]; - get_meta_data_strings(f_in, station_id_name, hdr_station_ids); + get_meta_data_strings(f_in, station_id_name, hdr_station_ids, ioda_format); } else { mlog << Debug(1) << method_name @@ -534,7 +596,10 @@ void process_ioda_file(int i_pb) { << "trouble getting longitude\n\n"; exit(1); } - if(!get_nc_data(&in_hdr_vld_var, hdr_vld_block, lengths, offsets)) { + + status = number_time ? get_nc_data(&in_hdr_vld_var, hdr_time_arr, lengths, offsets) + : get_nc_data(&in_hdr_vld_var, hdr_vld_block, lengths, offsets); + if(!status) { mlog << Error << "\n" << method_name << "trouble getting datetime\n\n"; exit(1); @@ -550,17 +615,21 @@ void process_ioda_file(int i_pb) { for(idx=0; idx 0) { @@ -620,11 +690,17 @@ void process_ioda_file(int i_pb) { } } - char valid_time[ndatetime+1]; - m_strncpy(valid_time, (const char *)(hdr_vld_block + (i_read * ndatetime)), - ndatetime, method_name_s, "valid_time", true); - valid_time[ndatetime] = 0; - msg_ut = yyyymmddThhmmss_to_unix(valid_time); + if (number_time) { + msg_ut = add_to_unixtime(base_ut, sec_per_unit, + hdr_time_arr[i_read], no_leap_year); + } + else { + char valid_time[ndatetime+1]; + m_strncpy(valid_time, (const char *)(hdr_vld_block + (i_read * ndatetime)), + ndatetime, method_name_s, "valid_time", true); + valid_time[ndatetime] = 0; + msg_ut = yyyymmddThhmmss_to_unix(valid_time); + } // Check to make sure that the message time hasn't changed // from one IODA message to the next @@ -638,8 +714,7 @@ void process_ioda_file(int i_pb) { // Check if valid_beg_ut and valid_end_ut were set on the // command line. If so, use them. If not, use beg_ds and // end_ds. - if(valid_beg_ut != (unixtime) 0 || - valid_end_ut != (unixtime) 0) { + if(valid_beg_ut != (unixtime) 0 || valid_end_ut != (unixtime) 0) { beg_ut = valid_beg_ut; end_ut = valid_end_ut; } @@ -1056,30 +1131,47 @@ bool keep_valid_time(const unixtime ut, //////////////////////////////////////////////////////////////////////// bool check_core_data(const bool has_msg_type, const bool has_station_id, - StringArray &dim_names, StringArray &metadata_vars) { + StringArray &dim_names, StringArray &metadata_vars, + e_ioda_format ioda_format) { bool is_netcdf_ready = true; static const char *method_name = "check_core_data() -> "; - - for(int idx=0; idx " - << "core dimension \"" << core_dims[idx] << "\" is missing.\n\n"; + << "core dimension \"" << t_core_dims[idx] << "\" is missing.\n\n"; is_netcdf_ready = false; } } - if(has_msg_type || has_station_id) { - if(!dim_names.has("nstring")) { - mlog << Error << "\n" << method_name << "-> " - << "core dimension \"nstring\" is missing.\n\n"; - is_netcdf_ready = false; + if (ioda_format == ioda_v1) { + if(has_msg_type || has_station_id) { + if(!dim_names.has("nstring")) { + mlog << Error << "\n" << method_name << "-> " + << "core dimension \"nstring\" is missing.\n\n"; + is_netcdf_ready = false; + } } } - for(int idx=0; idx 0) { + for (int idx2=0; idx2 < alt_names.n(); idx2++) { + if (core_meta_vars[idx] != alt_names[idx2]) { + found = metadata_vars.has(alt_names[idx2]); + if (found) break; + } + } + } + } + if(!found) { mlog << Error << "\n" << method_name << "-> " - << "core variable \"" << core_vars[idx] << "\" is missing.\n\n"; + << "core variable \"" << core_meta_vars[idx] << "\" is missing.\n\n"; is_netcdf_ready = false; } } @@ -1117,16 +1209,15 @@ ConcatString find_meta_name(StringArray metadata_names, StringArray config_names bool get_meta_data_float(NcFile *f_in, StringArray &metadata_vars, const char *metadata_key, float *metadata_buf, - const int nlocs) { + const int nlocs, e_ioda_format ioda_format) { bool status = false; static const char *method_name = "get_meta_data_float() -> "; ConcatString metadata_name = find_meta_name( metadata_vars, conf_info.metadata_map[metadata_key]); + if(metadata_name.length() > 0) { - ConcatString ioda_name = metadata_name; - ioda_name.add("@MetaData"); - NcVar meta_var = get_var(f_in, ioda_name.c_str()); + NcVar meta_var = get_var(f_in, metadata_name.c_str(), metadata_group_name); if(IS_VALID_NC(meta_var)) { status = get_nc_data(&meta_var, metadata_buf, nlocs); if(!status) mlog << Debug(3) << method_name @@ -1149,13 +1240,11 @@ bool get_meta_data_float(NcFile *f_in, StringArray &metadata_vars, //////////////////////////////////////////////////////////////////////// bool get_meta_data_strings(NcFile *f_in, const ConcatString metadata_name, - char *metadata_buf) { + char *metadata_buf, e_ioda_format ioda_format) { bool status = false; static const char *method_name = "get_meta_data_strings() -> "; - ConcatString ioda_name = metadata_name; - ioda_name.add("@MetaData"); - NcVar meta_var = get_var(f_in, ioda_name.c_str()); + NcVar meta_var = get_var(f_in, metadata_name.c_str(), metadata_group_name); if(IS_VALID_NC(meta_var)) { status = get_nc_data(&meta_var, metadata_buf); if(!status) { @@ -1171,7 +1260,7 @@ bool get_meta_data_strings(NcFile *f_in, const ConcatString metadata_name, bool get_obs_data_float(NcFile *f_in, const ConcatString var_name, NcVar *obs_var, float *obs_buf, int *qc_buf, - const int nlocs) { + const int nlocs, const e_ioda_format ioda_format) { bool status = false; static const char *method_name = "get_obs_data_float() -> "; @@ -1185,21 +1274,35 @@ bool get_obs_data_float(NcFile *f_in, const ConcatString var_name, << "trouble getting " << var_name << "\n\n"; } else mlog << Error << "\n" << method_name - << var_name << "@ObsValue does not exist!\n\n"; + << var_name << " or " << var_name << "@ObsValue does not exist!\n\n"; if(!status) exit(1); status = false; if(var_name.length() > 0) { - ConcatString ioda_name = var_name; - ioda_name.add("@PreQC"); - NcVar qc_var = get_var(f_in, ioda_name.c_str()); + ConcatString qc_name = var_name; + ConcatString qc_group = (ioda_format == ioda_v1) ? qc_postfix : qc_group_name; + if (ioda_format == ioda_v2) { + StringArray qc_names = conf_info.obs_to_qc_map[var_name]; + if (0 < qc_names.n()) { + for (int idx=0; idx