Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Feature #2068 ioda2nc v2.0 #2307

Merged
merged 25 commits into from
Oct 30, 2022
Merged

Feature #2068 ioda2nc v2.0 #2307

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
25 commits
Select commit Hold shift + click to select a range
75f3a41
#2068 Added obs_to_qc_map
Oct 8, 2022
95ed64d
#2068 Support obs_to_qc_map
Oct 8, 2022
e2c72ab
#2068 Support group
Oct 8, 2022
91c111a
#2068 Support IODA v2
Oct 8, 2022
90f63e2
Deleted temp codes
Oct 10, 2022
7701771
Merge remote-tracking branch 'origin/develop' into feature_2068_ioda2…
JohnHalleyGotway Oct 10, 2022
bf7943b
#20768 Checks use_var_id attribute before getting variables
Oct 10, 2022
e842e5f
#2068 Updated comments
Oct 10, 2022
a531011
#2068 calls get_cf_conventions
Oct 13, 2022
8e651ef
Added metadata names for elevation
Oct 17, 2022
a022a80
#2068 Added unit tests for format v2
Oct 17, 2022
434fd83
#2068 Added get_nc_data for string type variable and get_cf_conventio…
Oct 17, 2022
bc09238
Added datetime
Oct 17, 2022
0541012
#2068 Support string data type valid times
Oct 17, 2022
36011b0
#2068 Cleanup
Oct 18, 2022
74b067a
Merge remote-tracking branch 'origin/develop' into feature_2068_ioda2…
Oct 18, 2022
4811522
#2068 Added obs_to_qc_map
Oct 18, 2022
f43372c
#2068 Hanlde float anmd double data type separately
Oct 19, 2022
411fa68
#2028 Initialize temp buffer
Oct 19, 2022
11250b8
Per #2068, update to using METbaseimage version 1.2 which has been up…
Oct 29, 2022
68cdc0a
Per #2068, update MET_BASE_TAG=v1.2 in three more spots.
Oct 29, 2022
4b6fbfe
Per #2068, METbaseimage version 1.2 also uses python version 3.8. Try…
JohnHalleyGotway Oct 29, 2022
7203883
Per #2068, working on the docker build environment to match METbaseim…
JohnHalleyGotway Oct 29, 2022
f860aa6
Per #2068, revert to METbaseimage version 1.1.
JohnHalleyGotway Oct 29, 2022
5809c79
Merge remote-tracking branch 'origin/develop' into feature_2068_ioda2…
JohnHalleyGotway Oct 30, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/jobs/set_job_controls.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ run_unit_tests=false
run_diff=false
run_update_truth=false
met_base_repo=met-base
met_base_tag=v1.0
met_base_tag=v1.1
input_data_version=develop
truth_data_version=develop

Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/build_docker_and_trigger_metplus.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ jobs:
env:
SOURCE_BRANCH: ${{ steps.get_branch_name.outputs.branch_name }}-lite
MET_BASE_REPO: met-base
MET_BASE_TAG: v1.0
MET_BASE_TAG: v1.1

- name: Push Docker Image
run: .github/jobs/push_docker_image.sh
Expand Down
11 changes: 10 additions & 1 deletion data/config/IODA2NCConfig_default
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,16 @@ metadata_map = [
{ key = "station_id"; val = "station_id,report_identifier"; },
{ key = "pressure"; val = "air_pressure,pressure"; },
{ key = "height"; val = "height,height_above_mean_sea_level"; },
{ key = "elevation"; val = ""; }
{ key = "datetime"; val = "datetime,dateTime"; },
{ key = "elevation"; val = "elevation,station_elevation"; }
];

//
// Default mapping for obs to qc.
//
obs_to_qc_map = [
{ key = "wind_from_direction"; val = "eastward_wind,northward_wind"; },
{ key = "wind_speed"; val = "eastward_wind,northward_wind"; }
];

missing_thresh = [ <=-1e9, >=1e9, ==-9999 ];
Expand Down
13 changes: 12 additions & 1 deletion docs/Users_Guide/reformat_point.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -961,13 +961,24 @@ _____________________
{ key = "station_id"; val = "station_id,report_identifier"; },
{ key = "pressure"; val = "air_pressure,pressure"; },
{ key = "height"; val = "height,height_above_mean_sea_level"; },
{ key = "elevation"; val = ""; }
{ key = "elevation"; val = "elevation,station_elevation"; }
];

This entry is an array of dictionaries, each containing a **key** string and **val** string which define a mapping of metadata for IODA data files.

_____________________

.. code-block:: none

obs_to_qc_map = [
{ key = "wind_from_direction"; val = "eastward_wind,northward_wind"; },
{ key = "wind_speed"; val = "eastward_wind,northward_wind"; }
];

This entry is an array of dictionaries, each containing a **key** string and **val** string which define a mapping of QC variable name for IODA data files.

_____________________

.. code-block:: none

missing_thresh = [ <=-1e9, >=1e9, ==-9999 ];
Expand Down
2 changes: 1 addition & 1 deletion internal/scripts/docker/Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
ARG MET_BASE_REPO=met-base
ARG MET_BASE_TAG=v1.0
ARG MET_BASE_TAG=v1.1

FROM dtcenter/${MET_BASE_REPO}:${MET_BASE_TAG}
MAINTAINER John Halley Gotway <johnhg@ucar.edu>
Expand Down
2 changes: 1 addition & 1 deletion internal/scripts/docker/Dockerfile.copy
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
ARG MET_BASE_REPO=met-base-unit-test
ARG MET_BASE_TAG=v1.0
ARG MET_BASE_TAG=v1.1

FROM dtcenter/${MET_BASE_REPO}:${MET_BASE_TAG}
MAINTAINER John Halley Gotway <johnhg@ucar.edu>
Expand Down
2 changes: 1 addition & 1 deletion internal/scripts/environment/development.docker
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ export MET_FREETYPELIB=/usr/lib64
export MET_JASPERLIB=/usr/lib64

export MET_PYTHON=/usr/bin/python3
export MET_PYTHON_CC="-I/usr/include/python3.6m -I/usr/include/python3.6m"
export MET_PYTHON_CC="-I/usr/include/python3.6m"
export MET_PYTHON_LD="-L/usr/lib64 -lpython3.6m -lpthread -ldl -lutil -lm"

# -D__64BIT__ is required because we've compiled libgrib2c.a with that flag
Expand Down
38 changes: 38 additions & 0 deletions internal/test_unit/xml/unit_ioda2nc.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -83,4 +83,42 @@
</output>
</test>

<test name="ioda2nc_int_datetime">
<exec>&MET_BIN;/ioda2nc</exec>
<env>
<pair><name>STATION_ID</name> <value></value></pair>
<pair><name>MASK_GRID</name> <value></value></pair>
<pair><name>MASK_POLY</name> <value></value></pair>
<pair><name>MESSAGE_TYPE</name> <value></value></pair>
</env>
<param> \
&DATA_DIR_OBS;/ioda/jopa_satwind_20210701T1200Z_out_0000_reduced.nc4 \
&OUTPUT_DIR;/ioda2nc/jopa_satwind_20210701T1200Z_int_datetime.nc \
-config &CONFIG_DIR;/IODA2NCConfig_mask \
-v 2
</param>
<output>
<point_nc>&OUTPUT_DIR;/ioda2nc/jopa_satwind_20210701T1200Z_int_datetime.nc</point_nc>
</output>
</test>

<test name="ioda2nc_v2_string_sid">
<exec>&MET_BIN;/ioda2nc</exec>
<env>
<pair><name>STATION_ID</name> <value></value></pair>
<pair><name>MASK_GRID</name> <value></value></pair>
<pair><name>MASK_POLY</name> <value></value></pair>
<pair><name>MESSAGE_TYPE</name> <value></value></pair>
</env>
<param> \
&DATA_DIR_OBS;/ioda/2021081612_sonde_small.nc \
&OUTPUT_DIR;/ioda2nc/2021081612_sonde_small_sid.nc \
-config &CONFIG_DIR;/IODA2NCConfig_mask \
-v 2
</param>
<output>
<point_nc>&OUTPUT_DIR;/ioda2nc/2021081612_sonde_small_sid.nc</point_nc>
</output>
</test>

</met_test>
2 changes: 2 additions & 0 deletions src/basic/vx_config/config_constants.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -650,6 +650,7 @@ static const char conf_key_trunc_factor[] = "gaussian_trunc_factor";
static const char conf_key_eclv_points[] = "eclv_points";
static const char conf_key_var_name_map[] = "var_name_map";
static const char conf_key_metadata_map[] = "metadata_map";
static const char conf_key_obs_to_qc_map[] = "obs_to_qc_map";
static const char conf_key_missing_thresh[] = "missing_thresh";
static const char conf_key_control_id[] = "control_id";
static const char conf_key_ens_member_ids[] = "ens_member_ids";
Expand Down Expand Up @@ -897,6 +898,7 @@ static const char conf_key_do_polylines_flag [] = "do_polylines";
// PB2NC specific parameter key names
//

static const char conf_key_datetime[] = "datetime";
static const char conf_key_station_id[] = "station_id";
static const char conf_key_elevation_range[] = "elevation_range";
static const char conf_key_pb_report_type[] = "pb_report_type";
Expand Down
7 changes: 7 additions & 0 deletions src/basic/vx_config/config_util.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1099,6 +1099,13 @@ map<ConcatString,ConcatString> parse_conf_obs_name_map(Dictionary *dict) {

///////////////////////////////////////////////////////////////////////////////

map<ConcatString,StringArray> parse_conf_obs_to_qc_map(Dictionary *dict) {
const char *method_name = "parse_conf_obs_to_qc_map() -> ";
return parse_conf_key_values_map(dict, conf_key_obs_to_qc_map, method_name);
}

///////////////////////////////////////////////////////////////////////////////

map<ConcatString,UserFunc_1Arg> parse_conf_key_convert_map(
Dictionary *dict, const char *conf_key_map_name, const char *caller) {
Dictionary *map_dict = (Dictionary *) 0;
Expand Down
11 changes: 7 additions & 4 deletions src/basic/vx_config/config_util.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,18 +52,21 @@ extern NumArray parse_conf_eclv_points(Dictionary *dict);
extern ClimoCDFInfo parse_conf_climo_cdf(Dictionary *dict);
extern TimeSummaryInfo parse_conf_time_summary(Dictionary *dict);
extern std::map<ConcatString,ConcatString> parse_conf_key_value_map(
Dictionary *dict, const char *conf_key_map_name, const char *caller=0);
Dictionary *dict, const char *conf_key_map_name, const char *caller=0);
extern void parse_add_conf_key_value_map(
Dictionary *dict, const char *conf_key_map_name, std::map<ConcatString,ConcatString> *m);
Dictionary *dict, const char *conf_key_map_name, std::map<ConcatString,ConcatString> *m);
extern std::map<ConcatString,ConcatString>
parse_conf_message_type_map(Dictionary *dict);
extern std::map<ConcatString,StringArray>
parse_conf_message_type_group_map(Dictionary *dict);
extern std::map<ConcatString,StringArray> parse_conf_metadata_map(Dictionary *dict);
extern std::map<ConcatString,ConcatString>
parse_conf_obs_name_map(Dictionary *dict);
extern std::map<ConcatString,UserFunc_1Arg> parse_conf_key_convert_map(
Dictionary *dict, const char *conf_key_map_name, const char *caller=0);
extern std::map<ConcatString,StringArray>
parse_conf_obs_to_qc_map(Dictionary *dict);
extern std::map<ConcatString,UserFunc_1Arg>
parse_conf_key_convert_map(
Dictionary *dict, const char *conf_key_map_name, const char *caller=0);
extern BootInfo parse_conf_boot(Dictionary *dict);
extern RegridInfo parse_conf_regrid(Dictionary *dict, bool error_out = default_dictionary_error_out);
extern InterpInfo parse_conf_interp(Dictionary *dict, const char *);
Expand Down
7 changes: 1 addition & 6 deletions src/libcode/vx_data2d_factory/is_netcdf_file.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,9 +40,6 @@ static const char netcdf_magic [] = "CDF";
static const char hdf_magic [] = "HDF";
static const int netcdf_magic_len = m_strlen(netcdf_magic);

static const string nccf_att_name = "Conventions";
static const string nccf_att_name_l = "conventions";
static const string nccf_att_name_U = "CONVENTIONS";
static const string nccf_att_value = "CF-";
static const string nccf_att_value2 = "CF ";
static const string nccf_att_value3 = "COARDS";
Expand Down Expand Up @@ -96,9 +93,7 @@ bool is_nccf_file(const char * filename)
NcFile *nc_file = open_ncfile(filename);

if (!IS_INVALID_NC_P(nc_file)) {
bool found = get_global_att(nc_file, nccf_att_name, att_val);
if (!found) found = get_global_att(nc_file, nccf_att_name_l, att_val);
if (!found) found = get_global_att(nc_file, nccf_att_name_U, att_val);
bool found = get_cf_conventions(nc_file, att_val); // "Conventions" attrribute
if (found) {
status = (att_val.compare(0, nccf_att_value.length(),
nccf_att_value) == 0 ||
Expand Down
35 changes: 19 additions & 16 deletions src/libcode/vx_nc_obs/nc_obs_util.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -480,6 +480,9 @@ void NetcdfObsVars::read_dims_vars(NcFile *f_in) {
obs_dim = get_nc_dim(f_in, nc_dim_nobs); // Observation array length
hdr_dim = get_nc_dim(f_in, nc_dim_nhdr); // Header array length

use_var_id = false;
get_global_att(f_in, nc_att_use_var_id, use_var_id);

// Get netCDF header variables
hdr_typ_var = get_var(f_in, nc_var_hdr_typ); // Message type (String or int)
hdr_sid_var = get_var(f_in, nc_var_hdr_sid); // Station ID (String or int)
Expand All @@ -503,10 +506,14 @@ void NetcdfObsVars::read_dims_vars(NcFile *f_in) {
obs_arr_var = get_var(f_in, nc_var_obs_arr);
} else {
obs_hid_var = ncVar; // Obs. header id array
ncVar = get_var(f_in, nc_var_obs_gc);
if (!IS_INVALID_NC(ncVar)) obs_gc_var = ncVar; // Obs. grib code array
ncVar = get_var(f_in, nc_var_obs_vid);
if (!IS_INVALID_NC(ncVar)) obs_vid_var = ncVar; // Obs. variable id array
if (use_var_id) {
ncVar = get_var(f_in, nc_var_obs_vid);
if (!IS_INVALID_NC(ncVar)) obs_vid_var = ncVar; // Obs. variable id array
}
else {
ncVar = get_var(f_in, nc_var_obs_gc);
if (!IS_INVALID_NC(ncVar)) obs_gc_var = ncVar; // Obs. grib code array
}
obs_lvl_var = get_var(f_in, nc_var_obs_lvl); // Obs. pressure level array
obs_hgt_var = get_var(f_in, nc_var_obs_hgt); // Obs. highth array
obs_val_var = get_var(f_in, nc_var_obs_val); // Obs. value array
Expand All @@ -516,12 +523,14 @@ void NetcdfObsVars::read_dims_vars(NcFile *f_in) {
ncVar = get_var(f_in, nc_var_obs_qty_tbl);
if (!IS_INVALID_NC(ncVar)) obs_qty_tbl_var = ncVar;

ncVar = get_var(f_in, nc_var_obs_var);
if (!IS_INVALID_NC(ncVar)) obs_var = ncVar;
ncVar = get_var(f_in, nc_var_unit);
if (!IS_INVALID_NC(ncVar)) unit_var = ncVar;
ncVar = get_var(f_in, nc_var_desc);
if (!IS_INVALID_NC(ncVar)) desc_var = ncVar;
if (use_var_id) {
ncVar = get_var(f_in, nc_var_obs_var);
if (!IS_INVALID_NC(ncVar)) obs_var = ncVar;
ncVar = get_var(f_in, nc_var_unit);
if (!IS_INVALID_NC(ncVar)) unit_var = ncVar;
ncVar = get_var(f_in, nc_var_desc);
if (!IS_INVALID_NC(ncVar)) desc_var = ncVar;
}

// PrepBufr only headers
ncVar = get_var(f_in, nc_var_hdr_prpt_typ);
Expand All @@ -531,12 +540,6 @@ void NetcdfObsVars::read_dims_vars(NcFile *f_in) {
ncVar = get_var(f_in, nc_var_hdr_inst_typ);
if (!IS_INVALID_NC(ncVar)) hdr_inst_typ_var = ncVar;

bool _use_var_id = false;
if (!get_global_att(f_in, nc_att_use_var_id, _use_var_id)) {
_use_var_id = IS_VALID_NC(obs_var);
}

use_var_id = _use_var_id;
}

////////////////////////////////////////////////////////////////////////
Expand Down
Loading