Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fixed #3046 - Failures converting data to VDC format #3051

Merged
merged 5 commits into from
Feb 24, 2022
Merged

Fixed #3046 - Failures converting data to VDC format #3051

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
629c8e2
Changed NetCDFCpp::DefVar to not return an error if the same variable is
clyne Feb 23, 2022
fa5929b
Fixed #3046 - Failures converting data to VDC format
clyne Feb 23, 2022
80a7a5b
clang-format pre-push hook
clyne Feb 23, 2022
37a029c
Removed newly added, third overloaded version of VDC::DefineDataVar()
clyne Feb 23, 2022
7db9170
clang-format pre-push hook
clyne Feb 23, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 15 additions & 3 deletions apps/cfvdccreate/cfvdccreate.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -283,9 +283,14 @@ int main(int argc, char **argv)
ok = dccf.GetVarCoordVars(datanames[i], false, coordvars);
VAssert(ok);

if (!dvar.GetHasMissing() || !compress) {
rc = vdc.DefineDataVar(dvar.GetName(), dimnames, coordvars, dvar.GetUnits(), dvar.GetXType(), compress);
} else {
// Don't compress the variable if it is also a coordinate variable
// Compression errors in coordinate variables can lead to
// non-conformant meshes
//
bool doCompress = compress;
if (find(coordnames.begin(), coordnames.end(), datanames[i]) != coordnames.end()) { doCompress = false; }

if (dvar.GetHasMissing() && doCompress) {
vector<string> sdimnames;
bool ok = dccf.GetVarDimNames(datanames[i], true, sdimnames);
VAssert(ok);
Expand All @@ -294,6 +299,13 @@ int main(int argc, char **argv)
maskvar(sdimnames, maskvar_name);

rc = vdc.DefineDataVar(dvar.GetName(), dimnames, coordvars, dvar.GetUnits(), dvar.GetXType(), dvar.GetMissingValue(), maskvar_name);

} else if (dvar.GetHasMissing() && !doCompress) {
rc = vdc.DefineDataVar(dvar.GetName(), dimnames, coordvars, dvar.GetUnits(), dvar.GetXType(), dvar.GetMissingValue(), "");


} else {
rc = vdc.DefineDataVar(dvar.GetName(), dimnames, coordvars, dvar.GetUnits(), dvar.GetXType(), doCompress);
}

if (rc < 0) { return (1); }
Expand Down
9 changes: 5 additions & 4 deletions include/vapor/VDC.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -553,7 +553,7 @@ class VDF_API VDC : public VAPoR::DC {
int DefineDataVar(string varname, std::vector<string> dimnames, std::vector<string> coordvars, string units, XType type, bool compressed);

//!
//! Define a compressed data variable with missing data
//! Define a compressed or non-compressed data variable with missing data
//!
//! \copydoc VDC::DefineDataVar(
//! string varname, std::vector <string> dimnames,
Expand All @@ -571,7 +571,8 @@ class VDF_API VDC : public VAPoR::DC {
//! variable may be less than or equal to that of \p varname. The dimensions
//! of \p maskvar must match the fastest varying dimensions of \p varname.
//! The \p maskvar variable must have been previously defined with
//! DefineDataVar().
//! DefineDataVar(). If \p maskvar is empty, the variable will
//! not be compressed.
//!
//! \sa DefineDimension(), DefineCoordVar(), SetCompressionBlock()
//!
Expand Down Expand Up @@ -914,7 +915,7 @@ class VDF_API VDC : public VAPoR::DC {

bool _valid_mask_var(string varname, vector<DC::Dimension> dimensions, vector<size_t> bs, bool compressed, string maskvar) const;

bool _ValidDefineDataVar(string varname, vector<string> dimnames, vector<string> coordnames, string units, XType type, bool compressed, string maskvar) const;
bool _ValidDefineDataVar(string varname, vector<string> dimnames, vector<string> coordnames, string units, XType type, bool compressed, bool has_missing, string maskvar) const;

bool _ValidCompressionBlock(vector<size_t> bs, string wname, vector<size_t> cratios) const;

Expand All @@ -925,7 +926,7 @@ class VDF_API VDC : public VAPoR::DC {

void _DefineMesh(string meshname, vector<string> dim_names, vector<string> coord_vars);

int _DefineDataVar(string varname, std::vector<string> dimnames, std::vector<string> coordvars, string units, XType type, bool compressed, double mv, string maskvar);
int _DefineDataVar(string varname, std::vector<string> dimnames, std::vector<string> coordvars, string units, XType type, bool compressed, bool has_missing, double mv, string maskvar);

vector<string> _GetCoordVarDimNames(const CoordVar &var, bool &time_varying) const;

Expand Down
18 changes: 13 additions & 5 deletions lib/vdc/VDC.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -441,7 +441,7 @@ void VDC::_DefineMesh(string meshname, vector<string> dim_names, vector<string>
_meshes[meshname] = DC::Mesh(meshname, dim_names, coord_vars);
}

int VDC::_DefineDataVar(string varname, vector<string> dim_names, vector<string> coord_vars, string units, XType type, bool compressed, double mv, string maskvar)
int VDC::_DefineDataVar(string varname, vector<string> dim_names, vector<string> coord_vars, string units, XType type, bool compressed, bool has_missing, double mv, string maskvar)
{
if (!_defineMode) {
SetErrMsg("Not in define mode");
Expand All @@ -458,7 +458,7 @@ int VDC::_DefineDataVar(string varname, vector<string> dim_names, vector<string>
int rc = _DefineImplicitCoordVars(dim_names, coord_vars, coord_vars);
if (rc < 0) return (-1);

if (!_ValidDefineDataVar(varname, dim_names, coord_vars, units, type, compressed, maskvar)) { return (-1); }
if (!_ValidDefineDataVar(varname, dim_names, coord_vars, units, type, compressed, has_missing, maskvar)) { return (-1); }

//
// Dimensions must have previosly been defined
Expand Down Expand Up @@ -499,6 +499,8 @@ int VDC::_DefineDataVar(string varname, vector<string> dim_names, vector<string>

if (!maskvar.empty()) {
_dataVars[varname] = DataVar(varname, units, type, wname, cratios, periodic, meshname, time_coord_var, DC::Mesh::NODE, mv, maskvar);
} else if (has_missing) {
_dataVars[varname] = DataVar(varname, units, type, wname, cratios, periodic, meshname, time_coord_var, DC::Mesh::NODE, mv);
} else {
_dataVars[varname] = DataVar(varname, units, type, wname, cratios, periodic, meshname, time_coord_var, DC::Mesh::NODE);
}
Expand All @@ -508,14 +510,15 @@ int VDC::_DefineDataVar(string varname, vector<string> dim_names, vector<string>

int VDC::DefineDataVar(string varname, vector<string> dim_names, vector<string> coord_vars, string units, XType type, bool compressed)
{
return (VDC::_DefineDataVar(varname, dim_names, coord_vars, units, type, compressed, 0.0, ""));
return (VDC::_DefineDataVar(varname, dim_names, coord_vars, units, type, compressed, false, 0.0, ""));
}

int VDC::DefineDataVar(string varname, vector<string> dim_names, vector<string> coord_vars, string units, XType type, double mv, string maskvar

)
{
return (VDC::_DefineDataVar(varname, dim_names, coord_vars, units, type, true, mv, maskvar));
bool compressed = !maskvar.empty();
return (VDC::_DefineDataVar(varname, dim_names, coord_vars, units, type, compressed, true, mv, maskvar));
}

bool VDC::getDataVarInfo(string varname, DC::DataVar &datavar) const
Expand Down Expand Up @@ -1089,8 +1092,13 @@ bool VDC::_valid_mask_var(string varname, vector<DC::Dimension> dimensions, vect
return (true);
}

bool VDC::_ValidDefineDataVar(string varname, vector<string> dim_names, vector<string> coord_vars, string units, XType type, bool compressed, string maskvar) const
bool VDC::_ValidDefineDataVar(string varname, vector<string> dim_names, vector<string> coord_vars, string units, XType type, bool compressed, bool has_missing, string maskvar) const
{
if (compressed && has_missing && maskvar.empty()) {
SetErrMsg("Mask variable required for compressed data with missing values");
return (false);
}

if (dim_names.size() > 4) {
SetErrMsg("Invalid number of dimensions");
return (false);
Expand Down
2 changes: 1 addition & 1 deletion lib/vdc/VDC_c.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -536,7 +536,7 @@ int VDC_DefineDataVar(VDC *p, const char *varname, const char **dimnames, size_t
for (int i = 0; i < coordvarsCount; i++) coordvars_v.push_back(string(coordvars[i]));
VDC_DEBUG_printff_nRun(": calling VDC::DefineDataVar(\"%s\", %s, %s, \"%s\", %s, %s);\n", varname, _stringVectorToString(dimnames_v).c_str(), _stringVectorToString(coordvars_v).c_str(), units,
_XTypeToString(_IntToXType(xtype)), _boolToStr(compressed));
int ret = p->DefineDataVar(string(varname), dimnames_v, coordvars_v, string(units), _IntToXType(xtype), compressed);
int ret = p->DefineDataVar(string(varname), dimnames_v, coordvars_v, string(units), _IntToXType(xtype), (bool)compressed);
VDC_DEBUG_printff_nRun(": return (%i);\n", ret);
if (ret < 0) VDC_DEBUG_printff_error(": Error message = \"%s\"\n", Wasp::MyBase::GetErrMsg());
VDC_DEBUG_printff_nRun(": Current coordvars = %s\n", _stringVectorToString(p->GetCoordVarNames()).c_str());
Expand Down
34 changes: 33 additions & 1 deletion lib/wasp/NetCDFCpp.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,27 @@ using namespace VAPoR;
return (rc); \
}

namespace {

bool var_defs_match(int ncid, int varid, int dimids[], int ndimids, nc_type xtype)
{
int mydimids[NC_MAX_DIMS];
int myndims;
(void)nc_inq_varndims(ncid, varid, &myndims);
(void)nc_inq_vardimid(ncid, varid, mydimids);
for (int i = 0; i < ndimids; i++) {
if (mydimids[i] != dimids[i]) return (false);
}
nc_type myxtype;
(void)nc_inq_vartype(ncid, varid, &myxtype);
if (myxtype != xtype) return (false);

return (true);
}


}; // namespace

NetCDFCpp::NetCDFCpp()
{
_ncid = -1;
Expand Down Expand Up @@ -108,8 +129,19 @@ int NetCDFCpp::DefVar(string name, nc_type xtype, vector<string> dimnames)
MY_NC_ERR(rc, _path, "nc_inq_dimid(" + dimnames[i] + ")");
}

// Don't error out if variable already exists and has same definition
//
int varid;
int rc = nc_def_var(_ncid, name.c_str(), xtype, dimnames.size(), dimids, &varid);
int rc = nc_inq_varid(_ncid, name.c_str(), &varid);
if (rc == NC_NOERR) {
if (!var_defs_match(_ncid, varid, dimids, dimnames.size(), xtype)) {
MY_NC_ERR(NC_ENAMEINUSE, _path, "nc_def_var(" + name + ")");
return (NC_ENAMEINUSE);
}
return (NC_NOERR);
}

rc = nc_def_var(_ncid, name.c_str(), xtype, dimnames.size(), dimids, &varid);
MY_NC_ERR(rc, _path, "nc_def_var(" + name + ")");
return (NC_NOERR);
}
Expand Down