diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 97eaa684db..1df3a0e883 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -7,6 +7,7 @@ This file contains a high-level description of this package's evolution. Release ## 4.8.1 - TBD +* [Bug Fixes] The netcdf-c library was incorrectly determining the scope of dimension; similar to the type scope problem. See [Github #2012](https://github.com/Unidata/netcdf-c/pull/2012) for more information. * [Bug Fix] Re-enable DAP2 authorization testing. See [Github #2011](https://github.com/Unidata/netcdf-c/issues/2011). * [Bug Fix] Fix bug with windows version of mkstemp that causes failure to create more than 26 temp files. See [Github #1998](https://github.com/Unidata/netcdf-c/pull/1998). * [Bug Fix] Fix ncdump bug when printing VLENs with basetype char. See [Github #1986](https://github.com/Unidata/netcdf-c/issues/1986). diff --git a/include/netcdf.h b/include/netcdf.h index 7ad545c688..b6d434a780 100644 --- a/include/netcdf.h +++ b/include/netcdf.h @@ -635,7 +635,7 @@ nc_insert_array_compound(int ncid, nc_type xtype, const char *name, EXTERNL int nc_inq_type(int ncid, nc_type xtype, char *name, size_t *size); -/* Get the id of a type from the name. */ +/* Get the id of a type from the name, which might be a fully qualified name */ EXTERNL int nc_inq_typeid(int ncid, const char *name, nc_type *typeidp); diff --git a/libdispatch/dcopy.c b/libdispatch/dcopy.c index 8a73f9d82a..86fb641671 100644 --- a/libdispatch/dcopy.c +++ b/libdispatch/dcopy.c @@ -767,6 +767,7 @@ searchgrouptree(int ncid1, int tid1, int grp, int* tid2) id = ids[i]; nclistpush(queue,(void*)id); } + free(ids); ids = NULL; } /* Not found */ ret = NC_EBADTYPE; diff --git a/libsrc4/nc4dim.c b/libsrc4/nc4dim.c index be076f9b41..2082712d32 100644 --- a/libsrc4/nc4dim.c +++ b/libsrc4/nc4dim.c @@ -71,7 +71,7 @@ NC4_inq_unlimdim(int ncid, int *unlimdimidp) /** * @internal Given dim name, find its id. - * + * Fully qualified names are legal * @param ncid File and group ID. * @param name Name of the dimension to find. * @param idp Pointer that gets dimension ID. @@ -85,28 +85,56 @@ NC4_inq_unlimdim(int ncid, int *unlimdimidp) int NC4_inq_dimid(int ncid, const char *name, int *idp) { - NC *nc; - NC_GRP_INFO_T *grp, *g; - NC_FILE_INFO_T *h5; - NC_DIM_INFO_T *dim; + NC *nc = NULL; + NC_GRP_INFO_T *grp = NULL; + NC_GRP_INFO_T *g = NULL; + NC_FILE_INFO_T *h5 = NULL; + NC_DIM_INFO_T *dim = NULL; char norm_name[NC_MAX_NAME + 1]; - int retval; - int found; + int retval = NC_NOERR;; + int found = 0; LOG((2, "%s: ncid 0x%x name %s", __func__, ncid, name)); /* Check input. */ if (!name) - return NC_EINVAL; + {retval = NC_EINVAL; goto done;} + + /* If the first char is a /, this is a fully-qualified + * name. Otherwise, this had better be a local name (i.e. no / in + * the middle). */ + if (name[0] != '/' && strstr(name, "/")) + {retval = NC_EINVAL; goto done;} /* Find metadata for this file. */ if ((retval = nc4_find_nc_grp_h5(ncid, &nc, &grp, &h5))) - return retval; + goto done; assert(h5 && nc && grp); /* Normalize name. */ if ((retval = nc4_normalize_name(name, norm_name))) - return retval; + goto done;; + + /* If this is a fqn, then walk the sequence of parent groups to the last group + and see if that group has a dimension of the right name */ + if(name[0] == '/') { /* FQN */ + int rootncid = (grp->nc4_info->root_grp->hdr.id | grp->nc4_info->controller->ext_ncid); + int parent = 0; + char* lastname = strrchr(norm_name,'/'); /* break off the last segment: the type name */ + if(lastname == norm_name) + {retval = NC_EINVAL; goto done;} + *lastname++ = '\0'; /* break off the lastsegment */ + if((retval = NC4_inq_grp_full_ncid(rootncid,norm_name,&parent))) + goto done; + /* Get parent info */ + if((retval=nc4_find_nc4_grp(parent,&grp))) + goto done; + /* See if dim exists in this group */ + dim = (NC_DIM_INFO_T*)ncindexlookup(grp->dim,lastname); + if(dim == NULL) + {retval = NC_EBADTYPE; goto done;} + goto done; + } /* check for a name match in this group and its parents */ found = 0; @@ -115,11 +143,15 @@ NC4_inq_dimid(int ncid, const char *name, int *idp) if(dim != NULL) {found = 1; break;} } if(!found) - return NC_EBADDIM; - assert(dim != NULL); - if (idp) - *idp = dim->hdr.id; - return NC_NOERR; + {retval = NC_EBADDIM; goto done;} + +done: + if(retval == NC_NOERR) { + assert(dim != NULL); + if (idp) + *idp = dim->hdr.id; + } + return retval; } /** diff --git a/libsrc4/nc4grp.c b/libsrc4/nc4grp.c index dd08f7b806..f4a757baae 100644 --- a/libsrc4/nc4grp.c +++ b/libsrc4/nc4grp.c @@ -43,6 +43,15 @@ NC4_inq_ncid(int ncid, const char *name, int *grp_ncid) return retval; assert(h5); + /* Short circuit the case of name == NULL => return the root group */ + if(name == NULL) { + if(grp_ncid) { + NC_FILE_INFO_T* file = grp->nc4_info; + *grp_ncid = file->controller->ext_ncid | file->root_grp->hdr.id; + } + return NC_NOERR; + } + /* Normalize name. */ if ((retval = nc4_check_name(name, norm_name))) return retval; diff --git a/libsrc4/nc4type.c b/libsrc4/nc4type.c index 4e3592f180..9e9f369782 100644 --- a/libsrc4/nc4type.c +++ b/libsrc4/nc4type.c @@ -537,7 +537,7 @@ NC4_inq_enum_member(int ncid, nc_type typeid1, int idx, char *identifier, * @internal Get the id of a type from the name. * * @param ncid File and group ID. - * @param name Name of type. + * @param name Name of type; might be fully qualified. * @param typeidp Pointer that will get the type ID. * * @return ::NC_NOERR No error. diff --git a/nc_test4/test_filter_misc.c b/nc_test4/test_filter_misc.c index 711f5442ad..59a2249f12 100644 --- a/nc_test4/test_filter_misc.c +++ b/nc_test4/test_filter_misc.c @@ -349,6 +349,7 @@ buildbaseline(unsigned int testcasenumber) insert(12,&val8,sizeof(val8)); /* 12 unsigned long long */ break; case 2: + case 3: break; default: fprintf(stderr,"Unknown testcase number: %d\n",testcasenumber); @@ -418,6 +419,41 @@ test_test2(void) return ok; } +static int +test_test3(void) +{ + int ok = 1; + int stat = NC_NOERR; + + reset(); + + buildbaseline(3); + + fprintf(stderr,"test3: dimsize %% chunksize != 0: compress.\n"); + create(); + setchunking(); + setvarfilter(); + showparameters(); + CHECK(nc_enddef(ncid)); + + /* Fill in the array */ + fill(); + /* write array */ + stat = nc_put_var(ncid,varid,expected); + + fprintf(stderr,"test3: error code = %d\n",stat); + + CHECK(nc_close(ncid)); + + fprintf(stderr,"test3: dimsize %% chunksize != 0: decompress.\n"); + reset(); + openfile(); + CHECK(nc_get_var_float(ncid, varid, array)); + ok = compare(); + CHECK(nc_close(ncid)); + return ok; +} + /**************************************************/ /* Utilities */ @@ -510,5 +546,6 @@ main(int argc, char **argv) init(argc,argv); if(!test_test1()) ERRR; if(!test_test2()) ERRR; + if(!test_test3()) ERRR; exit(nerrs > 0?1:0); } diff --git a/nc_test4/tst_filter.sh b/nc_test4/tst_filter.sh index cc297c1553..f21fcc914c 100755 --- a/nc_test4/tst_filter.sh +++ b/nc_test4/tst_filter.sh @@ -103,7 +103,7 @@ rm -f ./tst_filter.txt trimleft ./tst_filter2.txt ./tst_filter.txt rm -f ./tst_filter2.txt cat >./tst_filter2.txt < 0) printf ("("); for (id = 0; id < var.ndims; id++) { - /* This dim may be in a parent group, so let's look up the - * name. */ + /* Get the base name of the dimension */ NC_CHECK( nc_inq_dimname(ncid, var.dims[id], dim_name) ); #ifdef USE_NETCDF4 - /* Subtlety: The following code block is needed because + /* This dim may be in a parent group, so let's look up dimid + * parent group; if it is not current group, then we will print + * the fully qualified name. + * Subtlety: The following code block is needed because * nc_inq_dimname() currently returns only a simple dimension * name, without a prefix identifying the group it came from. * That's OK unless the dimid identifies a dimension in an @@ -1746,45 +1755,83 @@ do_ncdump_rec(int ncid, const char *path) * group), in which case the simple name is ambiguous. This * code tests for that case and provides an absolute dimname * only in the case where a simple name would be - * ambiguous. */ - { - int dimid_test; /* to see if dimname is ambiguous */ - int target_dimid; /* from variable dim list */ - int locid; /* group id where dimension is defined */ - /*Locate the innermost definition of a dimension with given name*/ - NC_CHECK( nc_inq_dimid(ncid, dim_name, &dimid_test) ); - - /* Now, starting with current group, walk the parent chain - upward looking for the target dim_id */ - target_dimid = var.dims[id]; - locid = ncid; - while(target_dimid != dimid_test) {/*not in locid, try ancestors*/ - int parent_id; - NC_CHECK( nc_inq_grp_parent(locid, &parent_id) ); - locid = parent_id; - /* Is dim of this name defined in this group or higher? */ - NC_CHECK( nc_inq_dimid(locid, dim_name, &dimid_test) ); + * ambiguous. + * The algorithm is as follows: + * 1. Search up the tree of ancestor groups. + * 2. If one of those groups contains the dimid, then call it dimgrp. + * 3. If one of those groups contains a dim with the same name as the dimid, + * but with a different dimid, then record that as duplicate=true. + * 4. If dimgrp is defined and duplicate == false, then we do not need an fqn. + * 5. If dimgrp is defined and duplicate == true, then we do need an fqn to avoid using the duplicate. + * 6. if dimgrp is undefined, then do a preorder breadth-first search of all the groups looking for the + * dimid. + * 7. If found, then use the fqn of that dimension location. + * 8. If not found, then signal NC_EBADDIM. + */ + + int target_dimid, dimgrp, duplicate, stopsearch, usefqn; + + target_dimid = var.dims[id]; + dimgrp = ncid; /* start with the parent group of the variable */ + duplicate = 0; + usefqn = 0; + + /* Walk up the ancestor groups */ + for(stopsearch=0;stopsearch==0;) { + int tmpid; + int localdimid; + int ret = NC_NOERR; + ret = nc_inq_dimid(dimgrp,dim_name,&localdimid); + switch (ret) { + case NC_NOERR: /* We have a name match */ + if(localdimid == target_dimid) stopsearch = 1; /* 1 means stop because found */ + else duplicate = 1; + break; + case NC_EBADDIM: + break; /* no match at all */ + default: NC_CHECK(ret); } - /* innermost dimid with given name is in group locid. - If this is not current group, then use fully qualified - name (fqn) for the dimension name by prefixing dimname - with group name */ - if(locid != ncid) { /* We need to use fqn */ - size_t len; - char *locname; /* the group name */ - NC_CHECK( nc_inq_grpname_full(locid, &len, NULL) ); - locname = emalloc(len + 1); - NC_CHECK( nc_inq_grpname_full(locid, &len, locname) ); - print_name (locname); - if(strcmp("/", locname) != 0) { /* not the root group */ - printf("/"); /* ensure a trailing slash */ - } - free(locname); + if(stopsearch != 0) break; /* no need to continue */ + /* move to ancestor group */ + ret = nc_inq_grp_parent(dimgrp,&tmpid); + switch(ret) { + case NC_NOERR: + dimgrp = tmpid; + break; + case NC_ENOGRP: + /* we processed the root, so try the breadth-first search */ + stopsearch = -1; /* -1 means we hit the root group but did not find it */ + rootncid = dimgrp; + break; + default: NC_CHECK(ret); } - } -#endif /* USE_NETCDF4 */ - print_name (dim_name); - printf ("%s", id < var.ndims-1 ? ", " : ")"); + } + assert(stopsearch != 0); + if(stopsearch == 1) { + /* We found it; do we need to use fqn */ + usefqn = duplicate; + } else { /* stopsearch == -1 */ + /* do the whole-tree search */ + usefqn = 1; + NC_CHECK(searchgrouptreedim(rootncid,target_dimid,&dimgrp)); + /* group containing target dimid is in group dimgrp */ + } + if(usefqn) { + /* use fully qualified name (fqn) for the dimension name by prefixing dimname + with group name */ + size_t len; + char *grpfqn = NULL; /* the group fqn */ + NC_CHECK( nc_inq_grpname_full(dimgrp, &len, NULL) ); + grpfqn = emalloc(len + 1); + NC_CHECK( nc_inq_grpname_full(dimgrp, &len, grpfqn) ); + print_name (grpfqn); + if(strcmp("/", grpfqn) != 0) /* not the root group */ + printf("/"); /* ensure a trailing slash */ + free(grpfqn); + } +#endif /*USE_NETCDF4*/ + print_name (dim_name); + printf ("%s", id < var.ndims-1 ? ", " : RPAREN); } printf (" ;\n"); @@ -2417,3 +2464,83 @@ main(int argc, char *argv[]) error("%s", errmsg); exit(EXIT_FAILURE); } + +/* Helper function for searchgrouptreedim + search a specified group for matching dimid. +*/ +static int +searchgroupdim(int grp, int dimid) +{ + int i,ret = NC_NOERR; + int nids; + int* ids = NULL; + + /* Get all dimensions in parentid */ + if ((ret = nc_inq_dimids(grp, &nids, NULL, 0))) + goto done; + if (nids > 0) { + if (!(ids = (int *)malloc((size_t)nids * sizeof(int)))) + {ret = NC_ENOMEM; goto done;} + if ((ret = nc_inq_dimids(grp, &nids, ids, 0))) + goto done; + for(i = 0; i < nids; i++) { + if(ids[i] == dimid) goto done; + } + } else + ret = NC_EBADDIM; + +done: + nullfree(ids); + return ret; +} + +/* Helper function for do_ncdump_rec + search a tree of groups for a matching dimid + using a breadth first queue. Return the + immediately enclosing group. +*/ +static int +searchgrouptreedim(int ncid, int dimid, int* parentidp) +{ + int i,ret = NC_NOERR; + int nids; + int* ids = NULL; + NClist* queue = nclistnew(); + int gid; + uintptr_t id; + + id = ncid; + nclistpush(queue,(void*)id); /* prime the queue */ + while(nclistlength(queue) > 0) { + id = (uintptr_t)nclistremove(queue,0); + gid = (int)id; + switch (ret = searchgroupdim(gid,dimid)) { + case NC_NOERR: /* found it */ + if(parentidp) *parentidp = gid; + goto done; + case NC_EBADDIM: /* not in this group; keep looking */ + break; + default: goto done; + } + /* Get subgroups of gid and push onto front of the queue (for breadth first) */ + if((ret = nc_inq_grps(gid,&nids,NULL))) + goto done; + if (!(ids = (int *)malloc((size_t)nids * sizeof(int)))) + {ret = NC_ENOMEM; goto done;} + if ((ret = nc_inq_grps(gid, &nids, ids))) + goto done; + /* push onto the end of the queue */ + for(i=0;i #include #include + +#if defined(_WIN32) && ! defined(__MINGW32__) +#include "XGetopt.h" +#else +#include +#endif + #include #define CHECK(err) {if(err) report(err,__LINE__);} +/* Command line options */ +struct Options { + int debug; + enum What {NONE=0, DIM=1, TYPE=2} what; + char file[4096]; + char var[NC_MAX_NAME+1]; + char object[NC_MAX_NAME+1]; +} options; + +/**************************************************/ + + static void report(int err, int lineno) { @@ -16,25 +35,30 @@ report(int err, int lineno) void usage(void) { - fprintf(stderr,"usage: printfqn \n"); + fprintf(stderr,"usage: printfqn [-D] [-V] -t|-d -v -f \n"); exit(0); } int -get_type_parent(int ncid, int tid, int* parentp) +get_id_parent(int ncid, int id, int* parentp, enum What what) { int stat = NC_NOERR; int i; int nids; int ids[4096]; - /* Does this group have the type we are searching for? */ - if((stat=nc_inq_typeids(ncid,&nids,ids))) goto done; + /* Does this group have the id we are searching for? */ + if(what == TYPE) { + if((stat=nc_inq_typeids(ncid,&nids,ids))) goto done; + } else if(what == DIM) { + if((stat=nc_inq_dimids(ncid,&nids,ids,0))) goto done; + } else + abort(); assert(nids < 4096); - /* Search for this typeid */ + /* Search for this id */ for(i=0;i copy_$1.cdl diff -wB ${srcdir}/$1.cdl ${execdir}/copy_$1.cdl -REFT=`${execdir}/printfqn ${execdir}/$1.nc test_variable` -COPYT=`${execdir}/printfqn ${execdir}/$1_copy.nc test_variable` +} + +typescope() { +REFT=`${execdir}/printfqn -f ${execdir}/$1.nc -v test_variable -t` +COPYT=`${execdir}/printfqn -f ${execdir}/$1_copy.nc -v test_variable -t` if test "x$REFT" != "x$COPYT" ; then echo "***Fail: ref=${REFT} copy=${COPYT}" + exit 1 fi } -if test "x$SETUP" = x1 ; then -for t in $TSTS ; do - setup $t -done +dimscope() { +REFT=`${execdir}/printfqn -f ${execdir}/$1.nc -v test_variable -d` +COPYT=`${execdir}/printfqn -f ${execdir}/$1_copy.nc -v test_variable -d` +if test "x$REFT" != "x$COPYT" ; then + echo "***Fail: ref=${REFT} copy=${COPYT}" fi +} -for t in $TSTS ; do - testscope $t -done +for t in $TSTS ; do setup $t; done +for t in $TSTS ; do testcycle $t; done +for t in $TSTS ; do typescope $t; done +for t in $TSTS ; do dimscope $t; done exit 0 - - diff --git a/ncdump/type_preorder.cdl b/ncdump/type_preorder.cdl deleted file mode 100644 index 4b888fc6c3..0000000000 --- a/ncdump/type_preorder.cdl +++ /dev/null @@ -1,13 +0,0 @@ -netcdf type_preorder { - -group: preorder { -types: - ubyte enum test_enum_type {OPTION1 = 0, OPTION2 = 1, OPTION3 = 2} ; -} // group preorder - -group: test_group { - variables: - /preorder/test_enum_type test_variable ; - /preorder/test_enum_type test_variable:_FillValue = OPTION1 ; - } // group test_group -} diff --git a/plugins/H5Zmisc.c b/plugins/H5Zmisc.c index 8f562bad58..beb84aced2 100644 --- a/plugins/H5Zmisc.c +++ b/plugins/H5Zmisc.c @@ -95,6 +95,7 @@ H5Z_filter_test(unsigned int flags, size_t cd_nelmts, { void* newbuf; unsigned int testcase = 0; + size_t size = 1024 * sizeof(float) * 2; if(cd_nelmts == 0) goto fail; @@ -114,16 +115,36 @@ H5Z_filter_test(unsigned int flags, size_t cd_nelmts, default: break; } - if (flags & H5Z_FLAG_REVERSE) { - - /* Replace buffer */ + if (flags & H5Z_FLAG_REVERSE) { /* Decompress */ + + if(testcase == TC_EXPANDED) { + int i; + float* b = (float*)*buf; +fprintf(stderr,"TC_EXPANDED: decompress: nbytes=%u buf_size=%u xdata[0..8]=|",(unsigned)nbytes,(unsigned)*buf_size); + for(i=0;i<8;i++) { + fprintf(stderr," %u",(int)(b[1024+i])); + } + fprintf(stderr,"|\n"); + /* Replace buffer */ +#ifdef HAVE_H5ALLOCATE_MEMORY + newbuf = H5allocate_memory(*buf_size,0); +#else + newbuf = malloc(*buf_size); +#endif + if(newbuf == NULL) abort(); + memcpy(newbuf,*buf,*buf_size); + + } else { + /* Replace buffer */ #ifdef HAVE_H5ALLOCATE_MEMORY - newbuf = H5allocate_memory(*buf_size,0); + newbuf = H5allocate_memory(*buf_size,0); #else - newbuf = malloc(*buf_size); + newbuf = malloc(*buf_size); #endif - if(newbuf == NULL) abort(); - memcpy(newbuf,*buf,*buf_size); + if(newbuf == NULL) abort(); + memcpy(newbuf,*buf,*buf_size); + } + /* reclaim old buffer */ #ifdef HAVE_H5FREE_MEMORY H5free_memory(*buf); @@ -132,16 +153,37 @@ H5Z_filter_test(unsigned int flags, size_t cd_nelmts, #endif *buf = newbuf; - } else { - - /* Replace buffer */ + } else { /* Compress */ + if(testcase == TC_EXPANDED) { + int i; + float* b; +#if 0 +fprintf(stderr,"TC_EXPANDED: compress: nbytes=%u buf_size=%u size=%u\n",(unsigned)nbytes,(unsigned)*buf_size,(unsigned)size); +#endif + /* Replace buffer with one that is bigger than the chunk size */ #ifdef HAVE_H5ALLOCATE_MEMORY - newbuf = H5allocate_memory(*buf_size,0); + newbuf = H5allocate_memory(size,0); #else - newbuf = malloc(*buf_size); + newbuf = malloc(size); #endif - if(newbuf == NULL) abort(); - memcpy(newbuf,*buf,*buf_size); + if(newbuf == NULL) abort(); + b = (float*)newbuf; + for(i=0;i<1024*2;i++) { + b[i] = (float)(17+i); + } + memcpy(newbuf,*buf,*buf_size); + *buf_size = size; + } else { + /* Replace buffer */ +#ifdef HAVE_H5ALLOCATE_MEMORY + newbuf = H5allocate_memory(*buf_size,0); +#else + newbuf = malloc(*buf_size); +#endif + if(newbuf == NULL) abort(); + memcpy(newbuf,*buf,*buf_size); + } + /* reclaim old buffer */ #ifdef HAVE_H5FREE_MEMORY H5free_memory(*buf); @@ -149,7 +191,6 @@ H5Z_filter_test(unsigned int flags, size_t cd_nelmts, free(*buf); #endif *buf = newbuf; - } return *buf_size; diff --git a/plugins/h5misc.h b/plugins/h5misc.h index 11a1cd8c83..61f3178732 100644 --- a/plugins/h5misc.h +++ b/plugins/h5misc.h @@ -20,6 +20,7 @@ typedef enum H5testcase { TC_NONE = 0, TC_ENDIAN = 1, TC_ODDSIZE = 2, +TC_EXPANDED = 3, } H5testcase; /* declare the hdf5 interface */