diff --git a/include/netcdf.h b/include/netcdf.h index 4395b30e82..99bc0c2194 100644 --- a/include/netcdf.h +++ b/include/netcdf.h @@ -293,11 +293,14 @@ NOTE: The NC_MAX_DIMS, NC_MAX_ATTRS, and NC_MAX_VARS limits /** In HDF5 files you can set storage for each variable to be either * contiguous or chunked, with nc_def_var_chunking(). This define is - * used there. */ + * used there. Unknown storage is used for further extensions of HDF5 + * storage models, which should be handled transparently by netcdf */ /**@{*/ -#define NC_CHUNKED 0 -#define NC_CONTIGUOUS 1 -#define NC_COMPACT 2 +#define NC_CHUNKED 0 +#define NC_CONTIGUOUS 1 +#define NC_COMPACT 2 +#define NC_UNKNOWN_STORAGE 3 +#define NC_VIRTUAL 4 /**@}*/ /** In HDF5 files you can set check-summing for each variable. diff --git a/libhdf5/hdf5create.c b/libhdf5/hdf5create.c index 27a99405c9..747ee8accc 100644 --- a/libhdf5/hdf5create.c +++ b/libhdf5/hdf5create.c @@ -121,11 +121,12 @@ nc4_create_file(const char *path, int cmode, size_t initialsz, } /* Need this access plist to control how HDF5 handles open objects - * on file close. Setting H5F_CLOSE_SEMI will cause H5Fclose to - * fail if there are any open objects in the file. */ + * on file close. (Setting H5F_CLOSE_WEAK will cause H5Fclose not to + * fail if there are any open objects in the file. This may happen when virtual + * datasets are opened). */ if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0) BAIL(NC_EHDFERR); - if (H5Pset_fclose_degree(fapl_id, H5F_CLOSE_SEMI)) + if (H5Pset_fclose_degree(fapl_id, H5F_CLOSE_WEAK)) BAIL(NC_EHDFERR); #ifdef USE_PARALLEL4 diff --git a/libhdf5/hdf5open.c b/libhdf5/hdf5open.c index 06b5adbae2..c642a1c244 100644 --- a/libhdf5/hdf5open.c +++ b/libhdf5/hdf5open.c @@ -756,12 +756,13 @@ nc4_open_file(const char *path, int mode, void* parameters, int ncid) #endif /* !USE_PARALLEL4 */ /* Need this access plist to control how HDF5 handles open objects - * on file close. (Setting H5F_CLOSE_SEMI will cause H5Fclose to - * fail if there are any open objects in the file). */ + * on file close. (Setting H5F_CLOSE_WEAK will cause H5Fclose not to + * fail if there are any open objects in the file. This may happen when virtual + * datasets are opened). */ if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0) BAIL(NC_EHDFERR); - if (H5Pset_fclose_degree(fapl_id, H5F_CLOSE_SEMI) < 0) + if (H5Pset_fclose_degree(fapl_id, H5F_CLOSE_WEAK) < 0) BAIL(NC_EHDFERR); #ifdef USE_PARALLEL4 @@ -1140,6 +1141,16 @@ get_chunking_info(hid_t propid, NC_VAR_INFO_T *var) { var->storage = NC_COMPACT; } +#ifdef H5D_VIRTUAL + else if (layout == H5D_VIRTUAL) + { + var->storage = NC_VIRTUAL; + } +#endif + else + { + var->storage = NC_UNKNOWN_STORAGE; + } return NC_NOERR; } diff --git a/libsrc4/nc4internal.c b/libsrc4/nc4internal.c index 9c9d7ec956..968b2ab080 100644 --- a/libsrc4/nc4internal.c +++ b/libsrc4/nc4internal.c @@ -1709,8 +1709,12 @@ rec_print_metadata(NC_GRP_INFO_T *grp, int tab_count) strcat(storage_str, "contiguous"); else if (var->storage == NC_COMPACT) strcat(storage_str, "compact"); - else + else if (var->storage == NC_CHUNKED) strcat(storage_str, "chunked"); + else if (var->storage == NC_VIRTUAL) + strcat(storage_str, "virtual"); + else + strcat(storage_str, "unknown"); LOG((2, "%s VARIABLE - varid: %d name: %s ndims: %d " "dimids:%s storage: %s", tabs, var->hdr.id, var->hdr.name, var->ndims, diff --git a/nc_perf/tst_attsperf.c b/nc_perf/tst_attsperf.c index 98f233f72f..26a2b48440 100644 --- a/nc_perf/tst_attsperf.c +++ b/nc_perf/tst_attsperf.c @@ -137,7 +137,7 @@ readfile_hdf5(char *file_name, long long *delta, int do_inq, int num_vars) /* Open and close the root group. */ if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0) ERR; - if (H5Pset_fclose_degree(fapl_id, H5F_CLOSE_SEMI)) ERR; + if (H5Pset_fclose_degree(fapl_id, H5F_CLOSE_WEAK)) ERR; if ((hdfid = H5Fopen(file_name, H5F_ACC_RDONLY, fapl_id)) < 0) ERR; if ((hdf_grpid = H5Gopen2(hdfid, "/", H5P_DEFAULT)) < 0) ERR; diff --git a/nc_test4/CMakeLists.txt b/nc_test4/CMakeLists.txt index 980dab7fab..0da0cfb1b9 100644 --- a/nc_test4/CMakeLists.txt +++ b/nc_test4/CMakeLists.txt @@ -46,6 +46,10 @@ ENDIF(ENABLE_FILTER_TESTING) ENDIF(BUILD_UTILITIES) +IF(${HDF5_VERSION} VERSION_GREATER "1.10.0") + SET(NC4_TESTS ${NC4_TESTS} tst_virtual_datasets) +ENDIF(${HDF5_VERSION} VERSION_GREATER "1.10.0") + ## # The shell script, run_empty_vlen_test.sh, # depends on the 'tst_empty_vlen_unlim' binary. diff --git a/nc_test4/tst_h_vl2.c b/nc_test4/tst_h_vl2.c index 8f44e9a0f5..1f2b5cd3ef 100644 --- a/nc_test4/tst_h_vl2.c +++ b/nc_test4/tst_h_vl2.c @@ -36,7 +36,7 @@ main() /* Open the file and read the vlen data. */ if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0) ERR; - if (H5Pset_fclose_degree(fapl_id, H5F_CLOSE_SEMI)) ERR; + if (H5Pset_fclose_degree(fapl_id, H5F_CLOSE_WEAK)) ERR; if (H5Pset_libver_bounds(fapl_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) < 0) ERR; if ((fileid = H5Fopen(FILE_NAME, H5F_ACC_RDONLY, fapl_id)) < 0) ERR; diff --git a/nc_test4/tst_interops.c b/nc_test4/tst_interops.c index 64a0319472..74353d34be 100644 --- a/nc_test4/tst_interops.c +++ b/nc_test4/tst_interops.c @@ -390,7 +390,7 @@ main(int argc, char **argv) /* Open the file with HDF5 while netcdf still has it open. */ if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0) ERR; /* Turn this off for*/ - if (H5Pset_fclose_degree(fapl_id, H5F_CLOSE_SEMI)) ERR; + if (H5Pset_fclose_degree(fapl_id, H5F_CLOSE_WEAK)) ERR; if ((fileid = H5Fopen(FILE_NAME, H5F_ACC_RDONLY, fapl_id)) < 0) ERR; if (H5Pclose(fapl_id) < 0) ERR; if (H5Fclose(fileid) < 0) ERR; diff --git a/nc_test4/tst_interops5.c b/nc_test4/tst_interops5.c index cb4dd0e623..bb22fd0a27 100644 --- a/nc_test4/tst_interops5.c +++ b/nc_test4/tst_interops5.c @@ -50,10 +50,10 @@ main(int argc, char **argv) yscaleDims[0] = ncolCur; if ((xdimSpaceId = H5Screate_simple(1, xscaleDims, NULL)) < 0) ERR; - /* With the SEMI close degree, the HDF5 file close will fail if + /* With the WEAK close degree, the HDF5 file close will not fail if * anything is left open. */ if ((fapl = H5Pcreate(H5P_FILE_ACCESS)) < 0) ERR; - if (H5Pset_fclose_degree(fapl, H5F_CLOSE_SEMI)) ERR; + if (H5Pset_fclose_degree(fapl, H5F_CLOSE_WEAK)) ERR; /* Create file */ if((fileId = H5Fcreate(FILE_NAME, H5F_ACC_TRUNC, diff --git a/nc_test4/tst_virtual_datasets.c b/nc_test4/tst_virtual_datasets.c new file mode 100644 index 0000000000..c27455713e --- /dev/null +++ b/nc_test4/tst_virtual_datasets.c @@ -0,0 +1,87 @@ +#include +#include +#include +#include "err_macros.h" + +#define FILE_NAME_A "tst_virtual_a.nc" +#define FILE_NAME_B "tst_virtual_b.nc" +#define VARIABLE_SIZE 5 +#define VARIABLE_NAME "v" + +static void +create_dataset_a() { + hsize_t dims[1] = {VARIABLE_SIZE}; + float data[VARIABLE_SIZE]; + for(size_t i = 0; i < VARIABLE_SIZE; ++i) { + data[i] = (float)i; + } + + hid_t file = H5Fcreate(FILE_NAME_A, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); + hid_t space = H5Screate_simple(1, dims, NULL); + hid_t dset = H5Dcreate2(file, VARIABLE_NAME, H5T_IEEE_F32LE, space, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + + H5Dwrite(dset, H5T_IEEE_F32LE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data); + H5DSset_scale(dset, VARIABLE_NAME); + + H5Dclose(dset); + H5Sclose(space); + H5Fclose(file); +} + +static void +create_dataset_b() { + hsize_t dims[1] = {VARIABLE_SIZE}; + + hid_t file = H5Fcreate(FILE_NAME_B, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); + hid_t src_space = H5Screate_simple(1, dims, NULL); + hid_t space = H5Screate_simple(1, dims, NULL); + + hid_t dcpl = H5Pcreate(H5P_DATASET_CREATE); + H5Pset_virtual(dcpl, space, FILE_NAME_A, VARIABLE_NAME, src_space); + hid_t dset = H5Dcreate2(file, VARIABLE_NAME, H5T_IEEE_F32LE, space, H5P_DEFAULT, dcpl, H5P_DEFAULT); + + H5DSset_scale(dset, VARIABLE_NAME); + + H5Dclose(dset); + H5Pclose(dcpl); + H5Sclose(space); + H5Sclose(src_space); + H5Fclose(file); +} + +int read_back_contents(const char* filename) { + int ncid, varid, ndims; + float data[VARIABLE_SIZE]; + + char var_name[NC_MAX_NAME+1]; + int dimids_var[1], var_type, natts; + + printf("\treading back %s.\n", filename); + + if (nc_open(filename, 0, &ncid)) ERR; + if (nc_inq_varid(ncid, VARIABLE_NAME, &varid)) ERR; + if (nc_inq_var(ncid, varid, var_name, &var_type, &ndims, dimids_var, &natts)) ERR; + if (nc_get_var_float(ncid, varid, data)) ERR; + printf("\t %s: ", var_name); + for (size_t i = 0; i < VARIABLE_SIZE; ++i) { + printf("%f, ", data[i]); + } + printf("\n"); + if (nc_close(ncid)) ERR; + SUMMARIZE_ERR; + return 0; +} + +int +main(int argc, char **argv) +{ + + printf("\n*** Testing reading of virtual datasets.\n"); + + create_dataset_a(); + create_dataset_b(); + + int status; + if((status = read_back_contents(FILE_NAME_A))) return status; + if((status = read_back_contents(FILE_NAME_B))) return status; +} diff --git a/ncdump/ncdump.c b/ncdump/ncdump.c index e7ac9acf82..4b2231e3c9 100644 --- a/ncdump/ncdump.c +++ b/ncdump/ncdump.c @@ -988,7 +988,7 @@ pr_att_specials( } else if(contig == NC_COMPACT) { pr_att_name(ncid, varp->name, NC_ATT_STORAGE); printf(" = \"compact\" ;\n"); - } else { + } else if(contig == NC_CHUNKED) { size_t *chunkp; int i; pr_att_name(ncid, varp->name, NC_ATT_STORAGE); @@ -1002,6 +1002,12 @@ pr_att_specials( printf("%lu%s", (unsigned long)chunkp[i], i+1 < varp->ndims ? ", " : " ;\n"); } free(chunkp); + } else if(contig == NC_VIRTUAL) { + pr_att_name(ncid, varp->name, NC_ATT_STORAGE); + printf(" = \"virtual\" ;\n"); + } else { + pr_att_name(ncid, varp->name, NC_ATT_STORAGE); + printf(" = \"unknown\" ;\n"); } /* _Filter (including deflate and shuffle) */