From 84c69afca7cd4493905f208cfaeaa2224816325f Mon Sep 17 00:00:00 2001 From: Dennis Heimbigner Date: Mon, 11 May 2020 09:42:31 -0600 Subject: [PATCH] Allow redefinition of variable filters re: Github issue https://github.com/Unidata/netcdf-c/issues/1713 If nc_def_var_filter or nc_def_var_deflate or nc_def_var_szip is called multiple times with the same filter id, but possibly with different sets of parameters, then the first invocation is sticky and later invocations are ignored. The desired behavior is to have the last invocation be used. This PR implements that desired behavior, with some special cases. If you call nc_def_var_deflate multiple times, then the last invocation rule applies with respect to deflate. However, the shuffle filter, if enabled, is always applied just before applying deflate. Misc unrelated changes: 1. Make client-side filters be disabled by default 2. Fix the definition of uintptr_t and use in oc2 and libdap4 3. Add some test cases 4. modify filter order tests to use plugin filters rather than client-side filters --- CMakeLists.txt | 8 +- NUG/filters.md | 28 +++ RELEASE_NOTES.md | 3 + configure.ac | 14 ++ include/hdf5internal.h | 2 +- include/ncconfigure.h | 7 + libdap4/d4curlfunctions.c | 2 +- libdispatch/dfile.c | 8 +- libdispatch/dfilter.c | 2 + libhdf5/hdf5filter.c | 123 ++++++----- libhdf5/hdf5open.c | 46 ++--- libhdf5/nc4hdf.c | 10 +- libhdf5/nc4info.c | 3 +- nc_test4/CMakeLists.txt | 7 +- nc_test4/Makefile.am | 16 +- nc_test4/ref_filter_order.txt | 16 +- nc_test4/ref_filter_repeat.txt | 8 + nc_test4/test_filter_order.c | 280 ++----------------------- nc_test4/test_filter_repeat.c | 368 +++++++++++++++++++++++++++++++++ nc_test4/tst_filter.sh | 13 +- ncdump/Makefile.am | 3 - oc2/occurlfunctions.c | 7 +- plugins/CMakeLists.txt | 8 + plugins/H5Znoop.c | 39 +++- plugins/Makefile.am | 19 +- plugins/h5noop.h | 9 - 26 files changed, 651 insertions(+), 398 deletions(-) create mode 100644 nc_test4/ref_filter_repeat.txt create mode 100644 nc_test4/test_filter_repeat.c diff --git a/CMakeLists.txt b/CMakeLists.txt index f99d5984ea..fd523c68d3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1182,6 +1182,11 @@ IF(NOT BUILD_SHARED_LIBS) MESSAGE(WARNING "ENABLE_FILTER_TESTING requires shared libraries. Disabling.") SET(ENABLE_FILTER_TESTING OFF) ENDIF() +OPTION(ENABLE_CLIENTSIDE_FILTERS "Enable client-side filter registration." OFF) +IF(NOT ENABLE_FILTER_TESTING) +SET(ENABLE_CLIENTSIDE_FILTERS OFF) +ENDIF() + # Determine whether or not to generate documentation. OPTION(ENABLE_DOXYGEN "Enable generation of doxygen-based documentation." OFF) @@ -1776,6 +1781,7 @@ ENDIF() add_subdirectory(liblib) IF(ENABLE_FILTER_TESTING) + CONFIGURE_FILE(plugins/H5Znoop.c ${CMAKE_SOURCE_DIR}/plugins/H5Znoop1.c COPYONLY) add_subdirectory(plugins) ENDIF() @@ -2110,8 +2116,6 @@ install( # End export files #### - - # CPack inclusion must come last. # INCLUDE(CPack) INCLUDE(CMakeInstallation.cmake) diff --git a/NUG/filters.md b/NUG/filters.md index a15159f733..292896254f 100644 --- a/NUG/filters.md +++ b/NUG/filters.md @@ -482,6 +482,34 @@ demonstrate how to build the hdf5 plugin for bzip2. Notes ========== +Order of Invocation for Multiple Filters +----------- + +When multiple filters are defined on a variable, the +order of application, when writing data to the file, +is same as the order in which _nc_def_var_filter_ is called. +When reading a file the order of application is of necessity +the reverse. + +There are some special cases. + +1. The fletcher32 filter is always applied first, if enabled. +1. If _nc_def_var_filter_ or _nc_def_var_deflate_ or _nc_def_var_szip_ +is called multiple times with the same filter id, but possibly +with different sets of parameters, then the position of that filter +in the sequence of applictions does not change. However the last set +of parameters specified is used when actually writing the dataset. +1. Deflate and shuffle -- these two are inextricably linked in the +current API, but have quite different semantics. +If you call _nc_def_var_deflate_ multiple times, then +the previous rule applies with respect to deflate. However, +the shuffle filter, if enabled, is ''always'' applied before +applying any other filters, except fletcher32. +1. If you want to move the location of a filter in the application +sequence, then you must remove it using _nc_var_filter_remove_ +or using _nc_def_var_deflate_. The next time you add the filter +back, its position will be at the end of the current sequence. + Memory Allocation Issues ----------- diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 4db17c917c..b72aad356e 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -7,6 +7,9 @@ This file contains a high-level description of this package's evolution. Release ## 4.8.0 - TBD +* [Enhancement] When a filter is applied twice with different +parameters, then the second set is used for writing the dataset +[https://github.com/Unidata/netcdf-c/issues/1713]. * [Bug Fix] Now larger cache settings are used for sequential HDF5 file creates/opens on parallel I/O capable builds; see [Github #1716](https://github.com/Unidata/netcdf-c/issues/1716) for more information. * [Bug Fix] Add functions to libdispatch/dnotnc4.c to support dispatch table operations that should work for any dispatch diff --git a/configure.ac b/configure.ac index fddc67beba..1fe0e66f88 100644 --- a/configure.ac +++ b/configure.ac @@ -1428,6 +1428,20 @@ enable_filter_testing=no fi AM_CONDITIONAL(ENABLE_FILTER_TESTING, [test x$enable_filter_testing = xyes]) +# Enable client side filter registration +AC_MSG_CHECKING([If client-side filters are enabled (default off)]) +AC_ARG_ENABLE([clientside-filters], + [AS_HELP_STRING([--enable-clientside-filters], + [enable client side filters])], + [], + [enable_clientside_filters=no]) +test "x$enable_clientside_filters" = xyes || enable_clientside_filters=no +AC_MSG_RESULT($enable_clientside_filters) +if test "x$enable_clientside_filters" = xyes ; then +AC_DEFINE([ENABLE_CLIENTSIDE_FILTERS], [1], [if true, enable client-side filters]) +fi +AM_CONDITIONAL(ENABLE_CLIENTSIDE_FILTERS, [test x$enable_clientside_filters = xyes]) + AC_SUBST(NC_LIBS,[$NC_LIBS]) AC_SUBST(HAS_DAP,[$enable_dap]) AC_SUBST(HAS_DAP2,[$enable_dap]) diff --git a/include/hdf5internal.h b/include/hdf5internal.h index 95d2f9bdf3..7e51345fb8 100644 --- a/include/hdf5internal.h +++ b/include/hdf5internal.h @@ -183,7 +183,7 @@ int nc4_hdf5_get_chunk_cache(int ncid, size_t *sizep, size_t *nelemsp, /* Define Filter API Function */ int nc4_global_filter_action(int action, unsigned int id, struct NC_FILTER_OBJ_HDF5* infop); -int NC4_hdf5_addfilter(NC_VAR_INFO_T* var, int active, unsigned int id, size_t nparams, unsigned int* params); +int NC4_hdf5_addfilter(NC_VAR_INFO_T* var, int active, unsigned int id, size_t nparams, unsigned int* params, struct NC_FILTER_SPEC_HDF5**); int NC4_hdf5_remove_filter(NC_VAR_INFO_T* var, unsigned int filterid); /* Support functions for provenance info (defined in nc4hdf.c) */ diff --git a/include/ncconfigure.h b/include/ncconfigure.h index bf6b787497..48a799acb1 100644 --- a/include/ncconfigure.h +++ b/include/ncconfigure.h @@ -104,6 +104,13 @@ typedef unsigned short ushort; typedef unsigned int uint; #endif +#ifndef HAVE_UINTPTR_T +#if SIZEOF_VOIDP == 8 +typedef unsigned long uintptr_t; +#else +typedef unsigned int uintptr_t; +#endif +#endif /* Provide a fixed size alternative to off_t or off64_t */ typedef long long fileoffset_t; diff --git a/libdap4/d4curlfunctions.c b/libdap4/d4curlfunctions.c index 948153d114..e8eb0037c6 100644 --- a/libdap4/d4curlfunctions.c +++ b/libdap4/d4curlfunctions.c @@ -9,7 +9,7 @@ #define MAX_REDIRECTS 20L /* Mnemonic */ -#define OPTARG void* +#define OPTARG uintptr_t /* Condition on libcurl version */ /* Set up an alias as needed */ diff --git a/libdispatch/dfile.c b/libdispatch/dfile.c index 09e93ec63d..f3b2bc097d 100644 --- a/libdispatch/dfile.c +++ b/libdispatch/dfile.c @@ -1855,9 +1855,9 @@ NC_create(const char *path0, int cmode, size_t initialsz, for(p=(const unsigned char*)path0;*p;p++) {if(*p > ' ') break;} #ifdef WINPATH /* Need to do path conversion */ - path = NCpathcvt(p); + path = NCpathcvt((const char*)p); #else - path = nulldup(p); + path = nulldup((const char*)p); #endif } @@ -1999,8 +1999,8 @@ NC_open(const char *path0, int omode, int basepe, size_t *chunksizehintp, { /* Skip past any leading whitespace in path */ - const unsigned char* p; - for(p=(const unsigned char*)path0;*p;p++) {if(*p > ' ') break;} + const char* p; + for(p=(const char*)path0;*p;p++) {if(*p < 0 || *p > ' ') break;} #ifdef WINPATH /* Need to do path conversion */ path = NCpathcvt(p); diff --git a/libdispatch/dfilter.c b/libdispatch/dfilter.c index 0e490a3d69..b8ea17cd5d 100644 --- a/libdispatch/dfilter.c +++ b/libdispatch/dfilter.c @@ -381,6 +381,7 @@ NC4_filterfix8(unsigned char* mem, int decode) } +#ifdef ENABLE_CLIENTSIDE_FILTERS /* Support direct user defined filters */ /* Use void* to avoid having to include hdf.h*/ @@ -442,6 +443,7 @@ nc_filter_client_inq(unsigned int id, void* infop) #endif return stat; } +#endif /*ENABLE_CLIENTSIDE_FILTERS*/ /** Find the set of filters (if any) associated with a variable. diff --git a/libhdf5/hdf5filter.c b/libhdf5/hdf5filter.c index 963ef18c3f..ecd8c9bda3 100644 --- a/libhdf5/hdf5filter.c +++ b/libhdf5/hdf5filter.c @@ -23,23 +23,26 @@ #define FILTERACTIVE 1 -/* WARNING: GLOBAL VARIABLE */ +/**************************************************/ +/* Filter registration support */ +#ifdef ENABLE_CLIENTSIZE_FILTERS + +/* WARNING: GLOBAL VARIABLE */ /* Define list of registered filters */ static NClist* NC4_registeredfilters = NULL; /** List */ -/**************************************************/ -/* Filter registration support */ - static int -filterlookup(unsigned int id) +clientfilterlookup(unsigned int id) { int i; if(NC4_registeredfilters == NULL) NC4_registeredfilters = nclistnew(); for(i=0;iid == id) return i; /* return position */ + if(x != NULL && x->id == id) { + return i; /* return position */ + } } return -1; } @@ -77,36 +80,6 @@ dupfilterinfo(NC_FILTER_CLIENT_HDF5* info) return NULL; } -int -NC4_hdf5_addfilter(NC_VAR_INFO_T* var, int active, unsigned int id, size_t nparams, unsigned int* inparams) -{ - int stat = NC_NOERR; - NC_FILTER_SPEC_HDF5* fi = NULL; - unsigned int* params = NULL; - - if(var->filters == NULL) { - if((var->filters = nclistnew())==NULL) return THROW(NC_ENOMEM); - } - - if(nparams > 0 && inparams == NULL) - return THROW(NC_EINVAL); - if(inparams != NULL) { - if((params = malloc(sizeof(unsigned int)*nparams)) == NULL) - return THROW(NC_ENOMEM); - memcpy(params,inparams,sizeof(unsigned int)*nparams); - } - - if((fi = calloc(1,sizeof(NC_FILTER_SPEC_HDF5))) == NULL) - {nullfree(params); return THROW(NC_ENOMEM);} - - fi->active = active; - fi->filterid = id; - fi->nparams = nparams; - fi->params = params; - nclistpush(var->filters,fi); - return THROW(stat); -} - int nc4_global_filter_action(int op, unsigned int id, NC_FILTER_OBJ_HDF5* infop) { @@ -129,7 +102,7 @@ nc4_global_filter_action(int op, unsigned int id, NC_FILTER_OBJ_HDF5* infop) if(id != h5filterinfo->id) {stat = NC_EINVAL; goto done;} /* See if this filter is already defined */ - if((pos = filterlookup(id)) >= 0) + if((pos = clientfilterlookup(id)) >= 0) {stat = NC_ENAMEINUSE; goto done;} /* Already defined */ if((herr = H5Zregister(h5filterinfo)) < 0) {stat = NC_EFILTER; goto done;} @@ -144,7 +117,7 @@ nc4_global_filter_action(int op, unsigned int id, NC_FILTER_OBJ_HDF5* infop) if(id <= 0) {stat = NC_ENOTNC4; goto done;} /* See if this filter is already defined */ - if((pos = filterlookup(id)) < 0) + if((pos = clientfilterlookup(id)) < 0) {stat = NC_ENOFILTER; goto done;} /* Not defined */ if((herr = H5Zunregister(id)) < 0) {stat = NC_EFILTER; goto done;} @@ -153,7 +126,7 @@ nc4_global_filter_action(int op, unsigned int id, NC_FILTER_OBJ_HDF5* infop) case NCFILTER_CLIENT_INQ: if(infop == NULL) goto done; /* Look up the id in our local table */ - if((pos = filterlookup(id)) < 0) + if((pos = clientfilterlookup(id)) < 0) {stat = NC_ENOFILTER; goto done;} /* Not defined */ elem = (NC_FILTER_CLIENT_HDF5*)nclistget(NC4_registeredfilters,pos); if(elem == NULL) {stat = NC_EINTERNAL; goto done;} @@ -168,6 +141,61 @@ nc4_global_filter_action(int op, unsigned int id, NC_FILTER_OBJ_HDF5* infop) return THROW(stat); } +#endif /*ENABLE_CLIENTSIDE_FILTERS*/ + +/**************************************************/ +static int +filterlookup(NC_VAR_INFO_T* var, unsigned int id, NC_FILTER_SPEC_HDF5** fp) +{ + int i; + if(var->filters == NULL) + var->filters = nclistnew(); + for(i=0;ifilters);i++) { + NC_FILTER_SPEC_HDF5* x = nclistget(var->filters,i); + if(x != NULL && x->filterid == id) { + if(fp) *fp = x; + return i; /* return position */ + } + } + return -1; +} + +int +NC4_hdf5_addfilter(NC_VAR_INFO_T* var, int active, unsigned int id, size_t nparams, + unsigned int* inparams, NC_FILTER_SPEC_HDF5** filtspecp) +{ + int stat = NC_NOERR; + NC_FILTER_SPEC_HDF5* fi = NULL; + unsigned int* params = NULL; + int pos; + + if(var->filters == NULL) { + if((var->filters = nclistnew())==NULL) return THROW(NC_ENOMEM); + } + + if(nparams > 0 && inparams == NULL) + return THROW(NC_EINVAL); + if(inparams != NULL) { + if((params = malloc(sizeof(unsigned int)*nparams)) == NULL) + return THROW(NC_ENOMEM); + memcpy(params,inparams,sizeof(unsigned int)*nparams); + } + if((pos=filterlookup(var, id,&fi)) < 0) { + if((fi = calloc(1,sizeof(NC_FILTER_SPEC_HDF5))) == NULL) + {nullfree(params); return THROW(NC_ENOMEM);} + } + assert(fi != NULL); + fi->active = active; + fi->filterid = id; + fi->nparams = nparams; + if(fi->params) free(fi->params); + fi->params = params; + if(filtspecp) *filtspecp = fi; + if(pos < 0) nclistpush(var->filters,fi); + fi = NULL; + return THROW(stat); +} + /** * @internal Define filter settings. Called by nc_def_var_filter(). * @@ -238,7 +266,6 @@ NC4_filter_actions(int ncid, int varid, int op, NC_Filterobject* args) params = obj->u.spec.params; #ifdef HAVE_H5_DEFLATE if(id == H5Z_FILTER_DEFLATE) { - int k; int level; if(nparams != 1) return THROW(NC_EFILTER); /* incorrect no. of parameters */ @@ -247,11 +274,8 @@ NC4_filter_actions(int ncid, int varid, int op, NC_Filterobject* args) level > NC_MAX_DEFLATE_LEVEL) return THROW(NC_EINVAL); /* If szip compression is already applied, return error. */ - for(k=0;kfilters);k++) { - NC_FILTER_SPEC_HDF5* f = nclistget(var->filters,k); - if (f->filterid == H5Z_FILTER_SZIP) + if((filterlookup(var,H5Z_FILTER_SZIP,NULL)) >= 0) return THROW(NC_EINVAL); - } } #else /*!HAVE_H5_DEFLATE*/ if(id == H5Z_FILTER_DEFLATE) @@ -259,18 +283,14 @@ NC4_filter_actions(int ncid, int varid, int op, NC_Filterobject* args) #endif #ifdef HAVE_H5Z_SZIP if(id == H5Z_FILTER_SZIP) { /* Do error checking */ - int k; if(nparams != 2) return THROW(NC_EFILTER); /* incorrect no. of parameters */ /* Pixels per block must be an even number, < 32. */ if (params[1] % 2 || params[1] > NC_MAX_PIXELS_PER_BLOCK) return THROW(NC_EINVAL); /* If zlib compression is already applied, return error. */ - for(k=0;kfilters);k++) { - NC_FILTER_SPEC_HDF5* f = nclistget(var->filters,k); - if (f->filterid == H5Z_FILTER_DEFLATE) + if((filterlookup(var,H5Z_FILTER_DEFLATE,NULL)) >= 0) return THROW(NC_EINVAL); - } } #else /*!HAVE_H5Z_SZIP*/ if(id == H5Z_FILTER_SZIP) @@ -300,8 +320,9 @@ NC4_filter_actions(int ncid, int varid, int op, NC_Filterobject* args) return THROW(NC_EINVAL); } #endif - if((stat = NC4_hdf5_addfilter(var,!FILTERACTIVE,id,nparams,params))) - goto done; + if((stat = NC4_hdf5_addfilter(var,!FILTERACTIVE,id,nparams,params, NULL))) + goto done; + #ifdef USE_PARALLEL #ifdef HDF5_SUPPORTS_PAR_FILTERS /* Switch to collective access. HDF5 requires collevtive access diff --git a/libhdf5/hdf5open.c b/libhdf5/hdf5open.c index ab4c84e754..00e6f390a7 100644 --- a/libhdf5/hdf5open.c +++ b/libhdf5/hdf5open.c @@ -963,8 +963,8 @@ static int get_filter_info(hid_t propid, NC_VAR_INFO_T *var) { H5Z_filter_t filter; int num_filters; - unsigned int cd_values_zip[CD_NELEMS_ZLIB]; - size_t cd_nelems = CD_NELEMS_ZLIB; + unsigned int* cd_values = NULL; + size_t cd_nelems; int f; int stat = NC_NOERR; @@ -975,8 +975,12 @@ static int get_filter_info(hid_t propid, NC_VAR_INFO_T *var) for (f = 0; f < num_filters; f++) { - if ((filter = H5Pget_filter2(propid, f, NULL, &cd_nelems, cd_values_zip, - 0, NULL, NULL)) < 0) + cd_nelems = 0; + if ((filter = H5Pget_filter2(propid, f, NULL, &cd_nelems, NULL, 0, NULL, NULL)) < 0) + return NC_EHDFERR; + if((cd_values = calloc(sizeof(unsigned int),cd_nelems))==NULL) + return NC_ENOMEM; + if ((filter = H5Pget_filter2(propid, f, NULL, &cd_nelems, cd_values, 0, NULL, NULL)) < 0) return NC_EHDFERR; switch (filter) { @@ -990,36 +994,27 @@ static int get_filter_info(hid_t propid, NC_VAR_INFO_T *var) case H5Z_FILTER_DEFLATE: if (cd_nelems != CD_NELEMS_ZLIB || - cd_values_zip[0] > NC_MAX_DEFLATE_LEVEL) + cd_values[0] > NC_MAX_DEFLATE_LEVEL) return NC_EHDFERR; - if((stat = NC4_hdf5_addfilter(var,FILTERACTIVE,filter,cd_nelems,cd_values_zip))) - return stat; + if((stat = NC4_hdf5_addfilter(var,FILTERACTIVE,filter,cd_nelems,cd_values,NULL))) + return stat; break; case H5Z_FILTER_SZIP: { /* Szip is tricky because the filter code expands the set of parameters from 2 to 4 and changes some of the parameter values; try to compensate */ if(cd_nelems == 0) { - if((stat = NC4_hdf5_addfilter(var,FILTERACTIVE,filter,0,NULL))) + if((stat = NC4_hdf5_addfilter(var,FILTERACTIVE,filter,0,NULL,NULL))) return stat; } else { - /* We have to re-read the parameters based on actual nparams, - which in the case of szip, differs from users original nparams */ - unsigned int* realparams = (unsigned int*)calloc(1,sizeof(unsigned int)*cd_nelems); - if(realparams == NULL) - return NC_ENOMEM; - if((filter = H5Pget_filter2(propid, f, NULL, &cd_nelems, - realparams, 0, NULL, NULL)) < 0) - return NC_EHDFERR; /* fix up the parameters and the #params */ if(cd_nelems != 4) return NC_EHDFERR; cd_nelems = 2; /* ignore last two */ /* Fix up changed params */ - realparams[0] &= (H5_SZIP_ALL_MASKS); + cd_values[0] &= (H5_SZIP_ALL_MASKS); /* Save info */ - stat = NC4_hdf5_addfilter(var,FILTERACTIVE,filter,cd_nelems,realparams); - nullfree(realparams); + stat = NC4_hdf5_addfilter(var,FILTERACTIVE,filter,cd_nelems,cd_values,NULL); if(stat) return stat; } @@ -1027,21 +1022,14 @@ static int get_filter_info(hid_t propid, NC_VAR_INFO_T *var) default: if(cd_nelems == 0) { - if((stat = NC4_hdf5_addfilter(var,FILTERACTIVE,filter,0,NULL))) return stat; + if((stat = NC4_hdf5_addfilter(var,FILTERACTIVE,filter,0,NULL,NULL))) return stat; } else { - /* We have to re-read the parameters based on actual nparams */ - unsigned int* realparams = (unsigned int*)calloc(1,sizeof(unsigned int)*cd_nelems); - if(realparams == NULL) - return NC_ENOMEM; - if((filter = H5Pget_filter2(propid, f, NULL, &cd_nelems, - realparams, 0, NULL, NULL)) < 0) - return NC_EHDFERR; - stat = NC4_hdf5_addfilter(var,FILTERACTIVE,filter,cd_nelems,realparams); - nullfree(realparams); + stat = NC4_hdf5_addfilter(var,FILTERACTIVE,filter,cd_nelems,cd_values,NULL); if(stat) return stat; } break; } + nullfree(cd_values); cd_values = NULL; } return NC_NOERR; } diff --git a/libhdf5/nc4hdf.c b/libhdf5/nc4hdf.c index 68b8ef8aca..2973f6e3ea 100644 --- a/libhdf5/nc4hdf.c +++ b/libhdf5/nc4hdf.c @@ -902,6 +902,12 @@ var_create_dataset(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var, nc_bool_t write_dimid } } + /* If the user wants to fletcher error correction, set that up now. */ + /* Since it is a checksum of sorts, flatcher is always applied first */ + if (var->fletcher32) + if (H5Pset_fletcher32(plistid) < 0) + BAIL(NC_EHDFERR); + /* If the user wants to shuffle the data, set that up now. */ if (var->shuffle) { if (H5Pset_shuffle(plistid) < 0) @@ -947,10 +953,6 @@ var_create_dataset(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var, nc_bool_t write_dimid } } - /* If the user wants to fletcher error correction, set that up now. */ - if (var->fletcher32) - if (H5Pset_fletcher32(plistid) < 0) - BAIL(NC_EHDFERR); /* If ndims non-zero, get info for all dimensions. We look up the dimids and get the len of each dimension. We need this to create diff --git a/libhdf5/nc4info.c b/libhdf5/nc4info.c index 6f5cabf239..a0cb25f2da 100644 --- a/libhdf5/nc4info.c +++ b/libhdf5/nc4info.c @@ -56,7 +56,6 @@ NC4_provenance_init(void) unsigned major,minor,release; NCbytes* buffer = NULL; /* for constructing the global _NCProperties */ char printbuf[1024]; - const char* p = NULL; if(globalpropinitialized) return stat; @@ -91,11 +90,13 @@ NC4_provenance_init(void) ncbytescat(buffer,printbuf); #ifdef NCPROPERTIES_EXTRA + { const char* p; /* Add any extra fields */ p = NCPROPERTIES_EXTRA; if(p[0] == NCPROPSSEP2) p++; /* If leading separator */ ncbytesappend(buffer,NCPROPSSEP2); ncbytescat(buffer,p); + } #endif ncbytesnull(buffer); globalprovenance.ncproperties = ncbytesextract(buffer); diff --git a/nc_test4/CMakeLists.txt b/nc_test4/CMakeLists.txt index d51ebfada1..5dd5a85ca3 100644 --- a/nc_test4/CMakeLists.txt +++ b/nc_test4/CMakeLists.txt @@ -30,13 +30,16 @@ IF(BUILD_UTILITIES) add_sh_test(nc_test4 run_grp_rename) ADD_SH_TEST(nc_test4 tst_misc) IF(ENABLE_FILTER_TESTING) + build_bin_test(tst_filterparser) build_bin_test(test_filter) build_bin_test(test_filter_misc) - build_bin_test(test_filter_reg) build_bin_test(tst_multifilter) build_bin_test(test_filter_order) + build_bin_test(test_filter_repeat) ADD_SH_TEST(nc_test4 tst_filter) - SET(NC4_TESTS ${NC4_TESTS} tst_filterparser test_filter_reg) +IF(ENABLE_CLIENTSIDE_FILTERS) + add_bin_test(nc_test4 test_filter_reg) +ENDIF(ENABLE_CLIENTSIDE_FILTERS) ENDIF(ENABLE_FILTER_TESTING) ENDIF(BUILD_UTILITIES) diff --git a/nc_test4/Makefile.am b/nc_test4/Makefile.am index f799654b1f..82287329c7 100644 --- a/nc_test4/Makefile.am +++ b/nc_test4/Makefile.am @@ -72,10 +72,13 @@ endif # Filter Tests (requires ncdump and ncgen) if ENABLE_FILTER_TESTING extradir = -extra_PROGRAMS = test_filter test_filter_misc test_filter_order -check_PROGRAMS += test_filter_reg +check_PROGRAMS += test_filter test_filter_misc test_filter_order test_filter_repeat check_PROGRAMS += tst_multifilter -TESTS += tst_filter.sh test_filter_reg +TESTS += tst_filter.sh +if ENABLE_CLIENTSIDE_FILTERS +#check_PROGRAMS += test_filter_reg +TESTS += test_filter_reg +endif endif endif # BUILD_UTILITIES @@ -95,8 +98,9 @@ run_empty_vlen_test.sh ref_hdf5_compat1.nc ref_hdf5_compat2.nc \ ref_hdf5_compat3.nc tst_misc.sh tdset.h5 tst_szip.sh ref_szip.h5 \ ref_szip.cdl tst_filter.sh bzip2.cdl ref_filtered.cdl \ ref_unfiltered.cdl ref_bzip2.c findplugin.in ref_unfilteredvv.cdl \ -ref_filteredvv.cdl ref_multi.cdl ref_filter_order.txt \ -ref_ncgenF.cdl ref_nccopyF.cdl +ref_filteredvv.cdl ref_multi.cdl \ +ref_ncgenF.cdl ref_nccopyF.cdl \ +ref_filter_order.txt ref_filter_repeat.txt CLEANFILES = tst_mpi_parallel.bin cdm_sea_soundings.nc bm_chunking.nc \ tst_floats_1D.cdl floats_1D_3.nc floats_1D.cdl tst_*.nc \ @@ -106,7 +110,7 @@ tst_*.h5 tst_grp_rename.cdl tst_grp_rename.dmp ref_grp_rename.cdl \ foo1.nc tst_*.h4 test.nc testszip.nc test.h5 szip_dump.cdl \ perftest.txt bigmeta.nc bigvars.nc *.gz MSGCPP_*.nc \ floats*.nc floats*.cdl shorts*.nc shorts*.cdl ints*.nc ints*.cdl \ -testfilter_reg.nc +testfilter_reg.nc filterorder.txt filterrepeat.txt DISTCLEANFILES = findplugin.sh run_par_test.sh diff --git a/nc_test4/ref_filter_order.txt b/nc_test4/ref_filter_order.txt index 64d8436d21..f461bb0ff9 100644 --- a/nc_test4/ref_filter_order.txt +++ b/nc_test4/ref_filter_order.txt @@ -1,17 +1,9 @@ test1: filter order. -set var filter 0 -test: nparams=1: params= 0 -dimsizes=4,4,4,4 -chunksizes=4,4,4,4 -set var filter 1 -test: nparams=1: params= 1 -dimsizes=4,4,4,4 -chunksizes=4,4,4,4 test1: compression. -Apply filter: 0 -Apply filter: 1 +cd_nelmts=1 cd_values= 0 +cd_nelmts=1 cd_values= 1 test1: decompression. -Apply filter: 1 -Apply filter: 0 +cd_nelmts=1 cd_values= 1 +cd_nelmts=1 cd_values= 0 data comparison: |array|=256 no data errors diff --git a/nc_test4/ref_filter_repeat.txt b/nc_test4/ref_filter_repeat.txt new file mode 100644 index 0000000000..181fd3c1c0 --- /dev/null +++ b/nc_test4/ref_filter_repeat.txt @@ -0,0 +1,8 @@ +test1: def filter repeat . +test1: compression. +cd_nelmts=2 cd_values= 0 18 +test1: decompression. +filter(40000): |params|=2 params= 0 18 +cd_nelmts=2 cd_values= 0 18 +data comparison: |array|=256 +no data errors diff --git a/nc_test4/test_filter_order.c b/nc_test4/test_filter_order.c index 01dc17cad1..fc2284592f 100644 --- a/nc_test4/test_filter_order.c +++ b/nc_test4/test_filter_order.c @@ -10,10 +10,6 @@ #include #include -#include -/* Older versions of the hdf library may define H5PL_type_t here */ -#include - #include "netcdf.h" #include "netcdf_filter.h" @@ -21,7 +17,7 @@ #undef DEBUG -#define FILTER_ID 32768 +#define FILTER_ID 40000 #define MAXERRS 8 @@ -52,17 +48,6 @@ static size_t odom[MAXDIMS]; static float* array = NULL; static float* expected = NULL; -static size_t nparams = 0; -static unsigned int params[MAXPARAMS]; -static unsigned int baseline[2][NPARAMS] = {{0},{1}}; - -static struct Base { - unsigned int id; - H5Z_class2_t* info; -} baseinfo; - -static const H5Z_class2_t H5Z_REG[2]; - /* Forward */ static int filter_test1(void); static void init(int argc, char** argv); @@ -72,7 +57,6 @@ static int odom_more(void); static int odom_next(void); static int odom_offset(void); static float expectedvalue(void); -static void verifyparams(int); #define ERRR do { \ fflush(stdout); /* Make sure our stdout is synced with stderr. */ \ @@ -138,87 +122,29 @@ create(void) return NC_NOERR; } -static int -verifyfilterinfo(struct Base* info, struct Base* base) -{ - int stat = NC_NOERR; - if(info->id != base->id) - {stat = NC_EINVAL; fprintf(stderr,"verifyfilterinfo: id mismatch\n");} -#ifdef USE_HDF5 - H5Z_class2_t* h5info = info->info; - H5Z_class2_t* h5base = base->info; - if(info->id != base->id) - {stat = NC_EINVAL; fprintf(stderr,"verifyfilterinfo: H5Z_class_t: id mismatch\n");} - if(h5info->encoder_present != h5base->encoder_present) - {stat = NC_EINVAL; fprintf(stderr,"verifyfilterinfo: H5Z_class_t: encoder_present mismatch\n");} - if(h5info->decoder_present != h5base->decoder_present) - {stat = NC_EINVAL; fprintf(stderr,"verifyfilterinfo: H5Z_class_t: decoder_present mismatch\n");} - if(h5info->decoder_present != h5base->decoder_present) - {stat = NC_EINVAL; fprintf(stderr,"verifyfilterinfo: H5Z_class_t: decoder_present mismatch\n");} - if(strcmp(h5info->name,h5base->name) != 0) - {stat = NC_EINVAL; fprintf(stderr,"verifyfilterinfo: H5Z_class_t: name mismatch\n");} - if(h5info->can_apply != h5base->can_apply) - {stat = NC_EINVAL; fprintf(stderr,"verifyfilterinfo: H5Z_class_t: can_apply mismatch\n");} - if(h5info->set_local != h5base->set_local) - {stat = NC_EINVAL; fprintf(stderr,"verifyfilterinfo: H5Z_class_t: set_local mismatch\n");} - if(h5info->filter != h5base->filter) - {stat = NC_EINVAL; fprintf(stderr,"verifyfilterinfo: H5Z_class_t: filter mismatch\n");} -#else - stat = NC_ENOTBUILT; fprintf(stderr,"Unknown format\n")} -#endif - return stat; -} - static void -registerfilters(void) +deffilters(void) { - struct Base inqinfo; + unsigned int params[1]; + unsigned int filterids[2]; + size_t nfilters = 0; + size_t nparams = 0; /* Register filter 0 */ - baseinfo.id = FILTER_ID; - baseinfo.info = (H5Z_class2_t*)&H5Z_REG[0]; - CHECK(nc_filter_client_register(baseinfo.id,baseinfo.info)); - /* Verify by inquiry */ - memset(&inqinfo,0,sizeof(struct Base)); - inqinfo.id = FILTER_ID; - if((inqinfo.info = (H5Z_class2_t*)calloc(1,sizeof(H5Z_class2_t))) == NULL) - CHECK(NC_ENOMEM); - CHECK((nc_filter_client_inq(inqinfo.id,(void*)inqinfo.info))); - CHECK((verifyfilterinfo(&inqinfo,&baseinfo))); - nullfree(inqinfo.info); - - /* Register filter 1 */ - baseinfo.id = FILTER_ID+1; - baseinfo.info = (H5Z_class2_t*)&H5Z_REG[1]; - CHECK(nc_filter_client_register(baseinfo.id,baseinfo.info)); - /* Verify by inquiry */ - memset(&inqinfo,0,sizeof(struct Base)); - inqinfo.id = FILTER_ID+1; - if((inqinfo.info = (H5Z_class2_t*)calloc(1,sizeof(H5Z_class2_t))) == NULL) - CHECK(NC_ENOMEM); - CHECK((nc_filter_client_inq(inqinfo.id,(void*)inqinfo.info))); - CHECK((verifyfilterinfo(&inqinfo,&baseinfo))); - nullfree(inqinfo.info); -} - -static void -setvarfilter(int index) -{ - /* NOOP the params */ - CHECK(nc_def_var_filter(ncid,varid,FILTER_ID+index,NPARAMS,baseline[index])); - verifyparams(index); -} - -static void -verifyparams(int index) -{ - int i; - CHECK(nc_inq_var_filter_info(ncid,varid,FILTER_ID+index,&nparams,params)); - if(nparams != NPARAMS) REPORT("nparams mismatch"); - for(i=0;i 0?1:0); } - -/**************************************************/ -/* In-line filter code */ - -#define H5Z_FILTER_REG0 FILTER_ID -#define H5Z_FILTER_REG1 (FILTER_ID+1) - -#ifndef DLL_EXPORT -#define DLL_EXPORT -#endif - -static int paramcheck(size_t nparams, const unsigned int* params); - -/* Forward */ -static int paramcheck(size_t nparams, const unsigned int* params); - -/* Make this explicit */ -/* - * The "can_apply" callback returns positive a valid combination, zero for an - * invalid combination and negative for an error. - */ -static htri_t -H5Z_reg_can_apply(hid_t dcpl_id, hid_t type_id, hid_t space_id) -{ - return 1; /* Assume it can always apply */ -} - -/* -As a filter, it is the identity function, -passing input to output unchanged. -*/ - -size_t -H5Z_filter_reg(unsigned int flags, size_t cd_nelmts, - const unsigned int cd_values[], size_t nbytes, - size_t *buf_size, void **buf) -{ - void* newbuf; - - if(cd_nelmts == 0) - goto fail; - - if(!paramcheck(cd_nelmts,cd_values)) - goto fail; - - printf("Apply filter: %u\n",cd_values[0]); - - if (flags & H5Z_FLAG_REVERSE) { - /* Replace buffer */ -#ifdef HAVE_H5ALLOCATE_MEMORY - newbuf = H5allocate_memory(*buf_size,0); -#else - newbuf = malloc(*buf_size); -#endif - if(newbuf == NULL) abort(); - if(*buf != NULL) { - memcpy(newbuf,*buf,*buf_size); - /* reclaim old buffer */ -#ifdef HAVE_H5FREE_MEMORY - H5free_memory(*buf); -#else - free(*buf); -#endif - } - *buf = newbuf; - - } else { - - /* Replace buffer */ -#ifdef HAVE_H5ALLOCATE_MEMORY - newbuf = H5allocate_memory(*buf_size,0); -#else - newbuf = malloc(*buf_size); -#endif - if(newbuf == NULL) abort(); - if(*buf != NULL) { - memcpy(newbuf,*buf,*buf_size); - /* reclaim old buffer */ -#ifdef HAVE_H5FREE_MEMORY - H5free_memory(*buf); -#else - free(*buf); -#endif - } - *buf = newbuf; - - } - - return *buf_size; - -fail: - return 0; -} - -static int -paramcheck(size_t nparams, const unsigned int* params) -{ - if(nparams != 1) { - fprintf(stderr,"Incorrect parameter count: need=1 sent=%ld\n",(unsigned long)nparams); - goto fail; - } - - return 1; - -fail: - return 0; -} - -static const H5Z_class2_t H5Z_REG[2] = { -{ - H5Z_CLASS_T_VERS, /* H5Z_class_t version */ - (H5Z_filter_t)(H5Z_FILTER_REG0), /* Filter id number */ - 1, /* encoder_present flag (set to true) */ - 1, /* decoder_present flag (set to true) */ - "registered0", /* Filter name for debugging */ - (H5Z_can_apply_func_t)H5Z_reg_can_apply, /* The "can apply" callback */ - NULL, /* The "set local" callback */ - (H5Z_func_t)H5Z_filter_reg, /* The actual filter function */ -}, -{ - H5Z_CLASS_T_VERS, /* H5Z_class_t version */ - (H5Z_filter_t)(H5Z_FILTER_REG1), /* Filter id number */ - 1, /* encoder_present flag (set to true) */ - 1, /* decoder_present flag (set to true) */ - "registered1", /* Filter name for debugging */ - (H5Z_can_apply_func_t)H5Z_reg_can_apply, /* The "can apply" callback */ - NULL, /* The "set local" callback */ - (H5Z_func_t)H5Z_filter_reg, /* The actual filter function */ -} -}; diff --git a/nc_test4/test_filter_repeat.c b/nc_test4/test_filter_repeat.c new file mode 100644 index 0000000000..cc0acf811b --- /dev/null +++ b/nc_test4/test_filter_repeat.c @@ -0,0 +1,368 @@ +/* + Copyright 2018, UCAR/Unidata + See COPYRIGHT file for copying and redistribution conditions. +*/ + +#include "config.h" +#include +#include +#include +#include +#include + +#include "netcdf.h" +#include "netcdf_filter.h" + +#undef TESTODDSIZE + +#undef DEBUG + +#define FILTER_ID 40000 + +#define MAXERRS 8 + +#define MAXPARAMS 32 + +#define MAXDIMS 8 + +#define TESTFILE "testfilter_reg.nc" + +#define NPARAMS 1 +#define PARAMVAL 17 + +#define NDIMS 4 +static size_t dimsize[NDIMS] = {4,4,4,4}; +static size_t chunksize[NDIMS] = {4,4,4,4}; + +static size_t ndims = NDIMS; + +static size_t totalproduct = 1; /* x-product over max dims */ +static size_t actualproduct = 1; /* x-product over actualdims */ +static size_t chunkproduct = 1; /* x-product over actual chunks */ + +static int nerrs = 0; + +static int ncid, varid; +static int dimids[MAXDIMS]; +static size_t odom[MAXDIMS]; +static float* array = NULL; +static float* expected = NULL; + +/* Forward */ +static int filter_test1(void); +static void init(int argc, char** argv); +static void reset(void); +static void odom_reset(void); +static int odom_more(void); +static int odom_next(void); +static int odom_offset(void); +static float expectedvalue(void); + +#define ERRR do { \ +fflush(stdout); /* Make sure our stdout is synced with stderr. */ \ +fprintf(stderr, "Sorry! Unexpected result, %s, line: %d\n", \ + __FILE__, __LINE__); \ +nerrs++;\ +} while (0) + +static int +check(int err,int line) +{ + if(err != NC_NOERR) { + fprintf(stderr,"fail (%d): %s\n",line,nc_strerror(err)); + } + return NC_NOERR; +} + +static void +report(const char* msg, int lineno) +{ + fprintf(stderr,"fail: line=%d %s\n",lineno,msg); + exit(1); +} + +#define CHECK(x) check(x,__LINE__) +#define REPORT(x) report(x,__LINE__) + +static int +verifychunks(void) +{ + int i; + int store = -1; + size_t localchunks[MAXDIMS]; + memset(localchunks,0,sizeof(localchunks)); + CHECK(nc_inq_var_chunking(ncid, varid, &store, localchunks)); + if(store != NC_CHUNKED) { + fprintf(stderr,"bad chunk store\n"); + return 0; + } + for(i=0;i= MAXERRS) + break; + } + } + } else + { + odom_reset(); + while(odom_more()) { + int offset = odom_offset(); + float expect = expectedvalue(); + if(array[offset] != expect) { + fprintf(stderr,"data mismatch: array[%d]=%f expected=%f\n", + offset,array[offset],expect); + errs++; + if(errs >= MAXERRS) + break; + } + odom_next(); + } + } + + if(errs == 0) + printf("no data errors\n"); + return (errs == 0); +} + +static int +filter_test1(void) +{ + int ok = 1; + unsigned int params[MAXPARAMS]; + + reset(); + + printf("test1: def filter repeat .\n"); + create(); + setchunking(); + + params[0] = 1; + params[1] = 17; + deffilter(FILTER_ID,2,params); + + params[0] = 0; + params[1] = 18; + deffilter(FILTER_ID,2,params); + + CHECK(nc_enddef(ncid)); + + /* Fill in the array */ + fill(); + + printf("test1: compression.\n"); + /* write array */ + CHECK(nc_put_var(ncid,varid,expected)); + CHECK(nc_close(ncid)); + + printf("test1: decompression.\n"); + reset(); + openfile(); + CHECK(nc_get_var_float(ncid, varid, array)); + ok = compare(); + + CHECK(nc_close(ncid)); + return ok; +} + +/**************************************************/ +/* Utilities */ + +static void +reset() +{ + memset(array,0,sizeof(float)*actualproduct); +} + +static void +odom_reset(void) +{ + memset(odom,0,sizeof(odom)); +} + +static int +odom_more(void) +{ + return (odom[0] < dimsize[0]); +} + +static int +odom_next(void) +{ + int i; /* do not make unsigned */ + for(i=ndims-1;i>=0;i--) { + odom[i] += 1; + if(odom[i] < dimsize[i]) break; + if(i == 0) return 0; /* leave the 0th entry if it overflows*/ + odom[i] = 0; /* reset this position*/ + } + return 1; +} + +static int +odom_offset(void) +{ + int i; + int offset = 0; + for(i=0;i 0?1:0); +} diff --git a/nc_test4/tst_filter.sh b/nc_test4/tst_filter.sh index a7f213e3e4..e042dae1ff 100755 --- a/nc_test4/tst_filter.sh +++ b/nc_test4/tst_filter.sh @@ -12,6 +12,7 @@ NGC=1 MISC=1 MULTI=1 ORDER=1 +REP=1 # Load the findplugins function . ${builddir}/findplugin.sh @@ -219,13 +220,23 @@ diff -b -w ${srcdir}/ref_nccopyF.cdl ./ncgenFs.cdl echo "*** Pass: multiple filters" fi -if test "x$MULTI" = x1 ; then +if test "x$ORDER" = x1 ; then echo "*** Testing multiple filter order of invocation" rm -f filterorder.txt ${execdir}/test_filter_order >filterorder.txt diff -b -w ${srcdir}/ref_filter_order.txt filterorder.txt fi +if test "x$REP" = x1 ; then +echo "*** Testing filter re-definition invocation" +rm -f filterrepeat.txt +${execdir}/test_filter_repeat >filterrepeat.txt +pwd +ls -l *.txt +ls -l ${srcdir}/*.txt +diff -b -w ${srcdir}/ref_filter_repeat.txt filterrepeat.txt +fi + echo "*** Pass: all selected tests passed" #cleanup diff --git a/ncdump/Makefile.am b/ncdump/Makefile.am index 3e6b96d99b..3d1d0a2bad 100644 --- a/ncdump/Makefile.am +++ b/ncdump/Makefile.am @@ -50,9 +50,6 @@ noinst_PROGRAMS += ocprint ocprint_SOURCES = ocprint.c endif -noinst_PROGRAMS += ncdumpchunks -ncdumpchunks_SOURCES = ncdumpchunks.c - # This is the man page. man_MANS = ncdump.1 nccopy.1 diff --git a/oc2/occurlfunctions.c b/oc2/occurlfunctions.c index 3fb3c603ed..6f53d5cd90 100644 --- a/oc2/occurlfunctions.c +++ b/oc2/occurlfunctions.c @@ -2,6 +2,11 @@ See the COPYRIGHT file for more information. */ #include "config.h" +#include +#ifdef HAVE_STDINT_H +#include +#endif + #include "ncrc.h" #include "ocinternal.h" #include "ocdebug.h" @@ -10,7 +15,7 @@ #define OC_MAX_REDIRECTS 20L /* Mnemonic */ -#define OPTARG void* +#define OPTARG uintptr_t /* Define some .rc file entries of interest*/ #define NETRCFILETAG "HTTP.NETRC" diff --git a/plugins/CMakeLists.txt b/plugins/CMakeLists.txt index e2940fa97e..1ef40280d3 100644 --- a/plugins/CMakeLists.txt +++ b/plugins/CMakeLists.txt @@ -11,6 +11,7 @@ SET(libh5bzip2_SOURCES blocksort.c huffman.c crctable.c randtable.c compress.c d SET(libmisc_SOURCES H5Zmisc.c H5Zutil.c h5misc.h) SET(libnoop_SOURCES H5Znoop.c H5Zutil.c h5noop.h) +SET(libnoop1_SOURCES H5Znoop1.c H5Zutil.c h5noop.h) IF(ENABLE_FILTER_TESTING) IF(BUILD_UTILITIES) @@ -38,12 +39,19 @@ SET_TARGET_PROPERTIES(misc PROPERTIES ARCHIVE_OUTPUT_NAME "misc") SET_TARGET_PROPERTIES(misc PROPERTIES RUNTIME_OUTPUT_NAME "misc") TARGET_LINK_LIBRARIES(misc ${ALL_TLL_LIBS}) +SET_SOURCE_FILES_PROPERTIES(H5Znoop.c PROPERTIES COMPILE_OPTIONS -DNOOP_INSTANCE=0) ADD_LIBRARY(noop MODULE ${libnoop_SOURCES}) SET_TARGET_PROPERTIES(noop PROPERTIES LIBRARY_OUTPUT_NAME "noop") SET_TARGET_PROPERTIES(noop PROPERTIES ARCHIVE_OUTPUT_NAME "noop") SET_TARGET_PROPERTIES(noop PROPERTIES RUNTIME_OUTPUT_NAME "noop") TARGET_LINK_LIBRARIES(noop ${ALL_TLL_LIBS}) +SET_SOURCE_FILES_PROPERTIES(H5Znoop1.c PROPERTIES COMPILE_OPTIONS -DNOOP_INSTANCE=1) +ADD_LIBRARY(noop1 MODULE ${libnoop1_SOURCES}) +SET_TARGET_PROPERTIES(noop1 PROPERTIES LIBRARY_OUTPUT_NAME "noop1") +SET_TARGET_PROPERTIES(noop1 PROPERTIES ARCHIVE_OUTPUT_NAME "noop1") +SET_TARGET_PROPERTIES(noop1 PROPERTIES RUNTIME_OUTPUT_NAME "noop1") +TARGET_LINK_LIBRARIES(noop1 ${ALL_TLL_LIBS}) ENDIF(BUILD_UTILITIES) ENDIF(ENABLE_FILTER_TESTING) diff --git a/plugins/H5Znoop.c b/plugins/H5Znoop.c index 3b357e8360..66aea8be06 100644 --- a/plugins/H5Znoop.c +++ b/plugins/H5Znoop.c @@ -7,12 +7,22 @@ #include /* Older versions of the hdf library may define H5PL_type_t here */ #include -#include "h5noop.h" + #ifndef DLL_EXPORT #define DLL_EXPORT #endif +#if NOOP_INSTANCE == 1 +const static int instance1 = 1; +#endif +#if NOOP_INSTANCE == 0 +const static int instance0 = 1; +#endif + +/* use a temporary */ +#define H5Z_FILTER_NOOP 40000 + /* WARNING: Starting with HDF5 version 1.10.x, the plugin code MUST be careful when using the standard *malloc()*, *realloc()*, and @@ -33,12 +43,23 @@ will generate an error. extern void NC_filterfix8(void* mem, int decode); -const H5Z_class2_t H5Z_NOOP[1] = {{ +static htri_t H5Z_noop_can_apply(hid_t dcpl_id, hid_t type_id, hid_t space_id); +static size_t H5Z_filter_noop(unsigned int, size_t, const unsigned int cd_values[], size_t, size_t*, void**); + +static H5Z_class2_t H5Z_NOOP[1] = {{ H5Z_CLASS_T_VERS, /* H5Z_class_t version */ +#if NOOP_INSTANCE == 0 (H5Z_filter_t)(H5Z_FILTER_NOOP), /* Filter id number */ +#else + (H5Z_filter_t)(H5Z_FILTER_NOOP+1), /* Filter id number */ +#endif 1, /* encoder_present flag (set to true) */ 1, /* decoder_present flag (set to true) */ - "noop", /* Filter name for debugging */ +#if NOOP_INSTANCE == 0 + "noop", /* Filter name for debugging */ +#else + "noop1", /* Filter name for debugging */ +#endif (H5Z_can_apply_func_t)H5Z_noop_can_apply, /* The "can apply" callback */ NULL, /* The "set local" callback */ (H5Z_func_t)H5Z_filter_noop, /* The actual filter function */ @@ -62,21 +83,26 @@ H5PLget_plugin_info(void) * The "can_apply" callback returns positive a valid combination, zero for an * invalid combination and negative for an error. */ -htri_t +static htri_t H5Z_noop_can_apply(hid_t dcpl_id, hid_t type_id, hid_t space_id) { return 1; /* Assume it can always apply */ } -size_t +static size_t H5Z_filter_noop(unsigned int flags, size_t cd_nelmts, const unsigned int cd_values[], size_t nbytes, size_t *buf_size, void **buf) { void* newbuf; + size_t i; + + printf("cd_nelmts=%lu cd_values=",(unsigned long)cd_nelmts); + for(i=0;i $@ + cat ${srcdir}/H5Znoop.c >> $@ diff --git a/plugins/h5noop.h b/plugins/h5noop.h index 627692380c..e16ef95186 100644 --- a/plugins/h5noop.h +++ b/plugins/h5noop.h @@ -11,17 +11,8 @@ #define DECLSPEC extern #endif -/* use an integer greater than 256 to be id of the registered filter. */ -#define H5Z_FILTER_NOOP 40000 - /* declare the hdf5 interface */ DECLSPEC H5PL_type_t H5PLget_plugin_type(void); DECLSPEC const void* H5PLget_plugin_info(void); -DECLSPEC const H5Z_class2_t H5Z_NOOP[1]; - -/* Declare filter specific functions */ -DECLSPEC htri_t H5Z_noop_can_apply(hid_t dcpl_id, hid_t type_id, hid_t space_id); -DECLSPEC size_t H5Z_filter_noop(unsigned flags,size_t cd_nelmts,const unsigned cd_values[], - size_t nbytes,size_t *buf_size,void**buf); #endif /*H5MISC_H*/