diff --git a/CMakeLists.txt b/CMakeLists.txt index 846ceb6416..6a3629b05d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -36,10 +36,12 @@ SET(NC_VERSION_NOTE "-development") SET(netCDF_VERSION ${NC_VERSION_MAJOR}.${NC_VERSION_MINOR}.${NC_VERSION_PATCH}${NC_VERSION_NOTE}) SET(VERSION ${netCDF_VERSION}) SET(NC_VERSION ${netCDF_VERSION}) -SET(netCDF_LIB_VERSION 19) -SET(netCDF_SO_VERSION 19) SET(PACKAGE_VERSION ${VERSION}) +# These values should match those in configure.ac +SET(netCDF_LIB_VERSION 19}) +SET(netCDF_SO_VERSION 19) + # Version of the dispatch table. This must match the value in # configure.ac. SET(NC_DISPATCH_VERSION 5) @@ -1116,21 +1118,46 @@ ELSE() set(STD_FILTERS "${STD_FILTERS} bz2") ENDIF() +# If user wants, then install selected plugins (default on) +SET(PLUGIN_INSTALL_DIR "YES" CACHE STRING "Whether and where we should install plugins; defaults to yes") +SET(ENABLE_PLUGIN_INSTALL YES) +IF(DEFINED PLUGIN_INSTALL_DIR) #OR DEFINED CACHE{PLUGIN_INSTALL_DIR} + STRING(TOLOWER "${PLUGIN_INSTALL_DIR}" LCPID) + IF(LCPID STREQUAL "no") + SET(ENABLE_PLUGIN_INSTALL NO) + ELSE() + SET(ENABLE_PLUGIN_INSTALL YES) + ENDIF() +ENDIF() -# If user wants, then install selected plugins -SET(PLUGIN_INSTALL_DIR "" CACHE STRING "Whether and where we should install plugins") -SET(ENABLE_PLUGIN_INSTALL OFF) -if(DEFINED PLUGIN_INSTALL_DIR OR DEFINED CACHE{PLUGIN_INSTALL_DIR}) - IF(PLUGIN_INSTALL_DIR STREQUAL "") - MESSAGE(WARNING "No plugin directory value specified; option ignored.") - UNSET(PLUGIN_INSTALL_DIR) - UNSET(PLUGIN_INSTALL_DIR CACHE) - SET(PLUGIN_INSTALL_DIR_SETTING "N.A.") +IF(ENABLE_PLUGIN_INSTALL AND PLUGIN_INSTALL_DIR STREQUAL "YES") + # Default to last dir (lowest search priority) in HDF5_PLUGIN_PATH + IF(DEFINED ENV{HDF5_PLUGIN_PATH}) + SET(PLUGIN_PATH "$ENV{HDF5_PLUGIN_PATH}") + ELSE() + IF(ISMSVC OR ISMINGW) + SET(PLUGIN_PATH "$ENV{ALLUSERSPROFILE}\\hdf5\\lib\\plugin") + ELSE() + SET(PLUGIN_PATH "/usr/local/hdf5/lib/plugin") + ENDIF() + ENDIF() +ENDIF() + +IF(ENABLE_PLUGIN_INSTALL) + # Use the lowest priority dir in the path + IF(NOT ISMSVC AND NOT ISMINGW) + STRING(REPLACE ":" ";" PATH_LIST ${PLUGIN_PATH}) ELSE() - SET(PLUGIN_INSTALL_DIR_SETTING "${PLUGIN_INSTALL_DIR}") - SET(ENABLE_PLUGIN_INSTALL ON) + SET(PATH_LIST ${PLUGIN_PATH}) ENDIF() -ELSE() + + # Get last element + LIST(GET PATH_LIST -1 PLUGIN_INSTALL_DIR) + MESSAGE("Defaulting to -DPLUGIN_INSTALL_DIR=${PLUGIN_INSTALL_DIR}") + SET(PLUGIN_INSTALL_DIR_SETTING "${PLUGIN_INSTALL_DIR}") +ELSE() # No option specified + UNSET(PLUGIN_INSTALL_DIR) + UNSET(PLUGIN_INSTALL_DIR CACHE) SET(PLUGIN_INSTALL_DIR_SETTING "N.A.") ENDIF() @@ -1862,10 +1889,10 @@ ENDIF(ENABLE_MMAP) #CHECK_FUNCTION_EXISTS(alloca HAVE_ALLOCA) # Used in the `configure_file` calls below -SET(ISCMAKE "1") +SET(ISCMAKE "yes") IF(MSVC) -SET(ISMSVC ON CACHE BOOL "" FORCE) -SET(REGEDIT ON CACHE BOOL "" FORCE) +SET(ISMSVC yes CACHE BOOL "" FORCE) +SET(REGEDIT yes CACHE BOOL "" FORCE) # Get windows major version and build number EXECUTE_PROCESS(COMMAND "systeminfo" OUTPUT_VARIABLE WININFO) IF(WININFO STREQUAL "") @@ -2509,25 +2536,15 @@ configure_file( SET(EXTRA_DIST ${EXTRA_DIST} ${CMAKE_CURRENT_SOURCE_DIR}/test_common.in) SET(TOPSRCDIR "${CMAKE_CURRENT_SOURCE_DIR}") SET(TOPBUILDDIR "${CMAKE_CURRENT_BINARY_DIR}") -SET(ISMSVC "${MSVC}") configure_file(${CMAKE_CURRENT_SOURCE_DIR}/test_common.in ${CMAKE_CURRENT_BINARY_DIR}/test_common.sh @ONLY NEWLINE_STYLE LF) ##### -# Build nc_test4/findplugin.sh +# Build and copy nc_test4/findplugin.sh to various places ##### configure_file(${CMAKE_CURRENT_SOURCE_DIR}/nc_test4/findplugin.in ${CMAKE_CURRENT_BINARY_DIR}/nc_test4/findplugin.sh @ONLY NEWLINE_STYLE LF) - -IF(ENABLE_NCZARR) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/nc_test4/findplugin.in ${CMAKE_CURRENT_BINARY_DIR}/nczarr_test/findplugin.sh @ONLY NEWLINE_STYLE LF) -ENDIF() - -IF(ENABLE_PLUGINS) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/nc_test4/findplugin.in ${CMAKE_CURRENT_BINARY_DIR}/plugins/findplugin.sh @ONLY NEWLINE_STYLE LF) -ENDIF() - -IF(ENABLE_EXAMPLES) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/nc_test4/findplugin.in ${CMAKE_CURRENT_BINARY_DIR}/examples/C/findplugin.sh @ONLY NEWLINE_STYLE LF) -ENDIF() IF(ENABLE_TESTS) ##### diff --git a/Makefile.am b/Makefile.am index d0efa842f8..f5ea3906c3 100644 --- a/Makefile.am +++ b/Makefile.am @@ -10,6 +10,13 @@ # This directory stores libtool macros, put there by aclocal. ACLOCAL_AMFLAGS = -I m4 +## +# Turn off plugin directory during distcheck, see +# comment thread at https://github.com/Unidata/netcdf-c/pull/2348 +# for more information. +## +AM_DISTCHECK_CONFIGURE_FLAGS = --without-plugin-dir + # These files get added to the distribution. EXTRA_DIST = README.md COPYRIGHT INSTALL.md test_prog.c lib_flags.am \ cmake CMakeLists.txt COMPILE.cmake.txt config.h.cmake.in \ @@ -90,7 +97,7 @@ endif # Build Cloud Storage if desired. if ENABLE_NCZARR ZARR_TEST_DIR = nczarr_test -ZARR = libnczarr +ZARR = libnczarr endif # Optionally build test plugins @@ -107,7 +114,7 @@ endif # BUILD_BENCHMARKS # Define Test directories if BUILD_TESTSETS -TESTDIRS = $(H5_TEST_DIR) +TESTDIRS = $(H5_TEST_DIR) TESTDIRS += $(UNIT_TEST) $(V2_TEST) nc_test $(NC_TEST4) TESTDIRS += $(BENCHMARKS_DIR) $(HDF4_TEST_DIR) $(NCDAP2TESTDIR) $(NCDAP4TESTDIR) TESTDIRS += ${ZARR_TEST_DIR} diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 35280d5cd8..a187b42353 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -7,10 +7,10 @@ This file contains a high-level description of this package's evolution. Release ## 4.8.2 - TBD - +* [Enhancement] Improve filter installation process to avoid use of an extra shell script. See [Github #2348](https://github.com/Unidata/netcdf-c/pull/2348). * [Bug Fix] Get "make distcheck" to work See [Github #/2343](https://github.com/Unidata/netcdf-c/pull/2343). * [Enhancement] Allow the read/write of JSON-valued Zarr attributes to allow -for domain specific info such as used by GDAL/Zarr. See [Github #????](https://github.com/Unidata/netcdf-c/pull/????). +for domain specific info such as used by GDAL/Zarr. See [Github #2278](https://github.com/Unidata/netcdf-c/pull/2278). * [Enhancement] Turn on the XArray convention for NCZarr files by default. WARNING, this means that the mode should explicitly specify nczarr" or "zarr" even if "xarray" or "noxarray" is specified. See [Github #2257](https://github.com/Unidata/netcdf-c/pull/2257). * [Enhancement] Update the documentation to match the current filter capabilities See [Github #2249](https://github.com/Unidata/netcdf-c/pull/2249). * [Enhancement] Support installation of pre-built standard filters into user-specified location. See [Github #2318](https://github.com/Unidata/netcdf-c/pull/2318). diff --git a/configure.ac b/configure.ac index c33665e335..94aaab9348 100644 --- a/configure.ac +++ b/configure.ac @@ -24,6 +24,13 @@ AC_SUBST([NC_VERSION_MINOR]) NC_VERSION_MINOR=8 AC_SUBST([NC_VERSION_PATCH]) NC_VERSION_PATCH=2 AC_SUBST([NC_VERSION_NOTE]) NC_VERSION_NOTE="-development" +## +# These linker flags specify libtool version info. +# See http://www.gnu.org/software/libtool/manual/libtool.html#Libtool-versioning +# for information regarding incrementing `-version-info`. +# These values should match those in CMakeLists.txt +AC_SUBST([netCDF_SO_VERSION]) netCDF_SO_VERSION=19:1:0 + ##### # Set some variables used to generate a libnetcdf.settings file, # pattered after the files generated by libhdf4, libhdf5. @@ -679,6 +686,7 @@ AC_MSG_RESULT([${have_bz2}]) if test "x$have_bz2" = "xno" ; then have_local_bz2=yes + AC_MSG_NOTICE([Defaulting to internal libbz2]) else have_local_bz2=no fi @@ -1884,15 +1892,38 @@ AC_SUBST(STD_FILTERS,[$std_filters]) # If user wants, then install selected standard filters AC_MSG_CHECKING([whether and where we should install plugins]) -AC_ARG_WITH([plugin-dir], [AS_HELP_STRING([--with-plugin-dir=], - [Install selected standard filters in specified directory])]) +AC_ARG_WITH([plugin-dir], [AS_HELP_STRING([--with-plugin-dir=|no|--without-plugin-dir], + [Install selected standard filters in specified or default directory])], + [],[with_plugin_dir=yes]) AC_MSG_RESULT([$with_plugin_dir]) -if test "x$with_plugin_dir" = x ; then - AC_MSG_WARN([No plugin directory value specified; option ignored.]) - with_plugin_dir= +if test "x$with_plugin_dir" = xno ; then # option missing|disabled + with_plugin_dir=no with_plugin_dir_setting="N.A." enable_plugin_dir=no -else +elif test "x$with_plugin_dir" = xyes ; then # --with-plugin-dir, no argument + # Default to last dir (lowest search priority) in HDF5_PLUGIN_PATH + PLUGIN_PATH="$HDF5_PLUGIN_PATH" + if test "x${PLUGIN_PATH}" = x ; then + if test "x$ISMSVC" = xyes || test "x$ISMINGW" = xyes; then + PLUGIN_PATH="${ALLUSERSPROFILE}\\hdfd5\\lib\\plugin" + else + PLUGIN_PATH="/usr/local/hdf5/lib/plugin" + fi + fi + # Use the lowest priority dir in the path + if test "x$ISMSVC" = xyes || test "x$ISMINGW" = xyes; then + PLUGIN_DIR=`echo "$PLUGIN_PATH" | tr ';' ' '` + else + PLUGIN_DIR=`echo "$PLUGIN_PATH" | tr ':' ' '` + fi + for pp in ${PLUGIN_DIR} ; do last="$pp"; done + PLUGIN_DIR="$last" + with_plugin_dir_setting="$PLUGIN_DIR" + # canonical form is all forward slashes + with_plugin_dir=`echo "$PLUGIN_DIR" | tr '\\\\' '/'` + enable_plugin_dir=yes + AC_MSG_NOTICE([Defaulting to --with-plugin-dir=$with_plugin_dir]) +else # --with-plugin-dir= with_plugin_dir_setting="$with_plugin_dir" enable_plugin_dir=yes fi @@ -1991,7 +2022,6 @@ AC_CONFIG_FILES(test_common.sh:test_common.in) AC_CONFIG_FILES(nc_test4/findplugin.sh:nc_test4/findplugin.in) AC_CONFIG_FILES(nczarr_test/findplugin.sh:nc_test4/findplugin.in) AC_CONFIG_FILES(plugins/findplugin.sh:nc_test4/findplugin.in) -AC_CONFIG_FILES(plugins/stdinstall.sh:plugins/stdinstall.in) AC_CONFIG_FILES(examples/C/findplugin.sh:nc_test4/findplugin.in) AC_CONFIG_FILES(ncdap_test/findtestserver.c:ncdap_test/findtestserver.c.in) AC_CONFIG_FILES([nc_test/run_pnetcdf_tests.sh:nc_test/run_pnetcdf_tests.sh.in],[chmod ugo+x nc_test/run_pnetcdf_tests.sh]) diff --git a/docs/filters.md b/docs/filters.md index 26cffce152..82c2ad6c3d 100644 --- a/docs/filters.md +++ b/docs/filters.md @@ -1,13 +1,33 @@ NetCDF-4 Filter Support ============================ - + NetCDF-4 Filter Support {#filters} ================================== \tableofcontents +# Feature Overview {#filters_overview} + +NetCDF-C filters have some features of which the user +should be aware. + +* ***Auto Install of filters***
+An option is now provided to automatically install +HDF5 filters into a default location, or optionally +into a user-specified location. This is described in +[Appendix H](#filters_appendixh) +(with supporting information in [Appendix G](#filters_appendixg)). + +* ***NCZarr Filter Support***
+[NCZarr filters](#filters_nczarr) are now supported. +This essentially means that it is possible to specify +Zarr Codecs (Zarr equivalent of filters) in Zarr files +and have them processed using HDF5-style wrapper shared libraries. +Zarr filters can be used even if HDF5 support is disabled +in the netCDF-C library. + # Introduction to Filters {#filters_introduction} The netCDF library supports a general filter mechanism to apply @@ -48,10 +68,10 @@ A user may encounter an incompatibility if any of the following appears in user * The function *\_nc\_inq\_var\_filter* was returning the error value NC\_ENOFILTER if a variable had no associated filters. It has been reverted to the previous case where it returns NC\_NOERR and the returned filter id was set to zero if the variable had no filters. * The function *nc\_inq\_var\_filterids* was renamed to *nc\_inq\_var\_filter\_ids*. -* Some auxilliary functions for parsing textual filter specifications have been moved to the file *netcdf\_aux.h*. See Appendix A. +* Some auxilliary functions for parsing textual filter specifications have been moved to the file *netcdf\_aux.h*. See [Appendix A](#filters_appendixa). * All of the "filterx" functions have been removed. This is unlikely to cause problems because they had limited visibility. -For additional information, see Appendix B. +For additional information, see [Appendix B](#filters_appendixb). # Enabling A HDF5 Compression Filter {#filters_enable} @@ -292,7 +312,7 @@ The rules for all possible cases of the "-F none" flag are defined by this table # Filter Specification Syntax {#filters_syntax} The utilities ncgen and nccopy, and also the output of *ncdump*, support the specification of filter ids, formats, and parameters in text format. -The BNF specification is defined in Appendix C. +The BNF specification is defined in [Appendix C](#filters_appendixc). Basically, These specifications consist of a filter id, a comma, and then a sequence of comma separated constants representing the parameters. The constants are converted within the utility to a proper set of unsigned int constants (see the parameter encoding section). @@ -330,14 +350,33 @@ Each filter is assumed to be compiled into a separate dynamically loaded library For HDF5 conformant filters, these filter libraries are assumed to be in some specific location. The details for writing such a filter are defined in the HDF5 documentation[1,2]. -## Plugin directory {#filters_Plugindir} +## Plugin directory {#filters_plugindir} + +The HDF5 loader searches for plugins in a number of directories. +This search is contingent on the presence or absence of the environment +variable named ***HDF5_PLUGIN_PATH***. + +As with all other "...PATH" variables, it is a sequence of absolute +directories separated by a separator character. For *nix* operating systems, +this separator is the colon (':') character. For Windows and Mingw, the +separator is the semi-colon (';') character. So for example: +* Linux: export HDF5_PLUGIN_PATH=/usr/lib:/usr/local/lib +* Windows: export HDF5_PLUGIN_PATH=c:\ProgramData\hdf5\plugin;c:\tools\lib -The HDF5 loader expects plugins to be in a specified plugin directory. -The default directory is: -* "/usr/local/hdf5/lib/plugin” for linux/unix operating systems (including Cygwin) -* “%ALLUSERSPROFILE%\\hdf5\\lib\\plugin” for Windows systems, although the code does not appear to explicitly use this path. +If HDF5_PLUGIN_PATH is defined, then the loader will search each directory +in the path from left to right looking for shared libraries with specific +exported symbols representing the entry points into the library. -The default may be overridden using the environment variable *HDF5\_PLUGIN\_PATH*. +If HDF5_PLUGIN_PATH is not defined, the loader defaults to using +these default directories: +* Linux: "/usr/local/hdf5/lib/plugin” +* Windows: “%ALLUSERSPROFILE%\\hdf5\\lib\\plugin” + +It should be noted that there is a difference between the search order +for HDF5 versus NCZarr. The HDF5 loader will search only the directories +specificed in HDF5_PLUGIN_PATH. In NCZarr, the loader +searches HDF5_PLUGIN_PATH and as a last resort, +it also searches the default directory. ## Plugin Library Naming {#filters_Pluginlib} @@ -345,15 +384,17 @@ Given a plugin directory, HDF5 examines every file in that directory that confor
PlatformBasenameExtension
Linuxlib*.so* -
OSXlib*.so* +
OSXlib*.dylib*
Cygwincyg*.dll*
Windows*.dll
## Plugin Verification {#filters_Pluginverify} -For each dynamic library located using the previous patterns, HDF5 attempts to load the library and attempts to obtain information from it. -Specifically, It looks for two functions with the following signatures. +For each dynamic library located using the previous patterns, +HDF5 attempts to load the library and attempts to obtain +information from it. Specifically, It looks for two functions +with the following signatures. 1. *H5PL\_type\_t H5PLget\_plugin\_type(void)* — This function is expected to return the constant value *H5PL\_TYPE\_FILTER* to indicate that this is a filter library. 2. *const void* H5PLget\_plugin\_info(void)* — This function returns a pointer to a table of type *H5Z\_class2\_t*. @@ -491,7 +532,7 @@ The netcdf-c library processes all of the shared libraries by interrogating each Any libraries that do not export one or both of the well-known APIs is ignored. Internally, the netcdf-c library pairs up each HDF5 library API with a corresponding Codec API by invoking the relevant well-known functions -(See Appendix E/a>). +(See [Appendix E](#filters_appendixe). This results in this table for associated codec and hdf5 libraries.
HDF5 APICodec APIAction @@ -524,7 +565,7 @@ The netcdf-c library examines its list of known filters to find one matching the The set of parameters provided is stored internally. Then during writing of data, the corresponding HDF5 filter is invoked to encode the data. -When it comes time to write out the meta-data, the stored HDF5-style parameters are passed to a specific Codec function to obtain the corresponding JSON representation. Again see Appendix E. +When it comes time to write out the meta-data, the stored HDF5-style parameters are passed to a specific Codec function to obtain the corresponding JSON representation. Again see [Appendix E](#filters_appendixe). This resulting JSON is then written in the NCZarr metadata. ### Reading an NCZarr Container @@ -799,7 +840,7 @@ This leads to the following set of rules. Because of the encoding rules, this 8-byte value will be in LE format. 4. The filter must finally do an 8-byte byte-swap on that 8-byte value to convert it to desired BE format. -To support these rules, some utility programs exist and are discussed in Appendix B. +To support these rules, some utility programs exist and are discussed in [Appendix B](#filters_appendixb). # Appendix B. Support Utilities {#filters_appendixb} @@ -1017,7 +1058,7 @@ The list of returned items are used to try to provide defaults for any HDF5 filters that have no corresponding Codec. This is for internal use only. -# Appendix F. Standard Filters +# Appendix F. Standard Filters {#filters_appendixf} Support for a select set of standard filters is built into the NetCDF API. Generally, they are accessed using the following generic API, where XXXX is @@ -1042,44 +1083,87 @@ So aside from those four, the current set of standard filters is as follows.
bzip2307https://sourceware.org/bzip2/
-# Appendix G. Finding Filters +It is important to note that in order to use each standard filter, several additonal libraries must be installed. +Consider the zstandard compressor, which is one of the supported standard filters. +When installing the netcdf library, the following other libraries must be installed. + +1. *libzstd.so* | *zstd.dll* | *libzstd.dylib* -- The actual zstandard compressor library; typically installed by using your platform specific package manager. +2. The HDF5 wrapper for *libzstd.so* -- There are several options for obtaining this (see [Appendix G](#filters_appendixg).) +3. (Optional) The Zarr wrapper for *libzstd.so* -- you need this if you intend to read/write Zarr datasets that were compressed using zstandard; again see [Appendix G](#filters_appendixg). + +# Appendix G. Finding Filters {#filters_appendixg} -A major problem for filter users is finding an implementation for a filter. -There are several ways to do this. +A major problem for filter users is finding an implementation of an HDF5 filter wrapper and (optionally) +its corresponding NCZarr wrapper. There are several ways to do this. + +* **--with-plugin-dir** — An option to *./configure* that will install the necessary wrappers. + See [Appendix H](#filters_appendixh). * **HDF5 Assigned Filter Identifiers Repository [3]** — HDF5 maintains a page of standard filter identifiers along with additional contact information. This often includes a pointer -to source code. +to source code. This will provide only HDF5 wrappers and not NCZarr wrappers. * **Community Codec Repository** — The Community Codec Repository (CCR) project [8] provides filters, including HDF5 wrappers, for a number of filters. +It does not as yet provide Zarr wrappers. You can install this library to get access to these supported filters. It does not currently include the required NCZarr Codec API, so they are only usable with netcdf-4. This will change in the future. -* **NetCDF-C Test Plugins Directory** — -As part of the overall build process, a number of filters are built as shared libraries in the "plugins" directory. -They may be in that directory or the "plugins/.libs" subdirectory. -It may be possible for users to utilize some of those libraries to provide filter support for general use. +# Appendix H. Auto-Install of Filter Wrappers {#filters_appendixh} +As part of the overall build process, a number of filter wrappers are built as shared libraries in the "plugins" directory. +These wrappers can be installed as part of the overall netcdf-c installation process. +WARNING: the installer still needs to make sure that the actual filter/compression libraries are installed: e.g. libzstd and/or libblosc. - If the user is using NCZarr filters, then the plugins directory has at least the following shared libraries - * libh5shuffle.so — shuffle filter - * libh5fletcher32.so — fletcher32 checksum - * libh5deflate.so — deflate compression - * libnczdefaults.so — provide NCZarr support for shuffle, fletcher32, and deflate. - * *libh5bzip2.so* — an HDF5 filter for bzip2 compression - * *libh5blosc.so* — an HDF5 filter for blosc compression - * *libh5zstd.so* — an HDF5 filter for zstandard compression +The target location into which libraries in the "plugins" directory are installed is specified +using a special *./configure* option +```` +--with-plugin-dir= +or +--with-plugin-dir +```` +or its corresponding *cmake* option. +```` +-DPLUGIN_INSTALL_DIR= +or +-DPLUGIN_INSTALL_DIR=YES +```` +This option defaults to the value "yes", which means that filters are +installed by default. This can be disabled by one of the following options. +```` +--without-plugin-dir (automake) +or +--with-plugin-dir=no (automake) +or +-DPLUGIN_INSTALL_DIR=NO (CMake) +```` - The shuffle, fletcher32, and deflate filters in this case will - be ignored by HDF5 and only used by the NCZarr code. But in - order to use them, it needs additional Codec capabilities - provided by the libnczdefauts.so shared library. Note also that - if you disable HDF5 support, but leave NCZarr support enabled, - then all of the above filters should continue to work. +If the option is specified with no argument (automake) or with the value "YES" (CMake), +then it defaults (in order) to the following directories: +1. If the HDF5_PLUGIN_PATH environment variable is defined, then last directory in the list of directories in the path is used. +2. (a) "/usr/local/hdf5/lib/plugin” for linux/unix operating systems (including Cygwin)
+ (b) “%ALLUSERSPROFILE%\\hdf5\\lib\\plugin” for Windows and MinGW + +If NCZarr is enabled, then in addition to wrappers for the standard filters, +additional libraries will be installed to support NCZarr access to filters. +Currently, this list includes the following: +* shuffle — shuffle filter +* fletcher32 — fletcher32 checksum +* deflate — deflate compression +* (optional) szip — szip compression, if libsz is available +* bzip2 — an HDF5 filter for bzip2 compression +* lib__nczh5filters.so — provide NCZarr support for shuffle, fletcher32, deflate, and (optionally) szip. +* lib__nczstdfilters.so — provide NCZarr support for bzip2, (optionally)zstandard, and (optionally) blosc. + +The shuffle, fletcher32, and deflate filters in this case will +be ignored by HDF5 and only used by the NCZarr code. But in +order to use them, it needs additional Codec capabilities +provided by the *lib__nczh5filters.so* shared library. Note also that +if you disable HDF5 support, but leave NCZarr support enabled, +then all of the above filters should continue to work. ## HDF5_PLUGIN_PATH @@ -1092,16 +1176,23 @@ wrappers but also the NCZarr codec wrappers. path-list. That is it is a sequence of absolute directory paths separated by a specific separator character. For Windows, the separator character is a semicolon (';') and for Unix, it is a a -colon (':'). For convenience, NCZarr will also accept the -semicolon separator for Unix. +colon (':'). + +So, if HDF5_PLUGIN_PATH is defined at build time, and +*--with-plugin-dir* is specified with no argument then the last +directory in the path will be the one into which filter wrappers are +installed. Otherwise the default directories are used. + +The important thing to note is that at run-time, there are several cases to consider: -So, the user can add the CCR and/or the plugins directory to -the *HDF5\_PLUGIN\_PATH* environment variable to allow the netcdf-c -library to locate wrappers. +1. HDF5_PLUGIN_PATH is defined and is the same value as it was at build time -- no action needed +2. HDF5_PLUGIN_PATH is defined and is has a different value from build time -- the user is responsible for ensuring that the run-time path includes the same directory used at build time, otherwise this case will fail. +3. HDF5_PLUGIN_DIR is not defined at either run-time or build-time -- no action needed +4. HDF5_PLUGIN_DIR is not defined at run-time but was defined at build-time -- this will probably fail # Point of Contact {#filters_poc} *Author*: Dennis Heimbigner
*Email*: dmh at ucar dot edu
*Initial Version*: 1/10/2018
-*Last Revised*: 3/14/2022 +*Last Revised*: 5/18/2022 diff --git a/examples/C/run_filter.sh b/examples/C/run_filter.sh index 90e0935fe9..d87b60813c 100755 --- a/examples/C/run_filter.sh +++ b/examples/C/run_filter.sh @@ -15,11 +15,14 @@ set -e echo "*** running test_filter example..." . ${builddir}/findplugin.sh +echo "findplugin.sh loaded" # Locate the plugin path and the library names; argument order is critical # Find bzip2 and capture findplugin h5bzip2 -BZIP2PATH="${HDF5_PLUGIN_DIR}/${HDF5_PLUGIN_LIB}" +BZIP2LIB="${HDF5_PLUGIN_LIB}" +BZIP2DIR="${HDF5_PLUGIN_DIR}" +BZIP2PATH="${BZIP2DIR}/${BZIP2LIB}" # Verify if ! test -f ${BZIP2PATH} ; then echo "Unable to locate ${BZIP2PATH}"; exit 1; fi diff --git a/liblib/Makefile.am b/liblib/Makefile.am index 7a6f7eb0c7..b456aa439a 100644 --- a/liblib/Makefile.am +++ b/liblib/Makefile.am @@ -18,7 +18,7 @@ lib_LTLIBRARIES = libnetcdf.la # for information regarding incrementing `-version-info`. ## -libnetcdf_la_LDFLAGS = -version-info 19:1:0 ${NOUNDEFINED} +libnetcdf_la_LDFLAGS = -version-info @netCDF_SO_VERSION@ ${NOUNDEFINED} libnetcdf_la_CPPFLAGS = ${AM_CPPFLAGS} libnetcdf_la_LIBADD = diff --git a/libnczarr/zclose.c b/libnczarr/zclose.c index f287a2f5a0..f1f3354b1d 100644 --- a/libnczarr/zclose.c +++ b/libnczarr/zclose.c @@ -167,7 +167,7 @@ zclose_vars(NC_GRP_INFO_T* grp) #ifdef ENABLE_NCZARR_FILTERS /* Reclaim filters */ if(var->filters != NULL) { - (void)NCZ_filter_freelist(var); + (void)NCZ_filter_freelists(var); } var->filters = NULL; #endif diff --git a/libnczarr/zfilter.c b/libnczarr/zfilter.c index 963a977183..72d0e139af 100644 --- a/libnczarr/zfilter.c +++ b/libnczarr/zfilter.c @@ -65,6 +65,7 @@ /* Hold the loaded filter plugin information */ typedef struct NCZ_Plugin { + int incomplete; struct HDF5API { const H5Z_class2_t* filter; NCPSharedLib* hdf5lib; /* source of the filter */ @@ -134,11 +135,15 @@ typedef struct NCZ_Filter { # define FLAG_CODEC 4 /* If set, then visbile parameters come from an existing codec string */ # define FLAG_HDF5 8 /* If set, => visible parameters came from nc_def_var_filter */ # define FLAG_NEWVISIBLE 16 /* If set, => visible parameters were modified */ +# define FLAG_INCOMPLETE 32 /* If set, => filter has no complete matching plugin */ NCZ_HDF5 hdf5; NCZ_Codec codec; struct NCZ_Plugin* plugin; /**< Implementation of this filter. */ + int chainindex; /* Position in original chain */ } NCZ_Filter; +#define FILTERINCOMPLETE(f) ((f)->flags & FLAG_INCOMPLETE?1:0) + /* WARNING: GLOBAL DATA */ /* TODO: move to common global state */ @@ -280,12 +285,14 @@ static int pluginnamecheck(const char* name); */ int -NCZ_filter_freelist(NC_VAR_INFO_T* var) +NCZ_filter_freelists(NC_VAR_INFO_T* var) { int i, stat=NC_NOERR; - NClist* filters = (NClist*)var->filters; + NClist* filters = NULL; + NCZ_VAR_INFO_T* zvar = (NCZ_VAR_INFO_T*)var->format_var_info; ZTRACE(6,"var=%s",var->hdr.name); + filters = (NClist*)var->filters; if(filters == NULL) goto done; /* Free the filter list elements */ for(i=0;ifilters = NULL; + /* Free the incomplete filters */ + filters = (NClist*)zvar->incompletefilters; + if(filters == NULL) goto done; + /* Free the filter list elements */ + for(i=0;iincompletefilters = NULL; done: return ZUNTRACE(stat); } @@ -337,7 +354,7 @@ NCZ_addfilter(NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, unsigned int id, size_t int stat = NC_NOERR; struct NCZ_Filter* fi = NULL; NCZ_Plugin* plugin = NULL; - NCZ_HDF5 hdf5 = hdf5_empty; + NCZ_VAR_INFO_T* zvar = (NCZ_VAR_INFO_T*)var->format_var_info; ZTRACE(6,"file=%s var=%s id=%u nparams=%u params=%p",file->hdr.name,var->hdr.name,id,nparams,params); @@ -345,24 +362,16 @@ NCZ_addfilter(NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, unsigned int id, size_t {stat = NC_EINVAL; goto done;} if(var->filters == NULL) var->filters = (void*)nclistnew(); + if(zvar->incompletefilters == NULL) zvar->incompletefilters = (void*)nclistnew(); /* Before anything else, find the matching plugin */ if((stat = NCZ_plugin_loaded(id,&plugin))) goto done; - if(plugin == NULL || plugin->codec.codec == NULL) { /* fail */ + if(plugin == NULL) { ZLOG(NCLOGWARN,"no such plugin: %u",(unsigned)id); stat = NC_ENOFILTER; goto done; } - /* Fill in the hdf5 */ - hdf5 = hdf5_empty; - hdf5.id = id; - /* Capture the visible parameters */ - hdf5.visible.nparams = nparams; - if(nparams > 0) { - if((stat = paramclone(nparams,&hdf5.visible.params,params))) goto done; - } - /* Find the NCZ_Filter */ if((stat=NCZ_filter_lookup(var,id,&fi))) goto done; if(fi != NULL) { @@ -373,19 +382,31 @@ NCZ_addfilter(NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, unsigned int id, size_t if((fi = calloc(1,sizeof(struct NCZ_Filter))) == NULL) {stat = NC_ENOMEM; goto done;} fi->plugin = plugin; - nclistpush((NClist*)var->filters, fi); - } + if(plugin->incomplete) { + fi->flags |= (FLAG_INCOMPLETE); + nclistpush((NClist*)zvar->incompletefilters, fi); + } else + nclistpush((NClist*)var->filters, fi); + } - /* (over)write the HDF5 parameters */ - nullfree(fi->hdf5.visible.params); - nullfree(fi->hdf5.working.params); - fi->hdf5.working.nparams = 0; - fi->hdf5.working.params = NULL; - fi->hdf5 = hdf5; - hdf5 = hdf5_empty; - fi->flags |= FLAG_VISIBLE; + if(!FILTERINCOMPLETE(fi)) { + /* (over)write the HDF5 parameters */ + nullfree(fi->hdf5.visible.params); + nullfree(fi->hdf5.working.params); + /* Fill in the hdf5 */ + fi->hdf5 = hdf5_empty; /* struct copy */ + fi->hdf5.id = id; + /* Capture the visible parameters */ + fi->hdf5.visible.nparams = nparams; + if(nparams > 0) { + if((stat = paramclone(nparams,&fi->hdf5.visible.params,params))) goto done; + } + fi->hdf5.working.nparams = 0; + fi->hdf5.working.params = NULL; + fi->flags |= FLAG_VISIBLE; + } - fi = NULL; /* either way,its in the var->filters list */ + fi = NULL; /* either way,its in a filters list */ done: if(fi) NCZ_filter_free(fi); @@ -422,7 +443,6 @@ NCZ_filter_lookup(NC_VAR_INFO_T* var, unsigned int id, struct NCZ_Filter** specp int i; NClist* flist = (NClist*)var->filters; - ZTRACE(6,"var=%s id=%u",var->hdr.name,id); if(specp) *specp = NULL; @@ -434,7 +454,7 @@ NCZ_filter_lookup(NC_VAR_INFO_T* var, unsigned int id, struct NCZ_Filter** specp for(i=0;ihdf5.id == id) { + if(spec->hdf5.id == id && !FILTERINCOMPLETE(spec)) { if(specp) *specp = spec; break; } @@ -606,7 +626,7 @@ NCZ_inq_var_filter_ids(int ncid, int varid, size_t* nfiltersp, unsigned int* ids NC_GRP_INFO_T* grp = NULL; NC_VAR_INFO_T* var = NULL; NClist* flist = NULL; - size_t nfilters = 0; + size_t nfilters; ZTRACE(1,"ncid=%d varid=%d",ncid,varid); @@ -624,12 +644,12 @@ NCZ_inq_var_filter_ids(int ncid, int varid, size_t* nfiltersp, unsigned int* ids flist = var->filters; - nfilters = nclistlength(flist); + nfilters = nclistlength(flist); /* including incomplets */ if(nfilters > 0 && ids != NULL) { int k; for(k=0;khdf5.id; + ids[k] = f->hdf5.id; } } if(nfiltersp) *nfiltersp = nfilters; @@ -710,7 +730,7 @@ NCZ_inq_filter_avail(int ncid, unsigned id) if((stat = NCZ_filter_initialize())) goto done; /* Check the available filters list */ if((stat = NCZ_plugin_loaded((int)id, &plug))) goto done; - if(plug == NULL) + if(plug == NULL || plug->incomplete) stat = NC_ENOFILTER; done: return ZUNTRACE(stat); @@ -770,6 +790,9 @@ NCZ_filter_finalize(void) if(nclistlength(default_libs) > 0) { for(i=0;i>> DEBUGL: NCZ_filter_finalize: reclaim default_lib[i]=%p\n",l); +#endif if(l != NULL) (void)ncpsharedlibfree(l); } } @@ -822,7 +845,9 @@ NCZ_applyfilterchain(const NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, NClist* cha /* Make sure all the filters are loaded && setup */ for(i=0;ihdf5.id > 0 && f->plugin != NULL); + assert(f != NULL); + if(FILTERINCOMPLETE(f)) {stat = NC_ENOFILTER; goto done;} + assert(f->hdf5.id > 0 && f->plugin != NULL); if(!(f->flags & FLAG_WORKING)) {/* working not yet available */ if((stat = ensure_working(var,f))) goto done; } @@ -938,10 +963,9 @@ NCZ_filter_jsonize(const NC_FILE_INFO_T* file, const NC_VAR_INFO_T* var, NCZ_Fil return ZUNTRACEX(stat,"codec=%s",NULLIFY(filter->codec.codec)); } - /* Build filter from parsed Zarr metadata */ int -NCZ_filter_build(const NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, const NCjson* jfilter) +NCZ_filter_build(const NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, const NCjson* jfilter, int chainindex) { int i,stat = NC_NOERR; NCZ_Filter* filter = NULL; @@ -949,10 +973,12 @@ NCZ_filter_build(const NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, const NCjson* j NCZ_Plugin* plugin = NULL; NCZ_Codec codec = codec_empty; NCZ_HDF5 hdf5 = hdf5_empty; + NCZ_VAR_INFO_T* zvar = (NCZ_VAR_INFO_T*)var->format_var_info; ZTRACE(6,"file=%s var=%s jfilter=%s",file->hdr.name,var->hdr.name,NCJtrace(jfilter)); if(var->filters == NULL) var->filters = nclistnew(); + if(zvar->incompletefilters == NULL) zvar->incompletefilters = nclistnew(); /* Get the id of this codec filter */ if(NCJdictget(jfilter,"id",&jvalue)<0) {stat = NC_EFILTER; goto done;} @@ -969,10 +995,15 @@ NCZ_filter_build(const NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, const NCjson* j /* Find the plugin for this filter */ for(i=0;i<=loaded_plugins_max;i++) { if (!loaded_plugins[i]) continue; + if(!loaded_plugins[i] || !loaded_plugins[i]->codec.codec) continue; /* no plugin or no codec */ if(strcmp(NCJstring(jvalue), loaded_plugins[i]->codec.codec->codecid) == 0) {plugin = loaded_plugins[i]; break;} } + /* Will always have a filter; possibly unknown */ + if((filter = calloc(1,sizeof(NCZ_Filter)))==NULL) {stat = NC_ENOMEM; goto done;} + filter->chainindex = chainindex; + if(plugin != NULL) { /* Save the hdf5 id */ hdf5.id = plugin->codec.codec->hdf5id; @@ -984,13 +1015,16 @@ NCZ_filter_build(const NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, const NCjson* j #endif if(stat) goto done; } - - if((filter = calloc(1,sizeof(NCZ_Filter)))==NULL) {stat = NC_ENOMEM; goto done;} filter->flags |= FLAG_VISIBLE; filter->hdf5 = hdf5; hdf5 = hdf5_empty; filter->codec = codec; codec = codec_empty; filter->flags |= FLAG_CODEC; filter->plugin = plugin; plugin = NULL; + } else { + /* Create a fake filter so we do not forget about this codec */ + filter->hdf5 = hdf5_empty; + filter->codec = codec; codec = codec_empty; + filter->flags |= (FLAG_INCOMPLETE|FLAG_CODEC); } if(filter != NULL) { @@ -1112,12 +1146,10 @@ static int NCZ_load_all_plugins(void) { int i,j,ret = NC_NOERR; - const char* pluginroot = NULL; + const char* pluginroots = NULL; struct stat buf; NClist* dirs = nclistnew(); -#ifdef _WIN32 - char pluginpath32[4096]; -#endif + char* defaultpluginpath = NULL; ZTRACE(6,""); @@ -1125,29 +1157,37 @@ NCZ_load_all_plugins(void) fprintf(stderr,">>> DEBUGL: NCZ_load_all_plugins\n"); #endif - /* Find the plugin directory root(s) */ - pluginroot = getenv(plugin_env); - if(pluginroot == NULL || strlen(pluginroot) == 0) { + /* Setup the plugin path default */ + { #ifdef _WIN32 const char* win32_root; - win32_root = getenv(win32_root_env); + char dfalt[4096]; + win32_root = getenv(WIN32_ROOT_ENV); if(win32_root != NULL && strlen(win32_root) > 0) { - snprintf(pluginpath32,sizeof(pluginpath32),plugin_dir_win,win32_root); - pluginroot = pluginpath32; - } else - pluginroot = NULL; + snprintf(dfalt,sizeof(dfalt),PLUGIN_DIR_WIN,win32_root); + defaultpluginpath = strdup(dfalt); + } #else /*!_WIN32*/ - pluginroot = plugin_dir_unix; + defaultpluginpath = strdup(PLUGIN_DIR_UNIX); #endif } - ZTRACEMORE(6,"pluginroot=%s",(pluginroot?pluginroot:"null")); - if(pluginroot == NULL) { - ZLOG(NCLOGERR,"no pluginroot: %s",plugin_env); - ret = NC_ENOFILTER; goto done; + /* Find the plugin directory root(s) */ + pluginroots = getenv(PLUGIN_ENV); /* Usually HDF5_PLUGIN_PATH */ + if(pluginroots != NULL && strlen(pluginroots) == 0) pluginroots = NULL; + if(pluginroots == NULL) { + pluginroots = strdup(defaultpluginpath); } + assert(pluginroots != NULL); + ZTRACEMORE(6,"pluginroots=%s",(pluginroots?pluginroots:"null")); + + if((ret = NCZ_split_plugin_path(pluginroots,dirs))) goto done; - if((ret = NCZ_split_plugin_path(pluginroot,dirs))) goto done; + /* Add the default to end of the dirs list if not already there */ + if(!nclistmatch(dirs,defaultpluginpath,0)) { + nclistpush(dirs,defaultpluginpath); + defaultpluginpath = NULL; + } for(i=0;ihdf5.filter == NULL || p->codec.codec == NULL) { + /* mark this entry as incomplete */ + p->incomplete = 1; #ifdef DEBUGL - fprintf(stderr,">>> DEBUGL: Incomplete plugin: expunged: id=%u; reasons: %s %s\n",i, + fprintf(stderr,">>> DEBUGL: Incomplete plugin: id=%u; reasons: %s %s\n",i, (p->hdf5.filter==NULL?"hdf5":""),(p->codec.codec==NULL?"codec":"")); #endif - /* expunge this entry */ - (void)NCZ_unload_plugin(p); - loaded_plugins[i] = NULL; } #ifdef DEBUGL else @@ -1227,6 +1265,7 @@ NCZ_load_all_plugins(void) NCZ_Plugin* p; for(i=0;iincomplete) continue; if(p->hdf5.filter != NULL && p->codec.codec != NULL) { if(p->codec.codec && p->codec.codec->NCZ_codec_initialize) p->codec.codec->NCZ_codec_initialize(); @@ -1239,6 +1278,7 @@ NCZ_load_all_plugins(void) } done: + nullfree(defaultpluginpath); nclistfreeall(dirs); errno = 0; return ZUNTRACE(ret); @@ -1347,12 +1387,38 @@ NCZ_load_plugin_dir(const char* path) return ZUNTRACE(stat); } +int +loadcodecdefaults(const char* path, const NCZ_codec_t** cp, NCPSharedLib* lib, int* lib_usedp) +{ + int stat = NC_NOERR; + int lib_used = 0; + + nclistpush(default_libs,lib); + for(;*cp;cp++) { + struct CodecAPI* c0; +#ifdef DEBUGL + fprintf(stderr,"@@@ %s: %s = %u\n",path,(*cp)->codecid,(*cp)->hdf5id); +#endif + c0 = (struct CodecAPI*)calloc(1,sizeof(struct CodecAPI)); + if(c0 == NULL) {stat = NC_ENOMEM; goto done;} + c0->codec = *cp; + c0->codeclib = lib; + lib_used = 1; /* remember */ + nclistpush(codec_defaults,c0); c0 = NULL; + } +done: + if(lib_usedp) *lib_usedp = lib_used; + return stat; +} + static int NCZ_load_plugin(const char* path, struct NCZ_Plugin** plugp) { int stat = NC_NOERR; NCZ_Plugin* plugin = NULL; const H5Z_class2_t* h5class = NULL; + H5PL_type_t h5type = 0; + const NCZ_codec_t** cp = NULL; const NCZ_codec_t* codec = NULL; NCPSharedLib* lib = NULL; int flags = NCP_GLOBAL; @@ -1374,7 +1440,7 @@ NCZ_load_plugin(const char* path, struct NCZ_Plugin** plugp) if((stat = ncpload(lib,path,flags))) goto done; #ifdef DEBUGL - fprintf(stderr,">>> DEBUGL: NCZ_load_plugin: path=%s\n",path); + fprintf(stderr,">>> DEBUGL: NCZ_load_plugin: path=%s lib=%p\n",path,lib); #endif /* See what we have */ @@ -1384,49 +1450,27 @@ NCZ_load_plugin(const char* path, struct NCZ_Plugin** plugp) const NCZ_get_codec_info_proto npi = (NCZ_get_codec_info_proto)ncpgetsymbol(lib,"NCZ_get_codec_info"); const NCZ_codec_info_defaults_proto cpd = (NCZ_codec_info_defaults_proto)ncpgetsymbol(lib,"NCZ_codec_info_defaults"); - if(gpt == NULL && gpi == NULL && npi == NULL && cpd == NULL) + if(gpt == NULL && gpi == NULL && npi == NULL && cpd == NULL) {stat = NC_ENOFILTER; goto done;} + /* We can have cpd or we can have (gpt && gpi && npi) but not both sets */ if(cpd != NULL) { - /* Deal with defaults first */ - const NCZ_codec_t** cp = NULL; - nclistpush(default_libs,lib); cp = (const NCZ_codec_t**)cpd(); -#ifdef DEBUGL - fprintf(stderr,"@@@ %s: default codec library found: %p\n",path,cp); -#endif - for(;*cp;cp++) { - struct CodecAPI* c0; -#ifdef DEBUGL - fprintf(stderr,"@@@ %s: %s = %u\n",path,(*cp)->codecid,(*cp)->hdf5id); -#endif - c0 = (struct CodecAPI*)calloc(1,sizeof(struct CodecAPI)); - if(c0 == NULL) {stat = NC_ENOMEM; goto done1;} - c0->codec = *cp; - c0->codeclib = lib; - nclistpush(codec_defaults,c0); c0 = NULL; + } else {/* cpd => !gpt && !gpi && !npi */ + if(gpt != NULL && gpi != NULL) { /* get HDF5 info */ + h5type = gpt(); + h5class = gpi(); + /* Verify */ + if(h5type != H5PL_TYPE_FILTER) {stat = NC_EPLUGIN; goto done;} + if(h5class->version != H5Z_CLASS_T_VERS) {stat = NC_EFILTER; goto done;} + } + if(npi != NULL) {/* get Codec info */ + codec = npi(); + /* Verify */ + if(codec->version != NCZ_CODEC_CLASS_VER) {stat = NC_EPLUGIN; goto done;} + if(codec->sort != NCZ_CODEC_HDF5) {stat = NC_EPLUGIN; goto done;} } -done1: - lib = NULL; - goto done; - } - - if(gpt != NULL && gpi != NULL) { - /* get HDF5 info */ - H5PL_type_t h5type = gpt(); - h5class = gpi(); - /* Verify */ - if(h5type != H5PL_TYPE_FILTER) {stat = NC_EPLUGIN; goto done;} - if(h5class->version != H5Z_CLASS_T_VERS) {stat = NC_EFILTER; goto done;} - } - - if(npi != NULL) { - /* get Codec info */ - codec = npi(); - /* Verify */ - if(codec->version != NCZ_CODEC_CLASS_VER) {stat = NC_EPLUGIN; goto done;} - if(codec->sort != NCZ_CODEC_HDF5) {stat = NC_EPLUGIN; goto done;} - } + } } #ifdef DEBUGL @@ -1436,6 +1480,17 @@ if(codec) fprintf(stderr,">>> %u/%s",codec->hdf5id,codec->codecid); fprintf(stderr,">>> \n"); #endif + /* Handle defaults separately */ + if(cp != NULL) { + int used = 0; +#ifdef DEBUGL + fprintf(stderr,"@@@ %s: default codec library found: %p\n",path,cp); +#endif + if((stat = loadcodecdefaults(path,cp,lib,&used))) goto done; + if(used) lib = NULL; + goto done; + } + if(h5class != NULL && codec != NULL) { /* Verify consistency of the HDF5 and the Codec */ if(h5class->id != codec->hdf5id) goto done; /* ignore */ @@ -1455,39 +1510,36 @@ fprintf(stderr,">>> \n"); h5id = codec->hdf5id; if((stat = NCZ_plugin_loaded(codec->hdf5id,&plugin))) goto done; } + if(plugin == NULL) { /* create new entry */ if((plugin = (NCZ_Plugin*)calloc(1,sizeof(NCZ_Plugin)))==NULL) {stat = NC_ENOMEM; goto done;} - } - + } + /* Fill in the plugin */ - if(plugin->hdf5.filter == NULL) { + if(h5class != NULL && plugin->hdf5.filter == NULL) { plugin->hdf5.filter = h5class; plugin->hdf5.hdf5lib = lib; lib = NULL; } - if(plugin->codec.codec == NULL) { + if(codec != NULL && plugin->codec.codec == NULL) { plugin->codec.codec = codec; plugin->codec.codeclib = lib; lib = NULL; } - #ifdef DEBUGL - if(plugin) + if(plugin) fprintf(stderr,">>> DEBUGL: load_plugin: %s\n",printplugin(plugin)); #endif - - /* Cleanup */ - if(plugin->hdf5.hdf5lib == plugin->codec.codeclib) - plugin->codec.codeclib = NULL; + if(plugin->hdf5.hdf5lib == plugin->codec.codeclib) /* Works for NULL case also */ + plugin->codec.codeclib = NULL; if((stat=NCZ_plugin_save(h5id,plugin))) goto done; plugin = NULL; done: - if(lib) { - (void)ncpsharedlibfree(lib); - } + if(lib) + (void)ncpsharedlibfree(lib); if(plugin) NCZ_unload_plugin(plugin); return ZUNTRACEX(stat,"plug=%p",*plugp); } @@ -1541,7 +1593,6 @@ pluginnamecheck(const char* name) } #endif - /**************************************************/ /* _Codecs attribute */ @@ -1552,23 +1603,51 @@ NCZ_codec_attr(const NC_VAR_INFO_T* var, size_t* lenp, void* data) size_t len; char* contents = NULL; NCbytes* buf = NULL; + NCZ_VAR_INFO_T* zvar = (NCZ_VAR_INFO_T*)var->format_var_info; NClist* filters = (NClist*)var->filters; - + NClist* incfilters = (NClist*)zvar->incompletefilters; + NCZ_Filter** chain = NULL; + size_t nfilters; + ZTRACE(6,"var=%s",var->hdr.name); - if(nclistlength(filters) == 0) {stat = NC_ENOTATT; goto done;} + + nfilters = nclistlength(filters) + nclistlength(incfilters); + + if(nfilters == 0) + {stat = NC_ENOTATT; goto done;} + + /* Collect all filters in chain order */ + chain = (NCZ_Filter**)calloc(sizeof(NCZ_Filter*),nfilters); + if(chain == NULL) {stat = NC_ENOMEM; goto done;} + + /* Sort the complete and incomplete filters in order in the chain */ + for(i=0;ichainindex] == NULL); + chain[f->chainindex] = f; + } + for(i=0;ichainindex] == NULL); + chain[f->chainindex] = f; + } + + /* Now construct the attribute */ buf = ncbytesnew(); ncbytessetalloc(buf,1024); ncbytescat(buf,"["); - for(i=0;i 0) ncbytescat(buf,","); ncbytescat(buf,spec->codec.codec); } ncbytescat(buf,"]"); + len = ncbyteslength(buf); contents = nclistcontents(buf); if(lenp) *lenp = len; if(data) strncpy((char*)data,contents,len+1); done: + nullfree(chain); ncbytesfree(buf); return ZUNTRACEX(stat,"len=%u data=%p",(unsigned)len,data); } @@ -1577,6 +1656,7 @@ static int ensure_working(const NC_VAR_INFO_T* var, NCZ_Filter* filter) { int stat = NC_NOERR; + if(FILTERINCOMPLETE(filter)) {stat = NC_ENOFILTER; goto done;} if(!(filter->flags & FLAG_WORKING)) { const size_t oldnparams = filter->hdf5.visible.nparams; const unsigned* oldparams = filter->hdf5.visible.params; @@ -1683,7 +1763,9 @@ NCZ_filter_setup(NC_VAR_INFO_T* var) filters = (NClist*)var->filters; for(i=0;iplugin != NULL); + assert(filter != NULL); + if(FILTERINCOMPLETE(filter)) continue; /* ignore these */ + assert(filter->plugin != NULL); assert((filter->flags & FLAG_VISIBLE)); /* Assume visible params are defined */ /* verify */ assert(filter->hdf5.id > 0 && (filter->hdf5.visible.nparams == 0 || filter->hdf5.visible.params != NULL)); diff --git a/libnczarr/zfilter.h b/libnczarr/zfilter.h index 38d8b0c3d8..4a9ae0b101 100644 --- a/libnczarr/zfilter.h +++ b/libnczarr/zfilter.h @@ -20,10 +20,10 @@ #define ENCODING 1 /* list of environment variables to check for plugin roots */ -#define plugin_env "HDF5_PLUGIN_PATH" -#define plugin_dir_unix "/usr/local/hdf5/plugin" -#define plugin_dir_win "%s/hdf5/lib/plugin" -#define win32_root_env "ALLUSERSPROFILE" +#define PLUGIN_ENV "HDF5_PLUGIN_PATH" +#define PLUGIN_DIR_UNIX "/usr/local/hdf5/plugin" +#define PLUGIN_DIR_WIN "%s/hdf5/lib/plugin" +#define WIN32_ROOT_ENV "ALLUSERSPROFILE" /* Opaque */ struct NCZ_Filter; @@ -32,11 +32,11 @@ int NCZ_filter_initialize(void); int NCZ_filter_finalize(void); int NCZ_addfilter(NC_FILE_INFO_T*, NC_VAR_INFO_T* var, unsigned int id, size_t nparams, const unsigned int* params); int NCZ_filter_setup(NC_VAR_INFO_T* var); -int NCZ_filter_freelist(NC_VAR_INFO_T* var); +int NCZ_filter_freelists(NC_VAR_INFO_T* var); int NCZ_codec_freelist(NCZ_VAR_INFO_T* zvar); int NCZ_applyfilterchain(const NC_FILE_INFO_T*, NC_VAR_INFO_T*, NClist* chain, size_t insize, void* indata, size_t* outlen, void** outdata, int encode); int NCZ_filter_jsonize(const NC_FILE_INFO_T*, const NC_VAR_INFO_T*, struct NCZ_Filter* filter, struct NCjson**); -int NCZ_filter_build(const NC_FILE_INFO_T*, NC_VAR_INFO_T* var, const NCjson* jfilter); +int NCZ_filter_build(const NC_FILE_INFO_T*, NC_VAR_INFO_T* var, const NCjson* jfilter, int chainindex); int NCZ_codec_attr(const NC_VAR_INFO_T* var, size_t* lenp, void* data); #endif /*ZFILTER_H*/ diff --git a/libnczarr/zinternal.c b/libnczarr/zinternal.c index 018842d094..ba1b907c4c 100644 --- a/libnczarr/zinternal.c +++ b/libnczarr/zinternal.c @@ -358,7 +358,7 @@ close_vars(NC_GRP_INFO_T *grp) /* Reclaim filters */ if(var->filters != NULL) { - (void)NCZ_filter_freelist(var); + (void)NCZ_filter_freelists(var); } var->filters = NULL; diff --git a/libnczarr/zinternal.h b/libnczarr/zinternal.h index 3bc4b6b131..0e3cec55a4 100644 --- a/libnczarr/zinternal.h +++ b/libnczarr/zinternal.h @@ -185,6 +185,7 @@ typedef struct NCZ_VAR_INFO { struct NCZChunkCache* cache; struct NClist* xarray; /* names from _ARRAY_DIMENSIONS */ char dimension_separator; /* '.' | '/' */ + NClist* incompletefilters; } NCZ_VAR_INFO_T; /* Struct to hold ZARR-specific info for a field. */ diff --git a/libnczarr/zsync.c b/libnczarr/zsync.c index d42a311056..d5cbee966a 100644 --- a/libnczarr/zsync.c +++ b/libnczarr/zsync.c @@ -1486,6 +1486,7 @@ define_vars(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, NClist* varnames) NClist* dimnames = nclistnew(); #ifdef ENABLE_NCZARR_FILTERS NCjson* jfilter = NULL; + int chainindex; #endif zinfo = file->format_file_info; @@ -1654,8 +1655,10 @@ define_vars(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, NClist* varnames) /* Do filters key before compressor key so final filter chain is in correct order */ { if(var->filters == NULL) var->filters = (void*)nclistnew(); + if(zvar->incompletefilters == NULL) zvar->incompletefilters = (void*)nclistnew(); #ifdef ENABLE_NCZARR_FILTERS { int k; + chainindex = 0; /* track location of filter in the chain */ if((stat = NCZ_filter_initialize())) goto done; if((stat = NCJdictget(jvar,"filters",&jvalue))) goto done; if(jvalue != NULL && NCJsort(jvalue) != NCJ_NULL) { @@ -1665,7 +1668,7 @@ define_vars(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, NClist* varnames) jfilter = NCJith(jvalue,k); if(jfilter == NULL) break; /* done */ if(NCJsort(jfilter) != NCJ_DICT) {stat = NC_EFILTER; goto done;} - if((stat = NCZ_filter_build(file,var,jfilter))) goto done; + if((stat = NCZ_filter_build(file,var,jfilter,chainindex++))) goto done; } } } @@ -1682,7 +1685,7 @@ define_vars(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, NClist* varnames) if((stat = NCJdictget(jvar,"compressor",&jfilter))) goto done; if(jfilter != NULL && NCJsort(jfilter) != NCJ_NULL) { if(NCJsort(jfilter) != NCJ_DICT) {stat = NC_EFILTER; goto done;} - if((stat = NCZ_filter_build(file,var,jfilter))) goto done; + if((stat = NCZ_filter_build(file,var,jfilter,chainindex++))) goto done; } #endif } diff --git a/nc_test/Makefile.am b/nc_test/Makefile.am index e102346c6c..39ef40596f 100644 --- a/nc_test/Makefile.am +++ b/nc_test/Makefile.am @@ -7,10 +7,10 @@ # Ed Hartnett, Dennis Heimbigner, Ward Fisher # Un comment to use a more verbose test driver -SH_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose -sh_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose -LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose -TESTS_ENVIRONMENT = export SETX=1; +#SH_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose +#sh_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose +#LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose +#TESTS_ENVIRONMENT = export SETX=1; # Put together AM_CPPFLAGS and AM_LDFLAGS. include $(top_srcdir)/lib_flags.am diff --git a/nc_test/run_pnetcdf_tests.sh.in b/nc_test/run_pnetcdf_tests.sh.in index 2908f8d3f3..9ddb532afe 100755 --- a/nc_test/run_pnetcdf_tests.sh.in +++ b/nc_test/run_pnetcdf_tests.sh.in @@ -2,7 +2,6 @@ # This script runs some PnetCDF I/O tests -set -x set -e echo diff --git a/nc_test4/CMakeLists.txt b/nc_test4/CMakeLists.txt index e5962e332f..df4980265b 100644 --- a/nc_test4/CMakeLists.txt +++ b/nc_test4/CMakeLists.txt @@ -50,6 +50,7 @@ IF(USE_HDF5 AND ENABLE_FILTER_TESTING) build_bin_test(tst_filter_avail) build_bin_test(test_filter_vlen) ADD_SH_TEST(nc_test4 tst_filter) + ADD_SH_TEST(nc_test4 tst_unknown) ADD_SH_TEST(nc_test4 tst_specific_filters) IF(ENABLE_CLIENTSIDE_FILTERS) add_bin_test(nc_test4 test_filter_reg) diff --git a/nc_test4/Makefile.am b/nc_test4/Makefile.am index 7ce3b4c603..ef5cdb4d39 100644 --- a/nc_test4/Makefile.am +++ b/nc_test4/Makefile.am @@ -83,6 +83,7 @@ extradir = check_PROGRAMS += test_filter test_filter_misc test_filter_order test_filter_repeat test_filter_vlen check_PROGRAMS += tst_multifilter tst_filter_avail TESTS += tst_filter.sh +TESTS += tst_unknown.sh TESTS += tst_specific_filters.sh endif endif # USE_HDF5 @@ -109,8 +110,8 @@ ref_filteredvv.cdl ref_multi.cdl \ ref_ncgenF.cdl ref_nccopyF.cdl \ ref_filter_repeat.txt ref_fillonly.cdl test_fillonly.sh \ ref_filter_order_create.txt ref_filter_order_read.txt \ -ref_any.cdl tst_specific_filters.sh \ -tst_virtual_datasets.c noop1.cdl +ref_any.cdl tst_specific_filters.sh tst_unknown.sh \ +tst_virtual_datasets.c noop1.cdl unknown.cdl # The tst_filterinstall test can only be run after an install # occurred with --with-plugin-dir enabled. So there is no point @@ -129,7 +130,7 @@ floats*.nc floats*.cdl shorts*.nc shorts*.cdl ints*.nc ints*.cdl \ testfilter_reg.nc filterrepeat.txt tmp_fillonly.nc \ testfilter_order.nc crfilterorder.txt rdfilterorder.txt 1 \ tmp_*.txt tmp_*.nc tmp*.dump tmp*.cdl tmp*.txt tmp*.tmp \ -tmp_bzip2.c bzip2.nc noop.nc +tmp_bzip2.c bzip2.nc noop.nc tmp_*.dmp DISTCLEANFILES = findplugin.sh run_par_test.sh diff --git a/nc_test4/findplugin.in b/nc_test4/findplugin.in old mode 100644 new mode 100755 index a8f2a4e3b8..7146bb0382 --- a/nc_test4/findplugin.in +++ b/nc_test4/findplugin.in @@ -20,27 +20,43 @@ # is obviated by setting the LIBRARY_OUTPUT_DIRECTORY # variables: see hdf5plugins/CMakeLists.txt -findplugin() { +# Define location of execution +TOPSRCDIR='@abs_top_srcdir@' +TOPBUILDDIR='@abs_top_builddir@' -FP_NAME="$1" +# Need info from test_common +if test "x$srcdir" = x ; then srcdir=`pwd`; fi +. ${TOPBUILDDIR}/test_common.sh -FP_PLUGIN_LIB= +findpluginext() { + # Infer the expected plugin shared library extension + # Note: will fail if used before plugins is built + # Also assumes that misc filter is always built + # Approach is to use find to see what is in plugins directory. + TSO=`find ${TOPBUILDDIR}/plugins -name '*misc.so'` + TDY=`find ${TOPBUILDDIR}/plugins -name '*misc.dylib'` + TCYG=`find ${TOPBUILDDIR}/plugins -name 'cyg*misc.dll'` + TDLL=`find ${TOPBUILDDIR}/plugins -name '*misc.dll'` + if test "x$TSO" != x ; then + FP_PLUGIN_EXT="so" + FP_PLUGIN_PRE="lib__nc" + elif test "x$TDY" != x ; then + FP_PLUGIN_EXT="dylib" + FP_PLUGIN_PRE="lib__nc" + elif test "x$TCYG" != x ; then + FP_PLUGIN_EXT="dll" + FP_PLUGIN_PRE="cyg__nc" + elif test "x$TDLL" != x ; then + FP_PLUGIN_EXT="dll" + FP_PLUGIN_PRE="__nc" + else # unknown + unset FP_PLUGIN_EXT + unset FP_PLUGIN_PRE + fi +} + +findplugindir() { FP_PLUGIN_DIR= - -# Figure out the plugin file name -# Test for visual studio before cygwin since both might be true -if test "x$FP_ISMSVC" != x ; then - FP_PLUGIN_LIB="${FP_NAME}.dll" -elif test "x$FP_ISMINGW" != x || test "x$FP_ISMSYS" != x ; then - FP_PLUGIN_LIB="${FP_NAME}.dll" -elif test "x$FP_ISCYGWIN" != x ; then - FP_PLUGIN_LIB="cyg${FP_NAME}.dll" -elif test "x$FP_ISOSX" != x ; then - FP_PLUGIN_LIB="lib${FP_NAME}.so" # Should this include the version number in the name? -else # Presumably some form on *nix" - FP_PLUGIN_LIB="lib${FP_NAME}.so" -fi - # Figure out the path to where the lib is stored # This can probably be simplified @@ -52,15 +68,15 @@ cd ${CURWD} # Case 1: Cmake with Visual Studio if test "x$FP_ISCMAKE" != x -a "x${FP_ISMSVC}" != x ; then # Case 1a: ignore the build type directory - if test -f "${FP_PLUGINS}/${FP_PLUGIN_LIB}" ; then + if test -e "${FP_PLUGINS}/${FP_PLUGIN_LIB}" ; then FP_PLUGIN_DIR="${FP_PLUGINS}" fi else # Case 2: automake # Case 2a: look in .libs - if test -f "${FP_PLUGINS}/.libs/${FP_PLUGIN_LIB}" ; then + if test -e "${FP_PLUGINS}/.libs" ; then FP_PLUGIN_DIR="${FP_PLUGINS}/.libs" else # Case 2: look in FP_PLUGINS directly - if test -f "${FP_PLUGINS}/${FP_PLUGIN_LIB}" ; then + if test -e "${FP_PLUGINS}" ; then FP_PLUGIN_DIR="${FP_PLUGINS}" fi fi @@ -71,16 +87,35 @@ if test "x$FP_PLUGIN_DIR" = x ; then echo "***Fail: Could not locate a usable HDF5_PLUGIN_DIR" return 1 fi + +# Make local path +FP_PLUGIN_DIR=`${NCPATHCVT} -F $FP_PLUGIN_DIR` +HDF5_PLUGIN_DIR="$FP_PLUGIN_DIR" +} + +findplugin() { + +FP_NAME="$1" + +FP_PLUGIN_LIB= + +# Figure out the plugin file name +FP_PLUGIN_LIB="${FP_PLUGIN_PRE}${FP_NAME}.${FP_PLUGIN_EXT}" + +# Verify if ! test -f "$FP_PLUGIN_DIR/$FP_PLUGIN_LIB" ; then echo "***Fail: Could not locate a usable HDF5_PLUGIN_LIB" return 1 fi -FP_PLUGIN_DIR=`${NCPATHCVT} $FP_PLUGIN_DIR` - # Set the final output variables HDF5_PLUGIN_LIB="$FP_PLUGIN_LIB" HDF5_PLUGIN_DIR="$FP_PLUGIN_DIR" return 0 } + +# init +unset HDF5_PLUGIN_DIR +findpluginext +findplugindir diff --git a/nc_test4/tst_filter.sh b/nc_test4/tst_filter.sh index b478079092..e777956d0b 100755 --- a/nc_test4/tst_filter.sh +++ b/nc_test4/tst_filter.sh @@ -9,7 +9,6 @@ set -e API=1 NG=1 NCP=1 -UNK=1 NGC=1 MISC=1 MULTI=1 @@ -41,18 +40,6 @@ trimleft() { sed -e 's/[ ]*\([^ ].*\)/\1/' <$1 >$2 } -# Hide/unhide the noop filter -hidenoop() { - rm -fr ${HDF5_PLUGIN_DIR}/save - mkdir ${HDF5_PLUGIN_DIR}/save - mv ${NOOPDIR} ${HDF5_PLUGIN_DIR}/save -} - -unhidenoop() { - mv ${HDF5_PLUGIN_DIR}/save/${NOOPLIB} ${HDF5_PLUGIN_DIR} - rm -fr ${HDF5_PLUGIN_DIR}/save -} - # Locate the plugin dir and the library names; argument order is critical # Find bzip2 and capture findplugin h5bzip2 @@ -180,36 +167,6 @@ echo " *** Pass: -F var,none" echo "*** Pass: all nccopy filter tests" fi -if test "x$UNK" = x1 ; then -echo "*** Testing access to filter info when filter dll is not available" -rm -f noop.nc ./tmp_filter.txt -# xfail build noop.nc -hidenoop -if ${NCGEN} -lb -4 -o noop.nc ${srcdir}/noop.cdl ; then - echo "*** FAIL: ncgen" -else - echo "*** XFAIL: ncgen" -fi -unhidenoop -# build noop.nc -${NCGEN} -lb -4 -o noop.nc ${srcdir}/noop.cdl -# Now hide the filter code -hidenoop -rm -f ./tmp_filter.txt -# This will xfail -if ${NCDUMP} -s noop.nc > ./tmp_filter.txt ; then - echo "*** FAIL: ncdump -hs noop.nc" -else - echo "*** XFAIL: ncdump -hs noop.nc" -fi -# Restore the filter code -unhidenoop -# Verify we can see filter when using -h -rm -f ./tmp_filter.txt -${NCDUMP} -hs noop.nc > ./tmp_filter.txt -echo "*** Pass: unknown filter" -fi - if test "x$NGC" = x1 ; then rm -f ./tmp_bzip2.c echo "*** Testing dynamic filters using ncgen with -lc" diff --git a/nc_test4/tst_filterinstall.sh b/nc_test4/tst_filterinstall.sh index 465a42141d..9b119d88e9 100755 --- a/nc_test4/tst_filterinstall.sh +++ b/nc_test4/tst_filterinstall.sh @@ -1,6 +1,8 @@ #!/bin/bash # Test the filter install +# This cannot be run as a regular test +# because installation will not have occurred if test "x$srcdir" = x ; then srcdir=`pwd`; fi . ../test_common.sh diff --git a/nc_test4/tst_unknown.sh b/nc_test4/tst_unknown.sh new file mode 100755 index 0000000000..76a7b1b6ab --- /dev/null +++ b/nc_test4/tst_unknown.sh @@ -0,0 +1,108 @@ +#!/bin/bash + +if test "x$srcdir" = x ; then srcdir=`pwd`; fi +. ../test_common.sh + +if test "x$TESTNCZARR" = x1 ; then +. "$srcdir/test_nczarr.sh" +fi + +set -e + +# Load the findplugins function +. ${builddir}/findplugin.sh +echo "findplugin.sh loaded" + +# Set up HDF5_PLUGIN_PATH +export HDF5_PLUGIN_PATH=${HDF5_PLUGIN_DIR} + +# Test operation with an unknown filter + +# Function to remove selected -s attributes from file; +# These attributes might be platform dependent +sclean() { + cat $1 \ + | sed -e '/:_IsNetcdf4/d' \ + | sed -e '/:_Endianness/d' \ + | sed -e '/_NCProperties/d' \ + | sed -e '/_SuperblockVersion/d' \ + | cat > $2 +} + +# Function to extract _Filter attribute from a file +# These attributes might be platform dependent +getfilterattr() { +sed -e '/var.*:_Filter/p' -ed <$1 >$2 +} + +# Function to extract _Codecs attribute from a file +# These attributes might be platform dependent +getcodecsattr() { +sed -e '/var.*:_Codecs/p' -ed <$1 >$2 +} + +trimleft() { +sed -e 's/[ ]*\([^ ].*\)/\1/' <$1 >$2 +} + +# Locate the plugin path and the library names; argument order is critical + +# Find noop and capture +findplugin h5unknown +UNKNOWNDIR="${HDF5_PLUGIN_DIR}" +UNKNOWNLIB="${HDF5_PLUGIN_LIB}" +UNKNOWNFILTER="${HDF5_PLUGIN_DIR}/${UNKNOWNLIB}" + +# Getting the name is especially tricky for dylib, which puts the version before the .dylib + +# Verify +if ! test -f ${UNKNOWNFILTER} ; then echo "Unable to locate ${UNKNOWNFILTER}"; exit 1; fi + +testunk() { +zext=$1 +echo "*** Testing access to filter info when filter implementation is not available for map $zext" +if test "x$TESTNCZARR" = x1 ; then +fileargs tmp_known +deletemap $zext $file +else +file="tmp_known_${zfilt}.nc" +rm -f $file +fileurl="$file" +fi +# build .nc file using unknown +${NCGEN} -lb -4 -o $fileurl ${srcdir}/../nc_test4/unknown.cdl +# dump and clean file when filter is avail +${NCDUMP} -hs $fileurl > ./tmp_known_$zext.txt +# Remove irrelevant -s output +sclean ./tmp_known_$zext.txt tmp_known_$zext.dump +# Hide the filter +rm -fr ${UNKNOWNDIR}/save +mkdir -p ${UNKNOWNDIR}/save +# Figure out all matching libs; make sure to remove .so, so.0, etc +LSRC=`${execdir}/../ncdump/ncpathcvt -F "${UNKNOWNDIR}"` +LDST=`${execdir}/../ncdump/ncpathcvt -F ${UNKNOWNDIR}/save` +mv ${LSRC}/*unknown* ${LDST} +# Verify that the filter is no longer defined +# Try to read the data; should xfail +if ${NCDUMP} -s $fileurl > ./tmp_unk_$zext.dmp ; then +echo "*** FAIL: filter found" +found=1 +else +echo "*** XFAIL: filter not found" +found=0 +fi +# Restore the filter +mv ${LDST}/*unknown* ${LSRC} +rm -fr ${UNKNOWNDIR}/save +if test "x$found" = x1 ; then exit 1; fi +} + +if test "x$TESTNCZARR" = x1 ; then + testunk file + if test "x$FEATURE_NCZARR_ZIP" = xyes ; then testunk zip ; fi + if test "x$FEATURE_S3TESTS" = xyes ; then testunk s3 ; fi +else +testunk nc +fi + +exit 0 diff --git a/nc_test4/unknown.cdl b/nc_test4/unknown.cdl new file mode 100644 index 0000000000..f3c4c8f39e --- /dev/null +++ b/nc_test4/unknown.cdl @@ -0,0 +1,83 @@ +netcdf bzip2 { +dimensions: + dim0 = 4 ; + dim1 = 4 ; + dim2 = 4 ; + dim3 = 4 ; +variables: + float var(dim0, dim1, dim2, dim3) ; + var:_Storage = "chunked" ; + var:_ChunkSizes = 4, 4, 4, 4 ; + var:_Filter = "33000" ; + var:_NoFill = "true" ; + +// global attributes: + :_Format = "netCDF-4" ; +data: + + var = + 0, 1, 2, 3, + 4, 5, 6, 7, + 8, 9, 10, 11, + 12, 13, 14, 15, + 16, 17, 18, 19, + 20, 21, 22, 23, + 24, 25, 26, 27, + 28, 29, 30, 31, + 32, 33, 34, 35, + 36, 37, 38, 39, + 40, 41, 42, 43, + 44, 45, 46, 47, + 48, 49, 50, 51, + 52, 53, 54, 55, + 56, 57, 58, 59, + 60, 61, 62, 63, + 64, 65, 66, 67, + 68, 69, 70, 71, + 72, 73, 74, 75, + 76, 77, 78, 79, + 80, 81, 82, 83, + 84, 85, 86, 87, + 88, 89, 90, 91, + 92, 93, 94, 95, + 96, 97, 98, 99, + 100, 101, 102, 103, + 104, 105, 106, 107, + 108, 109, 110, 111, + 112, 113, 114, 115, + 116, 117, 118, 119, + 120, 121, 122, 123, + 124, 125, 126, 127, + 128, 129, 130, 131, + 132, 133, 134, 135, + 136, 137, 138, 139, + 140, 141, 142, 143, + 144, 145, 146, 147, + 148, 149, 150, 151, + 152, 153, 154, 155, + 156, 157, 158, 159, + 160, 161, 162, 163, + 164, 165, 166, 167, + 168, 169, 170, 171, + 172, 173, 174, 175, + 176, 177, 178, 179, + 180, 181, 182, 183, + 184, 185, 186, 187, + 188, 189, 190, 191, + 192, 193, 194, 195, + 196, 197, 198, 199, + 200, 201, 202, 203, + 204, 205, 206, 207, + 208, 209, 210, 211, + 212, 213, 214, 215, + 216, 217, 218, 219, + 220, 221, 222, 223, + 224, 225, 226, 227, + 228, 229, 230, 231, + 232, 233, 234, 235, + 236, 237, 238, 239, + 240, 241, 242, 243, + 244, 245, 246, 247, + 248, 249, 250, 251, + 252, 253, 254, 255 ; +} diff --git a/ncdump/ncpathcvt.c b/ncdump/ncpathcvt.c index 3e14916e56..01beaf812d 100755 --- a/ncdump/ncpathcvt.c +++ b/ncdump/ncpathcvt.c @@ -29,12 +29,13 @@ #include "ncpathmgr.h" static const char* USAGE = -"ncpathcvt [-c|-C|-m|-u|-w] [-h] [-e] [-d ] [-B] [-k] [-p] PATH\n" +"ncpathcvt [-c|-C|-m|-u|-w] [-h] [-e] [-F] [-d ] [-B] [-k] [-p] PATH\n" "Options\n" " -h help" " -e add backslash escapes to '\' and ' '\n" " -d use driveletter when needed; defaults to 'c'\n" -" -B convert occurrences of to ' '\n" +" -B convert occurrences of to blank\n" +" -F convert occurrences of '\\' to '/'" "Output type options:\n" " -c convert to Cygwin form of path\n" " -C return canonical form of path\n" @@ -57,6 +58,7 @@ struct Options { int debug; int canon; int blank; + int slash; int pathkind; } cvtoptions; @@ -93,6 +95,27 @@ escape(const char* path) return epath; } +static char* +slash(const char* path) +{ + size_t slen = strlen(path); + const char* p; + char* q; + char* epath = NULL; + + epath = (char*)malloc(slen + 1); + if(epath == NULL) usage("out of memtory"); + p = path; + q = epath; + for(;*p;p++) { + if(*p == '\\') + *q++ = '/'; + else *q++ = *p; + } + *q = '\0'; + return epath; +} + void printlocalkind(void) { @@ -173,7 +196,7 @@ main(int argc, char** argv) memset((void*)&cvtoptions,0,sizeof(cvtoptions)); cvtoptions.drive = 'c'; - while ((c = getopt(argc, argv, "B:CcD:d:ehkmpuwX")) != EOF) { + while ((c = getopt(argc, argv, "B:CFcD:d:ehkmpuwX")) != EOF) { switch(c) { case 'c': cvtoptions.target = NCPD_CYGWIN; break; case 'd': cvtoptions.drive = optarg[0]; break; @@ -190,6 +213,7 @@ main(int argc, char** argv) usage("Bad -B argument"); break; case 'C': cvtoptions.canon = 1; break; + case 'F': cvtoptions.slash = 1; break; case 'D': sscanf(optarg,"%d",&cvtoptions.debug); break; @@ -245,6 +269,11 @@ main(int argc, char** argv) cvtpath = escape(path); free(path); } + if(cvtpath && cvtoptions.slash) { + char* path = cvtpath; cvtpath = NULL; + cvtpath = slash(path); + free(path); + } printf("%s",cvtpath); done: if(canon) free(canon); diff --git a/nctest/compare_test_files.sh b/nctest/compare_test_files.sh index f9712adbe1..b0d7b780f6 100755 --- a/nctest/compare_test_files.sh +++ b/nctest/compare_test_files.sh @@ -5,7 +5,7 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi set -e -WS=`${NCPATHCVT} ${srcdir}` +WS=`${NCPATHCVT} -F ${srcdir}` cmp nctest_classic.nc "$WS/ref_nctest_classic.nc" cmp nctest_64bit_offset.nc "$WS/ref_nctest_64bit_offset.nc" diff --git a/nczarr_test/CMakeLists.txt b/nczarr_test/CMakeLists.txt index ef31d15df1..a19eaa0557 100644 --- a/nczarr_test/CMakeLists.txt +++ b/nczarr_test/CMakeLists.txt @@ -26,6 +26,15 @@ FILE(COPY ${CMAKE_CURRENT_SOURCE_DIR}/run_specific_filters.sh DESTINATION ${CMAK FILE(REMOVE ${CMAKE_CURRENT_SOURCE_DIR}/run_specific_filters.1) FILE(REMOVE ${CMAKE_CURRENT_SOURCE_DIR}/run_specific_filters.sh) +FILE(READ ${CMAKE_CURRENT_SOURCE_DIR}/../nc_test4/tst_unknown.sh SPSOURCE) +STRING(PREPEND SPSOURCE "#!/bin/bash\n") +STRING(PREPEND SPSOURCE "TESTNCZARR=1\n") +FILE(WRITE ${CMAKE_CURRENT_SOURCE_DIR}/run_unknown.1 "${SPSOURCE}") +CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/run_unknown.1 ${CMAKE_CURRENT_SOURCE_DIR}/run_unknown.sh @ONLY NEWLINE_STYLE LF) +FILE(COPY ${CMAKE_CURRENT_SOURCE_DIR}/run_unknown.sh DESTINATION ${CMAKE_CURRENT_BINARY_DIR} FILE_PERMISSIONS OWNER_WRITE OWNER_READ OWNER_EXECUTE) +FILE(REMOVE ${CMAKE_CURRENT_SOURCE_DIR}/run_unknown.1) +FILE(REMOVE ${CMAKE_CURRENT_SOURCE_DIR}/run_unknown.sh) + FILE(GLOB COPY_FILES ${CMAKE_CURRENT_SOURCE_DIR}/*.sh ${CMAKE_CURRENT_SOURCE_DIR}/ref*.cdl ${CMAKE_CURRENT_SOURCE_DIR}/ref*.txt) @@ -127,6 +136,7 @@ IF(ENABLE_TESTS) build_bin_test(test_filter_avail) ADD_SH_TEST(nczarr_test run_nczfilter) ADD_SH_TEST(nczarr_test run_filter) + ADD_SH_TEST(nczarr_test run_unknown) ADD_SH_TEST(nczarr_test run_specific_filters) ENDIF(ENABLE_FILTER_TESTING) if(ENABLE_NCZARR_ZIP) diff --git a/nczarr_test/Makefile.am b/nczarr_test/Makefile.am index c413fcf91e..ca43d28b35 100644 --- a/nczarr_test/Makefile.am +++ b/nczarr_test/Makefile.am @@ -10,10 +10,10 @@ TESTS_ENVIRONMENT = #TEST_EXTENSIONS = .sh -SH_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose -sh_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose -LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose -TESTS_ENVIRONMENT += export SETX=1; +#SH_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose +#sh_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose +#LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose +#TESTS_ENVIRONMENT += export SETX=1; AM_CPPFLAGS += -I${top_srcdir} -I${top_srcdir}/libnczarr AM_LDFLAGS += ${top_builddir}/liblib/libnetcdf.la @@ -95,6 +95,7 @@ TESTS += run_nczfilter.sh # Echo filter tests from nc_test4 check_PROGRAMS += testfilter testfilter_misc testfilter_order testfilter_repeat testfilter_multi test_filter_avail TESTS += run_filter.sh +TESTS += run_unknown.sh TESTS += run_specific_filters.sh endif #ENABLE_FILTER_TESTING @@ -128,9 +129,8 @@ run_ut_map.sh run_ut_mapapi.sh run_ut_misc.sh run_ut_chunk.sh run_ncgen4.sh \ run_nccopyz.sh run_fillonlyz.sh run_chunkcases.sh test_nczarr.sh run_perf_chunks1.sh run_s3_cleanup.sh \ run_purezarr.sh run_interop.sh run_misc.sh \ run_filter.sh run_specific_filters.sh \ -run_newformat.sh run_nczarr_fill.sh run_quantize.sh run_jsonconvention.sh \ -run_nczfilter.sh - +run_newformat.sh run_nczarr_fill.sh run_quantize.sh \ +run_jsonconvention.sh run_nczfilter.sh run_unknown.sh EXTRA_DIST += \ ref_ut_map_create.cdl ref_ut_map_writedata.cdl ref_ut_map_writemeta2.cdl ref_ut_map_writemeta.cdl \ @@ -167,6 +167,15 @@ test_filter_avail.c: $(top_srcdir)/nc_test4/tst_filter_avail.c echo "#define TESTNCZARR" > $@ cat $(top_srcdir)/nc_test4/tst_filter_avail.c >> $@ +run_unknown.sh: $(top_srcdir)/nc_test4/tst_unknown.sh + rm -f $@ run_unknown.tmp + echo "#!/bin/bash" > run_unknown.tmp + echo "TESTNCZARR=1" >> run_unknown.tmp + cat $(top_srcdir)/nc_test4/tst_unknown.sh >> run_unknown.tmp + tr -d '\r' < run_unknown.tmp > $@ + chmod a+x $@ + rm -f run_unknown.tmp + run_specific_filters.sh: $(top_srcdir)/nc_test4/tst_specific_filters.sh rm -f $@ run_specific_filters.tmp echo "#!/bin/bash" > run_specific_filters.tmp diff --git a/nczarr_test/ref_multi.cdl b/nczarr_test/ref_multi.cdl index a3b41f6d63..b81c63c020 100644 --- a/nczarr_test/ref_multi.cdl +++ b/nczarr_test/ref_multi.cdl @@ -9,7 +9,7 @@ variables: var:_Storage = "chunked" ; var:_ChunkSizes = 4, 4, 4, 4 ; var:_Filter = "307,9|1,2|40000" ; - var:_Codecs = "[{\"id\": \"bz2\",\"level\": \"9\"},{\"id\": \"zlib\",\"level\": \"2\"},{\"id\": \"noop0\"}]" ; + var:_Codecs = "[{\"id\": \"bz2\",\"level\": \"9\"},{\"id\": \"zlib\",\"level\": \"2\"},{\"id\": \"noop\"}]" ; var:_NoFill = "true" ; // global attributes: diff --git a/nczarr_test/run_filter.sh b/nczarr_test/run_filter.sh index 04c4291409..0bf6437d15 100755 --- a/nczarr_test/run_filter.sh +++ b/nczarr_test/run_filter.sh @@ -50,16 +50,19 @@ sed -e 's/[ ]*\([^ ].*\)/\1/' <$1 >$2 # Find misc and capture findplugin h5misc -MISCDIR="${HDF5_PLUGIN_DIR}/${HDF5_PLUGIN_LIB}" +MISCLIB="${HDF5_PLUGIN_LIB}" +MISCDIR="${HDF5_PLUGIN_DIR}" +MISCPATH="${MISCDIR}/${MISCLIB}" # Find bzip2 and capture findplugin h5bzip2 BZIP2LIB="${HDF5_PLUGIN_LIB}" -BZIP2DIR="${HDF5_PLUGIN_DIR}/${BZIP2LIB}" +BZIP2DIR="${HDF5_PLUGIN_DIR}" +BZIP2PATH="${BZIP2DIR}/${BZIP2LIB}" # Verify -if ! test -f ${BZIP2DIR} ; then echo "Unable to locate ${BZIP2DIR}"; exit 1; fi -if ! test -f ${MISCDIR} ; then echo "Unable to locate ${MISCDIR}"; exit 1; fi +if ! test -f ${BZIP2path} ; then echo "Unable to locate ${BZIP2PATH}"; exit 1; fi +if ! test -f ${MISCPATH} ; then echo "Unable to locate ${MISCPATH}"; exit 1; fi # Execute the specified tests diff --git a/plugins/CMakeLists.txt b/plugins/CMakeLists.txt index e606b392a3..8ca75c8eb6 100644 --- a/plugins/CMakeLists.txt +++ b/plugins/CMakeLists.txt @@ -6,6 +6,18 @@ # See netcdf-c/COPYRIGHT file for more info. SET(CMAKE_BUILD_TYPE "") +IF(MSVC) + SET(PLUGINEXT "dll") + SET(PLUGINPRE "__nc") +ELSE() + SET(PLUGINPRE "lib__nc") + if(APPLE) + SET(PLUGINEXT "dylib") + ELSE() + SET(PLUGINEXT "so") + ENDIF() +ENDIF() + FILE(READ H5Znoop.c NOOP_SOURCE) FILE(WRITE ${CMAKE_CURRENT_BINARY_DIR}/H5Znoop1.c "#define NOOP_INSTANCE 1\n") FILE(APPEND ${CMAKE_CURRENT_BINARY_DIR}/H5Znoop1.c "${NOOP_SOURCE}") @@ -18,6 +30,8 @@ SET_SOURCE_FILES_PROPERTIES(H5Znoop.c PROPERTIES COMPILE_OPTIONS -DNOOP_INSTANCE SET(h5noop1_SOURCES ${CMAKE_CURRENT_BINARY_DIR}/H5Znoop1.c H5Zutil.c h5noop.h) SET_SOURCE_FILES_PROPERTIES(${CMAKE_CURRENT_BINARY_DIR}/H5Znoop1.c PROPERTIES COMPILE_OPTIONS -DNOOP_INSTANCE=1) +SET(h5unknown_SOURCES H5Zunknown.c) + SET(h5shuffle_SOURCES H5Zshuffle.c) SET(h5fletcher32_SOURCES H5Zfletcher32.c H5checksum.c) SET(h5deflate_SOURCES H5Zdeflate.c) @@ -34,10 +48,12 @@ SET(CMAKE_LIBRARY_OUTPUT_DIRECTORY_DEBUG "${CMAKE_CURRENT_BINARY_DIR}") SET(CMAKE_LIBRARY_OUTPUT_DIRECTORY_RELEASE "${CMAKE_CURRENT_BINARY_DIR}") MACRO(buildplugin TARGET TARGETLIB) + SET(MANGLELIB "${PLUGINPRE}${TARGETLIB}") ADD_LIBRARY(${TARGET} MODULE ${${TARGET}_SOURCES}) - SET_TARGET_PROPERTIES(${TARGET} PROPERTIES LIBRARY_OUTPUT_NAME ${TARGETLIB}) - SET_TARGET_PROPERTIES(${TARGET} PROPERTIES ARCHIVE_OUTPUT_NAME ${TARGETLIB}) - SET_TARGET_PROPERTIES(${TARGET} PROPERTIES RUNTIME_OUTPUT_NAME ${TARGETLIB}) + SET_TARGET_PROPERTIES(${TARGET} PROPERTIES LIBRARY_OUTPUT_NAME ${MANGLELIB}) + SET_TARGET_PROPERTIES(${TARGET} PROPERTIES ARCHIVE_OUTPUT_NAME ${MANGLELIB}) + SET_TARGET_PROPERTIES(${TARGET} PROPERTIES RUNTIME_OUTPUT_NAME ${MANGLELIB}) + SET_TARGET_PROPERTIES(${TARGET} PROPERTIES PREFIX "") # Critical that this be set to "" TARGET_LINK_LIBRARIES(${TARGET} ${ALL_TLL_LIBS};${ARGN}) IF(MSVC) target_compile_options(${TARGET} PRIVATE /Zi) @@ -51,14 +67,15 @@ ENDMACRO() buildplugin(h5misc "h5misc") buildplugin(h5noop "h5noop") buildplugin(h5noop1 "h5noop1") +buildplugin(h5unknown "h5unknown") buildplugin(h5shuffle "h5shuffle") buildplugin(h5fletcher32 "h5fletcher32") buildplugin(h5deflate "h5deflate") -buildplugin(nczmisc "nczmisc") -buildplugin(nczhdf5filters "nczhdf5filters" netcdf) -buildplugin(nczstdfilters "nczstdfilters" netcdf) +buildplugin(nczmisc "zmisc") +buildplugin(nczhdf5filters "zhdf5filters" netcdf) +buildplugin(nczstdfilters "zstdfilters" netcdf) IF(ENABLE_BLOSC) SET(h5blosc_SOURCES H5Zblosc.c) @@ -89,13 +106,9 @@ ENDIF() IF(ENABLE_PLUGIN_INSTALL) MACRO(installplugin PLUG) -IF(MSVC) - SET(BUILD_PLUGIN_LIB "${PLUG}.dll") -ELSE() - SET(BUILD_PLUGIN_LIB "lib${PLUG}.so") -ENDIF() -MESSAGE(STATUS "Installing: ${BUILD_PLUGIN_LIB} into ${PLUGIN_INSTALL_DIR}") -install(PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/${BUILD_PLUGIN_LIB} DESTINATION ${PLUGIN_INSTALL_DIR}) + SET(INSTALLED_PLUGIN_LIB "${PLUGINPRE}${PLUG}.${PLUGINEXT}") + MESSAGE(STATUS "Installing: ${INSTALLED_PLUGIN_LIB} into ${PLUGIN_INSTALL_DIR}") + install(PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/${INSTALLED_PLUGIN_LIB} DESTINATION ${PLUGIN_INSTALL_DIR}) ENDMACRO() install(DIRECTORY DESTINATION ${PLUGIN_INSTALL_DIR}) @@ -112,8 +125,8 @@ IF(ENABLE_NCZARR) installplugin(h5fletcher32) installplugin(h5shuffle) installplugin(h5deflate) - installplugin(nczhdf5filters) - installplugin(nczstdfilters) + installplugin(zhdf5filters) + installplugin(zstdfilters) IF(Szip_FOUND) installplugin(h5szip) ENDIF() diff --git a/plugins/H5Znoop.c b/plugins/H5Znoop.c index 03fa6d6bbe..d50fe14233 100644 --- a/plugins/H5Znoop.c +++ b/plugins/H5Znoop.c @@ -154,7 +154,7 @@ static NCZ_codec_t NCZ_noop_codec = {/* NCZ_codec_t codec fields */ NCZ_CODEC_CLASS_VER, /* Struct version number */ NCZ_CODEC_HDF5, /* Struct sort */ #if NOOP_INSTANCE == 0 - "noop0", /* Standard name/id of the codec */ + "noop", /* Standard name/id of the codec */ H5Z_FILTER_NOOP, /* HDF5 alias for noop */ #else "noop1", /* Standard name/id of the codec */ diff --git a/plugins/H5Zunknown.c b/plugins/H5Zunknown.c new file mode 100644 index 0000000000..9cd8af135f --- /dev/null +++ b/plugins/H5Zunknown.c @@ -0,0 +1,147 @@ +#include "config.h" +#include +#include +#include +#include +#include + +#include "netcdf_filter_build.h" + +/* use a temporary filter id*/ +#define H5Z_FILTER_UNKNOWN 33000 + +#undef DEBUG + +static htri_t H5Z_unknown_can_apply(hid_t dcpl_id, hid_t type_id, hid_t space_id); +static size_t H5Z_filter_unknown(unsigned int, size_t, const unsigned int cd_values[], size_t, size_t*, void**); + +static H5Z_class2_t H5Z_UNKNOWN[1] = {{ + H5Z_CLASS_T_VERS, /* H5Z_class_t version */ + (H5Z_filter_t)(H5Z_FILTER_UNKNOWN), /* Filter id number */ + 1, /* encoder_present flag (set to true) */ + 1, /* decoder_present flag (set to true) */ + "unknown", /* Filter name for debugging */ + (H5Z_can_apply_func_t)H5Z_unknown_can_apply, /* The "can apply" callback */ + NULL, /* The "set local" callback */ + (H5Z_func_t)H5Z_filter_unknown, /* The actual filter function */ +}}; + +/* External Discovery Functions */ +DLLEXPORT +H5PL_type_t +H5PLget_plugin_type(void) +{ + return H5PL_TYPE_FILTER; +} + +DLLEXPORT +const void* +H5PLget_plugin_info(void) +{ + return H5Z_UNKNOWN; +} + +/* Make this explicit */ +/* + * The "can_apply" callback returns positive a valid combination, zero for an + * invalid combination and negative for an error. + */ +static htri_t +H5Z_unknown_can_apply(hid_t dcpl_id, hid_t type_id, hid_t space_id) +{ + return 1; /* Assume it can always apply */ +} + +static size_t +H5Z_filter_unknown(unsigned int flags, size_t cd_nelmts, + const unsigned int cd_values[], size_t nbytes, + size_t *buf_size, void **buf) +{ + void* newbuf; + + if (flags & H5Z_FLAG_REVERSE) { + /* Replace buffer */ + newbuf = H5allocate_memory(*buf_size,0); + if(newbuf == NULL) abort(); + memcpy(newbuf,*buf,*buf_size); + /* reclaim old buffer */ + H5free_memory(*buf); + *buf = newbuf; + + } else { + /* Replace buffer */ + newbuf = H5allocate_memory(*buf_size,0); + if(newbuf == NULL) abort(); + memcpy(newbuf,*buf,*buf_size); + /* reclaim old buffer */ + H5free_memory(*buf); + *buf = newbuf; + + } + return *buf_size; +} + +/**************************************************/ +/* NCZarr Codec API */ + +/* Codec Format +{ +"id": "unknown", +} +*/ + +/* Forward */ +static int NCZ_unknown_codec_to_hdf5(const char* codec, size_t* nparamsp, unsigned** paramsp); +static int NCZ_unknown_hdf5_to_codec(size_t nparams, const unsigned* params, char** codecp); + +/* Structure for NCZ_PLUGIN_CODEC */ +static NCZ_codec_t NCZ_unknown_codec = {/* NCZ_codec_t codec fields */ + NCZ_CODEC_CLASS_VER, /* Struct version number */ + NCZ_CODEC_HDF5, /* Struct sort */ + "unknown", /* Standard name/id of the codec */ + H5Z_FILTER_UNKNOWN, /* HDF5 alias for unknown */ + NULL, /*NCZ_unknown_codec_initialize*/ + NULL, /*NCZ_unknown_codec_finalize*/ + NCZ_unknown_codec_to_hdf5, + NCZ_unknown_hdf5_to_codec, + NULL, /*NCZ_unknown_modify_parameters*/ +}; + +/* External Export API */ +DLLEXPORT +const void* +NCZ_get_codec_info(void) +{ + return (void*)&NCZ_unknown_codec; +} + +/* NCZarr Interface Functions */ + +static int +NCZ_unknown_codec_to_hdf5(const char* codec_json, size_t* nparamsp, unsigned** paramsp) +{ + int stat = NC_NOERR; + + *nparamsp = 0; + *paramsp = NULL; + + return stat; +} + +static int +NCZ_unknown_hdf5_to_codec(size_t nparams, const unsigned* params, char** codecp) +{ + int stat = NC_NOERR; + char json[8192]; + + if(nparams != 0 || params != NULL) + {stat = NC_EINVAL; goto done;} + + snprintf(json,sizeof(json),"{\"id\": \"%s\"}",NCZ_unknown_codec.codecid); + if(codecp) { + if((*codecp = strdup(json))==NULL) {stat = NC_ENOMEM; goto done;} + } + +done: + return stat; +} diff --git a/plugins/Makefile.am b/plugins/Makefile.am index ded5b30760..777495ff65 100644 --- a/plugins/Makefile.am +++ b/plugins/Makefile.am @@ -7,8 +7,28 @@ include $(top_srcdir)/lib_flags.am AM_LDFLAGS += -module -avoid-version -shared -export-dynamic \ -rpath ${abs_builddir} ${NOUNDEFINED} -lib_LTLIBRARIES = -noinst_LTLIBRARIES = +# Create an alternate directory if not installing or for noinst installs. +ALTPLUGINDIR = ${abs_top_builddir}/plugins/plugindir + +# This is where the plugins are to be installed +if ENABLE_PLUGIN_DIR +plugindir = @PLUGIN_INSTALL_DIR@ +else +plugindir = ${ALTPLUGINDIR} +endif + +plugin_LTLIBRARIES = + +# Apparently one cannot have plugin_LTLIBRARIES and also noinst_LTLIBRARIES. +# So create a tmp location for "noinst" shared libraries. +tmpdir = ${ALTPLUGINDIR} + +tmp_LTLIBRARIES = + +# This linker flag specifies libtool version info. +# See http://www.gnu.org/software/libtool/manual/libtool.html#Libtool-versioning +# for information regarding incrementing `-version-info`. +plugin_version_info = -version-info 0:0:0 if ISMINGW LDADD = ${top_builddir}/liblib/libnetcdf.la @@ -17,7 +37,7 @@ endif EXTRA_DIST = CMakeLists.txt # The HDF5 filter wrappers -EXTRA_DIST += stdinstall.sh \ +EXTRA_DIST += \ H5Ztemplate.c H5Zmisc.c H5Zutil.c H5Znoop.c h5noop.h NCZmisc.c \ H5Zshuffle.c H5Zdeflate.c H5Zszip.c H5Zszip.h \ H5Zbzip2.c h5bzip2.h H5Zblosc.c H5Zblosc.h H5Zzstd.c H5Zzstd.h @@ -29,74 +49,85 @@ EXTRA_DIST += H5checksum.c if ENABLE_FILTER_TESTING -noinst_LTLIBRARIES += libh5misc.la libh5noop.la libh5noop1.la libnczmisc.la - if ENABLE_NCZARR_FILTERS -noinst_LTLIBRARIES += libh5fletcher32.la libh5shuffle.la libh5deflate.la -libh5shuffle_la_SOURCES = H5Zshuffle.c -libh5fletcher32_la_SOURCES = H5Zfletcher32.c H5checksum.c -libh5deflate_la_SOURCES = H5Zdeflate.c +plugin_LTLIBRARIES += lib__nch5fletcher32.la lib__nch5shuffle.la lib__nch5deflate.la +lib__nch5shuffle_la_SOURCES = H5Zshuffle.c +lib__nch5fletcher32_la_SOURCES = H5Zfletcher32.c H5checksum.c +lib__nch5deflate_la_SOURCES = H5Zdeflate.c + +lib__nch5shuffle_la_LDFLAGS = ${plugin_version_info} +lib__nch5deflate_la_LDFLAGS = ${plugin_version_info} +lib__nch5fletcher32_la_LDFLAGS = ${plugin_version_info} # Need our version of szip if libsz available and we are not using HDF5 if HAVE_SZ -noinst_LTLIBRARIES += libh5szip.la -libh5szip_la_SOURCES = H5Zszip.c H5Zszip.h +plugin_LTLIBRARIES += lib__nch5szip.la +lib__nch5szip_la_SOURCES = H5Zszip.c H5Zszip.h +lib__nch5szip_la_LDFLAGS = ${plugin_version_info} endif endif # ENABLE_NCZARR_FILTERS if ENABLE_PLUGINS -libnczstdfilters_la_SOURCES = NCZstdfilters.c +# The NCZarr codec libraries +lib__nczstdfilters_la_SOURCES = NCZstdfilters.c +lib__nczhdf5filters_la_SOURCES = NCZhdf5filters.c -libnczhdf5filters_la_SOURCES = NCZhdf5filters.c +plugin_LTLIBRARIES += lib__nczhdf5filters.la +plugin_LTLIBRARIES += lib__nczstdfilters.la if HAVE_BLOSC -libh5blosc_la_SOURCES = H5Zblosc.c H5Zblosc.h +lib__nch5blosc_la_SOURCES = H5Zblosc.c H5Zblosc.h +lib__nch5blosc_la_LDFLAGS = ${plugin_version_info} +plugin_LTLIBRARIES += lib__nch5blosc.la endif if HAVE_ZSTD -libh5zstd_la_SOURCES = H5Zzstd.c H5Zzstd.h -endif - -noinst_LTLIBRARIES += libnczhdf5filters.la -noinst_LTLIBRARIES += libnczstdfilters.la - -if HAVE_BLOSC -noinst_LTLIBRARIES += libh5blosc.la -endif -if HAVE_ZSTD -noinst_LTLIBRARIES += libh5zstd.la +lib__nch5zstd_la_SOURCES = H5Zzstd.c H5Zzstd.h +lib__nch5zstd_la_LDFLAGS = ${plugin_version_info} +plugin_LTLIBRARIES += lib__nch5zstd.la endif endif #ENABLE_PLUGINS -libh5misc_la_SOURCES = H5Zmisc.c H5Zutil.c h5misc.h -libnczmisc_la_SOURCES = NCZmisc.c - # The noop filter is to allow testing of multifilters and filter order # Need two distinct instances -libh5noop_la_SOURCES = H5Znoop.c H5Zutil.c h5noop.h -libh5noop1_la_SOURCES = H5Znoop1.c H5Zutil.c h5noop.h +lib__nch5noop_la_SOURCES = H5Znoop.c H5Zutil.c h5noop.h +lib__nch5noop1_la_SOURCES = H5Znoop1.c H5Zutil.c h5noop.h +lib__nch5noop_la_LDFLAGS = ${plugin_version_info} +lib__nch5noop1_la_LDFLAGS = ${plugin_version_info} + +# The misc filter is to allow testing of filter arguments +lib__nch5misc_la_SOURCES = H5Zmisc.c H5Zutil.c h5misc.h +lib__nch5misc_la_LDFLAGS = ${plugin_version_info} +lib__nczmisc_la_SOURCES = NCZmisc.c +lib__nczmisc_la_LDFLAGS = ${plugin_version_info} + +# Provide a filter to test missing filter +lib__nch5unknown_la_SOURCES = H5Zunknown.c +lib__nch5unknown_la_LDFLAGS = ${plugin_version_info} + +tmp_LTLIBRARIES += lib__nch5noop.la lib__nch5noop1.la lib__nch5misc.la lib__nczmisc.la lib__nch5unknown.la # Bzip2 is used to test more complex filters -libh5bzip2_la_SOURCES = H5Zbzip2.c h5bzip2.h +lib__nch5bzip2_la_SOURCES = H5Zbzip2.c h5bzip2.h BZIP2SRC = blocksort.c huffman.c crctable.c randtable.c compress.c decompress.c bzlib.c bzlib.h bzlib_private.h EXTRA_DIST += ${BZIP2SRC} BZIP2_LICENSE if HAVE_LOCAL_BZ2 -libh5bzip2_la_SOURCES += ${BZIP2SRC} +lib__nch5bzip2_la_SOURCES += ${BZIP2SRC} endif +lib__nch5bzip2_la_LDFLAGS = ${plugin_version_info} +plugin_LTLIBRARIES += lib__nch5bzip2.la endif #ENABLE_FILTER_TESTING BUILT_SOURCES = H5Znoop1.c -DISTCLEANFILES = H5Znoop1.c ncjson.h +DISTCLEANFILES = H5Znoop1.c H5Znoop2.c ncjson.h H5Znoop1.c: Makefile H5Znoop.c echo '#define NOOP_INSTANCE 1' > $@ cat ${srcdir}/H5Znoop.c >> $@ -noinst_LTLIBRARIES += libh5bzip2.la - # Record where bzip2 came from; may be out of date BZIP2VER = 1.0.8 BZIP2DIR = bzip2-${BZIP2VER} @@ -108,6 +139,7 @@ bzip2:: cd ${BZIP2DIR}; cp ${BZIP2SRC} ..; cp LICENSE ../BZIP2_LICENSE ; cd .. rm -fr ./${BZIP2DIR} -# Custom install -install-exec-hook: - sh ./stdinstall.sh +# Custom clean +clean-local: + rm -fr ${ALTPLUGINDIR} + diff --git a/plugins/NCZstdfilters.c b/plugins/NCZstdfilters.c index eda1865162..734541944b 100644 --- a/plugins/NCZstdfilters.c +++ b/plugins/NCZstdfilters.c @@ -38,10 +38,8 @@ Author: Dennis Heimbigner /* Forward */ -#ifdef HAVE_BZ2 static int NCZ_bzip2_codec_to_hdf5(const char* codec, size_t* nparamsp, unsigned** paramsp); static int NCZ_bzip2_hdf5_to_codec(size_t nparams, const unsigned* params, char** codecp); -#endif #ifdef HAVE_ZSTD static int NCZ_zstd_codec_to_hdf5(const char* codec, size_t* nparamsp, unsigned** paramsp); @@ -58,7 +56,6 @@ static void NCZ_blosc_codec_finalize(void); /**************************************************/ /* Provide the codec support for bzip2 filter */ -#ifdef HAVE_BZ2 static NCZ_codec_t NCZ_bzip2_codec = {/* NCZ_codec_t codec fields */ NCZ_CODEC_CLASS_VER, /* Struct version number */ NCZ_CODEC_HDF5, /* Struct sort */ @@ -139,7 +136,6 @@ NCZ_bzip2_hdf5_to_codec(size_t nparams, const unsigned* params, char** codecp) done: return stat; } -#endif /**************************************************/ /* Provide the codec support for zstandard filter */ @@ -438,9 +434,7 @@ NCZ_blosc_hdf5_to_codec(size_t nparams, const unsigned* params, char** codecp) /**************************************************/ NCZ_codec_t* NCZ_stdfilters_codecs[] = { -#ifdef HAVE_BZ2 &NCZ_bzip2_codec, -#endif #ifdef HAVE_ZSTD &NCZ_zstd_codec, #endif diff --git a/plugins/NCZtest.c b/plugins/NCZtest.c new file mode 100644 index 0000000000..06c6879756 --- /dev/null +++ b/plugins/NCZtest.c @@ -0,0 +1,179 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * Programmer: Robb Matzke + * Friday, August 27, 1999 + */ + +/* Converted to NCZarr support by Dennis Heimbigner 5/1/2021 */ + +#include +#include +#include +#include +#include + +#include "netcdf_json.h" + +#include "netcdf.h" +#include "netcdf_filter.h" +#include "netcdf_filter_build.h" + +#include "h5misc.h" + +/**************************************************/ +/* NCZarr Filter Objects */ +/* Codec Format +{ +"id": "test", +"testcase": "n", +"byte": "", +"ubyte": "", +"short": "", +"ushort": "", +"int": "", +"uint": "", +"float": "", +"double0": "", +"double1": "", +"int640": "", +"int641": "", +"uint640": "", +"uint641": "", +} +*/ + +/* Give unique dict key names for parameters */ +static const char* fields[14] = { +"testcase", +"byte", +"ubyte", +"short", +"ushort", +"int", +"uint", +"float", +"double_0", +"double_1", +"int64_0", +"int64_1", +"uint64_0", +"uint64_1" +}; + +/* Forward */ +static int NCZ_misc_codec_to_hdf5(const char* codec, size_t* nparamsp, unsigned** paramsp); +static int NCZ_misc_hdf5_to_codec(size_t nparams, const unsigned* params, char** codecp); + +/* Structure for NCZ_PLUGIN_CODEC */ +static NCZ_codec_t NCZ_misc_codec = {/* NCZ_codec_t codec fields */ + NCZ_CODEC_CLASS_VER, /* Struct version number */ + NCZ_CODEC_HDF5, /* Struct sort */ + "test", /* Standard name/id of the codec */ + H5Z_FILTER_TEST, /* HDF5 alias for misc */ + NULL, /*NCZ_misc_codec_initialize*/ + NULL, /*NCZ_misc_codec_finalize*/ + NCZ_misc_codec_to_hdf5, + NCZ_misc_hdf5_to_codec, + NULL, /*NCZ_misc_modify_parameters*/ +}; + +/* External Export API */ +DLLEXPORT +const void* +NCZ_get_codec_info(void) +{ + return (void*)&NCZ_misc_codec; +} + +/* NCZarr Interface Functions */ + +static int +NCZ_misc_codec_to_hdf5(const char* codec_json, size_t* nparamsp, unsigned** paramsp) +{ + int stat = NC_NOERR; + NCjson* jcodec = NULL; + NCjson* jtmp = NULL; + size_t i,nparams = 0; + unsigned* params = NULL; + + /* parse the JSON */ + if(NCJparse(codec_json,0,&jcodec)) + {stat = NC_EFILTER; goto done;} + if(NCJsort(jcodec) != NCJ_DICT) {stat = NC_EPLUGIN; goto done;} + + /* Verify the codec ID */ + if(NCJdictget(jcodec,"id",&jtmp)) + {stat = NC_EFILTER; goto done;} + if(jtmp == NULL || !NCJisatomic(jtmp)) {stat = NC_EINVAL; goto done;} + if(strcmp(NCJstring(jtmp),NCZ_misc_codec.codecid)!=0) {stat = NC_EINVAL; goto done;} + + /* The codec will have (2*14 + 1) +1 = 29 dict entries + id*/ + nparams = (2*14 + 1) + 1; + if(NCJlength(jcodec) != nparams) { + fprintf(stderr,"Incorrect no. of codec parameters: need=29 sent=%ld\n",(unsigned long)(nparams-1)); + stat = NC_EINVAL; + goto done; + } + + /* Actual # of parameters is 14 (ignoring the testcase number) */ + nparams = 14; + if((params = (unsigned*)calloc(nparams,sizeof(unsigned)))== NULL) + {stat = NC_ENOMEM; goto done;} + + for(i=0;i NC_MAX_UINT) {stat = NC_EINVAL; goto done;} + params[i] = (unsigned)jc.ival; + } + if(nparamsp) *nparamsp = nparams; + if(paramsp) {*paramsp = params; params = NULL;} + +done: + if(params) free(params); + NCJreclaim(jcodec); + return stat; +} + +static int +NCZ_misc_hdf5_to_codec(size_t nparams, const unsigned* params, char** codecp) +{ + int i,stat = NC_NOERR; + char json[4096]; + char value[1024]; + + if(nparams == 0 || params == NULL) + {stat = NC_EINVAL; goto done;} + if(nparams != 14) { + fprintf(stderr,"Incorrect no. of parameters: need=14 sent=%ld\n",(unsigned long)nparams); + stat = NC_EINVAL; + goto done; + } + snprintf(json,sizeof(json),"{\"id\": \"%s\"",NCZ_misc_codec.codecid); + for(i=0;i<14;i++) { + snprintf(value,sizeof(value),", \"%s\": \"%u\"",fields[i],params[i]); + strlcat(json,value,sizeof(json)); + } + strlcat(json,"}",sizeof(json)); + if(codecp) { + if((*codecp = strdup(json))==NULL) {stat = NC_ENOMEM; goto done;} + } + +done: + return stat; +} diff --git a/plugins/stdinstall.in b/plugins/stdinstall.in deleted file mode 100755 index 74a73765a8..0000000000 --- a/plugins/stdinstall.in +++ /dev/null @@ -1,44 +0,0 @@ -#!/bin/bash - -if test "x$srcdir" = x ; then srcdir=`pwd`; fi -. ../test_common.sh - -set -x - -INSTALLDIR=@PLUGIN_INSTALL_DIR@ - -# Load the findplugins function -. ${builddir}/findplugin.sh -echo "findplugin.sh loaded" - -installplugin() { -PLUG="$1" -# Locate the plugin path and the library name; argument order is critical -findplugin ${PLUG} -if ! test -f "$HDF5_PLUGIN_DIR/$HDF5_PLUGIN_LIB" ; then - echo "Not exists: ${HDF5_PLUGIN_DIR}/$HDF5_PLUGIN_LIB ; ignored" - return -fi -if ! test -d "${INSTALLDIR}" ; then - echo "Not exists: ${INSTALLDIR} ; creating" - mkdir ${INSTALLDIR} -fi -echo "Installing: $HDF5_PLUGIN_DIR/$HDF5_PLUGIN_LIB into $INSTALLDIR" -cp -f "$HDF5_PLUGIN_DIR/$HDF5_PLUGIN_LIB" $INSTALLDIR -} - -if test "x$USEPLUGINS" != x ; then - if test "x$INSTALLDIR" != x ; then - installplugin h5bzip2 - installplugin h5zstd - installplugin h5blosc - if test "x$FEATURE_NCZARR" ; then - installplugin h5fletcher32 - installplugin h5shuffle - installplugin h5deflate - installplugin h5szip - installplugin nczdefaults - installplugin nczszip - fi - fi -fi diff --git a/test_common.in b/test_common.in index eff6baf8e5..6624863a0e 100644 --- a/test_common.in +++ b/test_common.in @@ -15,12 +15,10 @@ FP_ISMSVC=@ISMSVC@ FP_ISCYGWIN=@ISCYGWIN@ FP_ISMINGW=@ISMINGW@ FP_ISMSYS=@ISMSYS@ - FP_ISOSX=@ISOSX@ -FP_ISREGEDIT=@ISREGEDIT@ -FP_USEPLUGINS=@USEPLUGINS@ FP_ISREGEDIT=@ISREGEDIT@ +FP_USEPLUGINS=@USEPLUGINS@ # Feature flags FEATURE_HDF5=@HAS_HDF5@ @@ -156,11 +154,13 @@ ncgen3c0="${top_srcdir}/ncgen3/c0.cdl" ncgenc0="${top_srcdir}/ncgen/c0.cdl" ncgenc04="${top_srcdir}/ncgen/c0_4.cdl" +# Set LC_ALL +if test "x$FP_ISMSVC" = xyes || test "x$FP_ISCYGWIN" = xyes; then export LC_ALL="en_US.utf8"; fi + # Test for filter availability avail() { if test yes = `${execdir}/../ncdump/ncfilteravail $1` ; then return 0 ; else echo "filter $1 not available" ; return 1; fi } -if test "x$FP_ISMSVC" = xyes || test "x$FP_ISCYGWIN" = xyes; then export LC_ALL="en_US.utf8"; fi # Make sure we are in builddir (not execdir) cd $builddir