From 44d0dcaad2511c3c248cd9baf1d35bf5414edb7b Mon Sep 17 00:00:00 2001 From: Dennis Heimbigner Date: Sun, 16 Feb 2020 12:59:33 -0700 Subject: [PATCH] Add support for multiple filters per variable. re: https://github.com/Unidata/netcdf-c/issues/1584 Support has been added for multiple filters per variable. This affects a number of components in netcdf. The new APIs are documented in NUG/filters.md. The primary changes are: * A set of new functions are provided (see __include/netcdf_filter.h__). - Obtain a list of the filters associated with a variable - Obtain the parameters for a specific filter. * The existing __nc_inq_var_filter__ function now returns info about the first defined filter. * The utilities (ncgen, ncdump, and nccopy) now support an extended format for specifying a sequence of filters. The general form is __|..._. * The ncdump **_Filter** attribute now dumps a list of all the filters associated with a variable using the above new format. * Filter specifications can now use a filter name instead of number for filters known to the netcdf library, which in turn is taken from the HDF5 filter registration page. * New errors are defined: NC_EFILTER and NC_ENOFILTER. The latter is returned if an attempt is made to access an unknown filter. * Internally, the dispatch table has been extended to add a function to handle all of the filter functions. * New, filter-related, tests were added to nc_test4. * A new plugin was added to the plugins directory to help with testing. Notes: 1. The shuffle and fletcher32 filters are not part of the multifilter system. Misc. changes: 1. A debug module was added to libhdf5 to help catch error locations. --- CMakeLists.txt | 1 + Makefile.am | 3 + NUG/filters.md | 341 ++++++++----- RELEASE_NOTES.md | 1 + configure.ac | 10 +- debug/cf | 28 +- examples/C/filter_example.c | 6 +- include/hdf5internal.h | 9 +- include/nc4dispatch.h | 3 + include/nc4internal.h | 62 ++- include/ncdispatch.h | 1 - include/netcdf.h | 14 +- include/netcdf_dispatch.h | 9 + include/netcdf_filter.h | 104 ++-- libdap2/ncd2dispatch.c | 2 + libdap4/ncd4dispatch.c | 9 + libdispatch/derror.c | 4 +- libdispatch/dfilter.c | 578 ++++++++++++++++++---- libdispatch/dnotnc4.c | 19 + libdispatch/dutil.c | 2 +- libdispatch/dvar.c | 35 +- libdispatch/dvarinq.c | 190 +++----- libhdf4/hdf4dispatch.c | 4 +- libhdf5/CMakeLists.txt | 3 +- libhdf5/Makefile.am | 3 +- libhdf5/hdf5debug.c | 47 ++ libhdf5/hdf5debug.h | 23 + libhdf5/hdf5dispatch.c | 1 + libhdf5/hdf5filter.c | 394 +++++++++++++++ libhdf5/hdf5open.c | 60 ++- libhdf5/hdf5type.c | 2 +- libhdf5/hdf5var.c | 192 ++------ libhdf5/nc4hdf.c | 224 +++------ libsrc/nc3dispatch.c | 1 + libsrc4/CMakeLists.txt | 4 +- libsrc4/Makefile.am | 1 + libsrc4/nc4dispatch.c | 1 + libsrc4/nc4internal.c | 27 +- libsrc4/nc4var.c | 31 +- libsrcp/ncpdispatch.c | 1 + nc_test4/CMakeLists.txt | 2 + nc_test4/Makefile.am | 7 +- nc_test4/ref_bzip2.c | 4 +- nc_test4/ref_filter_order.txt | 17 + nc_test4/ref_multi.cdl | 16 + nc_test4/ref_nccopyF.cdl | 21 + nc_test4/ref_ncgenF.cdl | 21 + nc_test4/test_filter_misc.c | 2 +- nc_test4/test_filter_order.c | 609 ++++++++++++++++++++++++ nc_test4/test_filter_reg.c | 52 +- nc_test4/test_szip.c | 7 +- nc_test4/tst_filter.sh | 91 +++- nc_test4/tst_filterparser.c | 197 ++------ nc_test4/tst_multifilter.c | 316 +++++++++++++ nc_test4/tst_vars3.c | 33 +- ncdump/nccopy.c | 286 +++++++---- ncdump/ncdump.c | 86 ++-- ncgen/genbin.c | 26 +- ncgen/genc.c | 82 ++-- ncgen/includes.h | 5 +- ncgen/ncgen.h | 12 +- ncgen/ncgen.y | 40 +- ncgen/ncgenl.c | 868 +++++++++++++--------------------- ncgen/ncgeny.c | 588 ++++++++++++----------- ncgen/ncgeny.h | 2 +- ncgen/util.c | 11 +- plugins/CMakeLists.txt | 8 + plugins/H5Znoop.c | 117 +++++ plugins/Makefile.am | 8 +- plugins/h5noop.h | 27 ++ 70 files changed, 3941 insertions(+), 2070 deletions(-) create mode 100644 libhdf5/hdf5debug.c create mode 100644 libhdf5/hdf5debug.h create mode 100644 libhdf5/hdf5filter.c create mode 100644 nc_test4/ref_filter_order.txt create mode 100644 nc_test4/ref_multi.cdl create mode 100644 nc_test4/ref_nccopyF.cdl create mode 100644 nc_test4/ref_ncgenF.cdl create mode 100644 nc_test4/test_filter_order.c create mode 100644 nc_test4/tst_multifilter.c create mode 100644 plugins/H5Znoop.c create mode 100644 plugins/h5noop.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 47b656c806..ea486188b2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -735,6 +735,7 @@ IF(USE_HDF5 OR ENABLE_NETCDF_4) SET(HAVE_H5Z_SZIP 1) SET(SZIP_LIBRARY ${SZIP}) SET(CMAKE_REQUIRED_LIBRARIES ${CMAKE_REQUIRED_LIBRARIES} ${SZIP}) + MESSAGE(STATUS "HDF5 has szip.") ELSE() MESSAGE(FATAL_ERROR "HDF5 Requires SZIP, but cannot find libszip or libsz.") ENDIF() diff --git a/Makefile.am b/Makefile.am index 61d3f523e7..6af0b19e07 100644 --- a/Makefile.am +++ b/Makefile.am @@ -49,11 +49,14 @@ LIBSRC4_DIR = libsrc4 endif # Is the user building with HDF5? +# Note that USE_HDF5 does not imply USE_NETCDF4 if USE_HDF5 H5_TEST_DIR = h5_test LIBHDF5 = libhdf5 +if USE_NETCDF4 NC_TEST4 = nc_test4 endif +endif # Build the dap2 client if ENABLE_DAP diff --git a/NUG/filters.md b/NUG/filters.md index cb4bfb17a8..a15159f733 100644 --- a/NUG/filters.md +++ b/NUG/filters.md @@ -7,38 +7,46 @@ NetCDF-4 Filter Support {#filters} [TOC] -# Introduction {#filters_intro} +The netCDF library supports a general filter mechanism to apply various +kinds of filters to datasets before reading or writing. -The HDF5 library (1.8.11 and later) -supports a general filter mechanism to apply various -kinds of filters to datasets before reading or writing. The netCDF enhanced (aka netCDF-4) library inherits this capability since it depends on the HDF5 library. +The HDF5 library (1.8.11 and later) supports filters, +and netCDF is based closely on that underlying HDF5 mechanism. Filters assume that a variable has chunking defined and each chunk is filtered before writing and "unfiltered" after reading and before passing the data to the user. +In the event that multiple filters are defined on a variable, +they are applied in first-defined order on writing +and on the reverse order when reading. + The most common kind of filter is a compression-decompression filter, and that is the focus of this document. +For now, this document is strongly influenced by the HDF5 mechanism. +When other implementations (e.g. Zarr) support filters, this document +will have multiple sections: one for each mechanism. + +# Enabling A HDF5 Compression Filter {#filters_enable} + HDF5 supports dynamic loading of compression filters using the following process for reading of compressed data. 1. Assume that we have a dataset with one or more variables that were compressed using some algorithm. How the dataset was compressed will be discussed subsequently. - 2. Shared libraries or DLLs exist that implement the compress/decompress algorithm. These libraries have a specific API so that the HDF5 library can locate, load, and utilize the compressor. These libraries are expected to installed in a specific directory. -# Enabling A Compression Filter {#filters_enable} - -In order to compress a variable, the netcdf-c library +In order to compress a variable with an HDF5 compliant filter, +the netcdf-c library must be given three pieces of information: (1) some unique identifier for the filter to be used, (2) a vector of parameters for @@ -54,32 +62,34 @@ It is legal to provide a zero-length set of parameters. Defaults are not provided, so this assumes that the filter can operate with zero parameters. -Filter ids are assigned by the HDF group. See [4] -for a current list of assigned filter ids. -Note that ids above 32767 can be used for testing without -registration. +Filter ids are assigned by the HDF group. See [4] for a current +list of assigned filter ids. Note that ids above 32767 can be +used for testing without registration. The first two pieces of information can be provided in one of three ways: using __ncgen__, via an API call, or via command line parameters to __nccopy__. In any case, remember that filtering also requires setting chunking, so the -variable must also be marked with chunking information. +variable must also be marked with chunking information. If compression +is set for a non-chunked variable, the variable will forcibly be +converted to chunked using a default chunking algorithm. ## Using The API {#filters_API} -The necessary API methods are included in __netcdf.h__ by default. -One API method is for setting the filter to be used -when writing a variable. The relevant signature is -as follows. +The necessary API methods are included in __netcdf_filter.h__ by default. +These functions implicitly use the HDF5 mechanisms and may produce an +error if applied to a file format that is not compatible with the +HDF5 mechanism. + +1. Add a filter to the set of filters to be used when writing a variable. ```` int nc_def_var_filter(int ncid, int varid, unsigned int id, size_t nparams, const unsigned int* parms); ```` This must be invoked after the variable has been created and before __nc_enddef__ is invoked. -A second API methods makes it possible to query a variable to -obtain information about any associated filter using this signature. +2. Query a variable to obtain information about the +first filter associated with the variable. ```` int nc_inq_var_filter(int ncid, int varid, unsigned int* idp, size_t* nparams, unsigned int* params); - ```` The filter id will be returned in the __idp__ argument (if non-NULL), the number of parameters in __nparamsp__ and the actual parameters in @@ -87,6 +97,28 @@ __params__. As is usual with the netcdf API, one is expected to call this function twice. The first time to get __nparams__ and the second to get the parameters in client-allocated memory. +3. Query a variable to obtain a list of all filters associated with a variable. +```` +int nc_inq_var_filterids(int ncid, int varid, int format, size_t* nfilters, unsigned int* filterids); +```` +The number of filters is determined by __nfilters__. +The set of filter ids will be returned in __filterids__. +As is usual with the netcdf API, one is expected to call +this function twice. The first time to get __nbytes__ and the +second to get the filter ids in client-allocated memory. + +4. Query a variable to obtain information about a specific +filter associated with the variable. +```` +int nc_inq_var_filter_info(int ncid, int varid, unsigned int id, size_t* nparamsp, unsigned int* params); +```` +The __id__ filter id indicates the filter of interest. +The actual parameters are in __params__. +The number of parameters is returned in nparamsp. +As is usual with the netcdf API, one is expected to call +this function twice. The first time to get __nparams__ and the +second to get the parameters in client-allocated memory. + ## Using ncgen {#filters_NCGEN} In a CDL file, compression of a variable can be specified @@ -96,22 +128,26 @@ by annotating it with the following attribute: constants specifying (1) the filter id to apply, and (2) a vector of constants representing the parameters for controlling the operation of the specified filter. -See the section on the parameter encoding syntax +See the section on the parameter encoding syntax for the details on the allowable kinds of constants. This is a "special" attribute, which means that it will normally be invisible when using __ncdump__ unless the -s flag is specified. +This attribute may be repeated to specify multiple filters. +As a rule, use the ___Deflate__ attribute instead of ___Filter__ +(but using ___Filter__ to specify deflation will work). + ### Example CDL File (Data elided) ```` -netcdf bzip2 { +netcdf bzip2szip { dimensions: dim0 = 4 ; dim1 = 4 ; dim2 = 4 ; dim3 = 4 ; variables: float var(dim0, dim1, dim2, dim3) ; - var:_Filter = "307,9" ; + var:_Filter = "307,9|szip,32,32" ; // bzip2+szip var:_Storage = "chunked" ; var:_ChunkSizes = 4, 4, 4, 4 ; data: @@ -131,11 +167,11 @@ Assume that __unfiltered.nc__ has a chunked but not bzip2 compressed variable named "var". This command will create that variable in the __filtered.nc__ output file but using filter with id 307 (i.e. bzip2) and with parameter(s) 9 indicating the compression level. -See the section on the parameter encoding syntax +See the section on the parameter encoding syntax for the details on the allowable kinds of constants. -The "-F" option can be used repeatedly as long as the variable name -part is different. A different filter id and parameters can be +The "-F" option can be used repeatedly, +as long as a different variable is specified for each occurrence. It can be convenient to specify that the same compression is to be @@ -143,9 +179,14 @@ applied to more than one variable. To support this, two additional *-F* cases are defined. 1. ````-F *,...``` means apply the filter to all variables in the dataset. -2. ````-F v1|v2|..,...``` means apply the filter to a multiple variables. +2. ````-F v1&v2&..,...``` means apply the filter to a multiple variables. + +Multiple filters can be specified using the pipeline notions '|'. +For example -Note that the characters '*' and '|' are bash reserved characters, +1. ````-F v1&v2,307,9|4,32,32`` means apply filter 307 (bzip2) then filter 4 (szip) to the multiple variables. + +Note that the characters '*', '&', and '|' are bash reserved characters, so you will probably need to escape or quote the filter spec in that environment. @@ -164,7 +205,7 @@ Suppression of output filtering for a specific set of variables can be accomplished using these formats. ```` nccopy -F "var,none" input.nc output.nc -nccopy -F "v1|v2|...,none" input.nc output.nc +nccopy -F "v1&v2&...,none" input.nc output.nc ```` where "var" and the "vi" are the fully qualified name of a variable. @@ -173,23 +214,24 @@ by this table.
-F none-Fvar,...Input FilterApplied Output Filter -
trueunspecifiedNAunfiltered -
true-Fvar,noneNAunfiltered -
true-Fvar,...NAuse output filter -
falseunspecifieddefineduse input filter -
false-Fvar,noneNAunfiltered -
false-Fvar,...NAuse output filter -
falseunspecifiednoneunfiltered +
trueundefinedNAunfiltered +
truenoneNAunfiltered +
truedefinedNAuse output filter(s) +
falseundefineddefineduse input filter(s) +
falsenoneNAunfiltered +
falsedefinedNAuse output filter(s) +
falseundefinedundefinedunfiltered +
falsedefineddefineduse output filter(s)
-# Parameter Encode/Decode {#filters_paramcoding} +# HDF5 Parameter Encode/Decode {#filters_paramcoding} -The parameters passed to a filter are encoded internally as a vector -of 32-bit unsigned integers. It may be that the parameters -required by a filter can naturally be encoded as unsigned integers. -The bzip2 compression filter, for example, expects a single -integer value from zero thru nine. This encodes naturally as a -single unsigned integer. +The parameters passed to an HDF5 format filter are encoded +internally as a vector of 32-bit unsigned integers. It may be +that the parameters required by a filter can naturally be +encoded as unsigned integers. The bzip2 compression filter, for +example, expects a single integer value from zero thru +nine. This encodes naturally as a single unsigned integer. Note that signed integers and single-precision (32-bit) float values also can easily be represented as 32 bit unsigned integers by @@ -202,7 +244,9 @@ zero extending. Machine byte order (aka endian-ness) is an issue for passing some kinds of parameters. You might define the parameters when compressing on a little endian machine, but later do the -decompression on a big endian machine. Byte order is not an +decompression on a big endian machine. + +When using HDF5 format filters, byte order is not an issue for 32-bit values because HDF5 takes care of converting them between the local machine byte order and network byte order. @@ -217,7 +261,7 @@ on a machine whose byte order is different than the machine in which the parameters were initially created, the two integers will be separately endian converted. But this will be incorrect for 64-bit values. -So, we have this situation: +So, we have this situation (for HDF5 only): 1. the 8 bytes come in as native machine order for the machine doing the call to *nc_def_var_filter*. @@ -227,7 +271,10 @@ So, we have this situation: but with the bytes in each piece consistent with the native machine order for the machine executing the filter. -## Encoding Algorithms +Note that the situation for non-HDF5 filters may be different +since the arguments are potentially 64 bit. + +## Encoding Algorithms for HDF5 In order to properly extract the correct 8-byte value, we need to ensure that the values stored in the HDF5 file have a known format independent of @@ -276,19 +323,27 @@ This leads to the following set of rules. to convert it to desired BE format. To support these rules, some utility programs exist and are discussed in -Appendix A. +Appendix A. # Filter Specification Syntax {#filters_syntax} -Both of the utilities -__ncgen__ +The utilities +ncgen and -__nccopy__ -allow the specification of filter parameters in text format. -These specifications consist of a sequence of comma -separated constants. The constants are converted -within the utility to a proper set of unsigned int -constants (see the parameter encoding section). +nccopy, +and also the output of __ncdump__, +support the specification of filter ids, formats, and parameters +in text format. The BNF specification is defined in +Appendix C. +Basically, These specifications consist of a filter id +a comma, and then a sequence of +comma separated constants representing the parameters +The id may be a name (case-insensitive) if it is known to the netcdf library +(see Appendix D). +otherwise it must be an unsigned integer. +The constants are converted within +the utility to a proper set of unsigned int constants (see the +parameter encoding section). To simplify things, various kinds of constants can be specified rather than just simple unsigned integers. The utilities will encode @@ -315,12 +370,15 @@ Some things to note. 1. In all cases, except for an untagged positive integer, the format tag is required and determines how the constant is converted to one or two unsigned int values. - The positive integer case is for backward compatibility. -2. For signed byte and short, the value is sign extended to 32 bits +1. For an untagged positive integer, the constant is treated + as of the smallest type into which it fits (i.e. 8,16,32, or 64 bit). +1. For signed byte and short, the value is sign extended to 32 bits and then treated as an unsigned int value. -3. For double, and signed|unsigned long long, they are converted +1. For double, and signed|unsigned long long, they are converted as specified in the section on parameter encode/decode. +1. In order to support mutiple filters, the argument to ''_Filter'' + may be a pipeline separated (using '|') to specify a list of filters specs. Dynamic Loading Process {#filters_Process} ========== @@ -392,20 +450,28 @@ Since ncdump is not being asked to access the data (the -h flag), it can obtain the filter information without failures. Then it can print out the filter id and the parameters (the -s flag). -Test Case {#filters_TestCase} +Test Cases {#filters_TestCase} ------- Within the netcdf-c source tree, the directory -__netcdf-c/nc_test4__ contains a test case (__test_filter.c__) for -testing dynamic filter writing and reading using -bzip2. Another test (__test_filter_misc.c__) validates -parameter passing. These tests are disabled if __--enable-shared__ +__netcdf-c/nc_test4__ contains a number of test cases for +testing dynamic filter writing and reading. These include +* __test_filter.c__ -- tests simple compression/decompression using +the bzip2 compressor in the directory __plugins__. +* __test_filterparser.c__ -- validates parameter passing. +* __test_filter_reg.c__ -- test user defined filter registration +(see Appendix B). +* __test_multifilter.c__ -- tests applying multiple filters to a +single variable: bzip2, deflate(zip), and szip (if enabled). +* __test_filter.sh__ -- test driver to execute the above tests. + +These tests are disabled if __--enable-shared__ is not set or if __--enable-netcdf-4__ is not set. Example {#filters_Example} ------- A slightly simplified version of the filter test case is also available as an example within the netcdf-c source tree -directory __netcdf-c/examples/C. The test is called __filter_example.c__ +directory __netcdf-c/examples/C__. The test is called __filter_example.c__ and it is executed as part of the __run_examples4.sh__ shell script. The test case demonstrates dynamic filter writing and reading. @@ -439,9 +505,10 @@ One can look at the the code in plugins/H5Zbzip2.c and H5Zmisc.c to see this. SZIP Issues ----------- -The current szip plugin code in the HDF5 library -has some behaviors that can catch the unwary. -Specifically, this filter may do two things. +The current szip plugin code in the HDF5 library has some +behaviors that can catch the unwary. These are handled +internally to (mostly) hide them so that they should not affect +users. Specifically, this filter may do two things. 1. Add extra parameters to the filter parameters: going from the two parameters provided by the user to four parameters @@ -457,11 +524,9 @@ the szip API provided by the underlying H5Pset_szip function is actually a subset of the capabilities of the real szip implementation. Presumably this is for historical reasons. -In any case, if the caller uses the __nc_inq_var_szip__, then -the values returned may differ from those originally specified. -If one used the __nc_inq_var_filter__ API calls, it may be the case that -both the number of parameters and the values will differ from the original -call to __nc_def_var_filter__. +In any case, if the caller uses the __nc_inq_var_szip__ +or the __nc_inq_var_filter__ functions, then +the parameter values returned may differ from those originally specified. Supported Systems ----------------- @@ -480,26 +545,27 @@ has been known to work. gcc -g -O0 -shared -o libbzip2.so -L${HDF5LIBDIR} -lhdf5_hl -lhdf5 -L${ZLIBDIR} -lz ```` -Appendix A. Support Utilities {#filters_AppendixA} +Appendix A. Support Utilities {#filters_appendixa} ========== -Two functions are exported from the netcdf-c library +Several functions are exported from the netcdf-c library for use by client programs and by filter implementations. -1. ````int NC_parsefilterspec(const char* spec, unsigned int* idp, size_t* nparamsp, unsigned int** paramsp);```` - * idp will contain the filter id value from the spec. - * nparamsp will contain the number of 4-byte parameters - * paramsp will contain a pointer to the parsed parameters -- the caller - must free. +1. ````int NC_parsefilterlist(const char* listspec, int* formatp, size_t* nfiltersp, NC_Filterspec*** filtersp); + * listspec contains the text of a sequence of filter specs. + * formatp will contain the format of the filters (currently only HDF5 is supported) + * nfiltersp will contain the number of filters found + * filtersp will contain a vector of pointers to the parsed filters + -- the caller must free; also note that the number of filters will + depend on the format. This function can parse filter spec strings as defined in the section on Filter Specification Syntax. - This function parses the first argument and returns several values. -2. ````int NC_filterfix8(unsigned char* mem8, int decode);```` +2. ````int NC_hdf5_filterfix8(unsigned char* mem8, int decode);```` * mem8 is a pointer to the 8-byte value either to fix. * decode is 1 if the function should apply the 8-byte decoding algorithm else apply the encoding algorithm. - This function implements the 8-byte conversion algorithms. + This function implements the 8-byte conversion algorithms for HDF5 Before calling *nc_def_var_filter* (unless *NC_parsefilterspec* was used), the client must call this function with the decode argument set to 0. Inside the filter code, this function should be called with the decode @@ -508,7 +574,7 @@ for use by client programs and by filter implementations. Examples of the use of these functions can be seen in the test program *nc_test4/tst_filterparser.c*. -Appendix B. Programmatic Filter Definition {#filters_programmatic} +Appendix B. Programmatic Filter Definition {#filters_appendixb} ========== HDF5 provides an API [6] to allow for the programmatic definition @@ -528,21 +594,11 @@ Three concepts are used in this API. Currently, only *NC_FILTER_FORMAT_HDF5* is defined and corresponds to the existing HDF5 plugin format. 2. ID - this is an integer that is a unique identifier for the filter. - This value is interpreted in the context of the format, so the same - id might be assigned to different filters if the format is different. -3. The structure *NC_FILTER_INFO* that provides generic information - to the API and has a placeholder for format-specific information. - - typedef struct NC_FILTER_INFO { - int version; /* Of this structure */ - int format; /* Controls actual type of this structure */ - int id; /* Must be unique WRT format */ - void* info; /* The filter info as defined by the format. */ - } NC_FILTER_INFO; - When the format is the value NC_FILTER_FORMAT_HDF5, - then the info field is a pointer to an instance of - H5Z_class2_t as define in H5Zpublic.h. - The use of void* is, of course, to allow for passing arbitrary objects. +3. The structure H5Z_class2_t (defined in H5Zpublic.h). + This carries the information required by HDF5 to use the filter. + To avoid having to include the definition of H5Z_class2_t + when netcdf_filter is used, a pointer to H5Z_class2_t + is replaced with a __void*__ pointer. ### NetCDF API @@ -550,42 +606,95 @@ The following function signatures are provided (see *netcdf_filter.h*). 1. Register a filter - int nc_filter_register(NC_FILTER_INFO* filter_info); - Register a filter whose format and ID are specified in the 'filter_info' - argument. + int nc_filter_client_register(unsigned int id, void*/*H5Z_class2_t* */); + Register a filter of given id and H5Z_class_t info. 2. Unregister a filter - int nc_filter_unregister(int format, int id); + int nc_filter_client_unregister(int id); Unregister the filter specified by the id. Note that only filters registered using 'nc_filter_register' can be unregistered. 3. Inquire about a filter - int nc_filter_inq(int format, int id, NC_FILTER_INFO* filter_info); - Unregister the filter specified by the id. Note that only + int nc_filter_client_inq(unsigned int id, void*/*H5Z_class2_t* */); + Inquire about the filter specified by the id. Note that only filters registered using 'nc_filter_register' can be inquired. - The 'filter_info' is filled with a copy of the original argument to + The second argument is filled with a copy of the original argument to 'nc_filter_register'. ### Example - static const H5Z_class2_t H5Z_REG[1] = { +```` + static H5Z_class2_t H5Z_REG[1] = { ... }; ... - NC_FILTER_INFO info; + H5Z_class2_t* h5info = &H5Z_REG[0]; + H5Z_class2_t h5inq; ... - info.version = NC_FILTER_INFO_VERSION; - info.format = NC_FILTER_FORMAT_HDF5; - info.id = FILTER_ID; - info.info = (void*)&H5Z_REG[0]; - stat = nc_filter_register(&info); + stat = nc_user_filter_register(FILTER_ID,(void*)&h5info); ... - memset(&info,0,sizeof(NC_FILTER_INFO)); - stat = nc_filter_inq(NC_FILTER_FORMAT_HDF5, FILTER_ID, &info); + stat = nc_user_filter_inq(FILTER_ID, &h5inq); ... - stat = nc_filter_unregister(NC_FILTER_FORMAT_HDF5, FILTER_ID); + stat = nc_user_filter_unregister(FILTER_ID); +```` + +Appendix C. BNF for Specifying Filters in Utilities {#filters_appendixc} +========== + +```` +speclist: spec + | speclist '|' spec + ; +spec: filterid + | filterid ',' parameterlist + ; +filterid: unsigned32 + | name + ; +parameterlist: parameter + | parameterlist ',' parameter + ; +parameter: unsigned32 + +where +int: <32 bit integer> +unsigned32: <32 bit unsigned integer> +name: [a-zA-Z][a-zA-Z0-9-_]* +```` + +Appendix D. Known Filter Names {#filters_appendixd} +========== + +2
NameIdDescription +
zip2Standard zlib compression +
zlib
+
deflate2 +
szip4Standard szip compression +
bzip2307BZIP2 lossless compression used by PyTables +
lzf32000LZF lossless compression used by H5Py project +
blosc32001Blosc lossless compression used by PyTables +
mafisc32002Modified LZMA compression filter, MAFISC (Multidimensional Adaptive Filtering Improved Scientific data Compression) +
snappy32003Snappy lossless compression. +
lz432004LZ4 fast lossless compression algorithm +
apax32005Samplify's APAX Numerical Encoding Technology +
cbf32006All imgCIF/CBF compressions and decompressions, including Canonical, Packed, Packed Vesrsion 2, Byte Offset and Nibble Offset. +
jpeg-xr32007Enables images to be compressed/decompressed with JPEG-XR compression +
bitshuffle32008Extreme version of shuffle filter that shuffles data at bit level instead of byte level. +
spdp32009SPDP fast lossless compression algorithm for single- and double-precision floating-point data. +
lpc-rice32010LPC-Rice multi-threaded lossless compression +
ccsds-12332011ESA CCSDS-123 multi-threaded compression filter +
jpeg-ls32012CharLS JPEG-LS multi-threaded compression filter +
zfp32013Rate, accuracy or precision bounded compression for floating-point arrays +
fpzip32014Fast and Efficient Lossy or Lossless Compressor for Floating-Point Data +
zstandard32015Real-time compression algorithm with wide range of compression / speed trade-off and fast decoder +
b3d32016GPU based image compression method developed for light-microscopy applications +
sz32017An error-bounded lossy compressor for scientific floating-point data +
fcidecomp32018EUMETSAT CharLS compression filter for use with netCDF +
user-defined32768First user-defined filter +
+ # References {#filters_References} @@ -601,5 +710,5 @@ The following function signatures are provided (see *netcdf_filter.h*). __Author__: Dennis Heimbigner
__Email__: dmh at ucar dot edu __Initial Version__: 1/10/2018
-__Last Revised__: 2/5/2018 +__Last Revised__: 1/27/2020 diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index ffc221d1cb..28b07a99ee 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -7,6 +7,7 @@ This file contains a high-level description of this package's evolution. Release ## 4.7.4 - TBD +* [Enhancement] Support has been added for multiple filters per variable. See [https://github.com/Unidata/netcdf-c/issues/1584]. * [Enhancement] Now nc_inq_var_szip retuns 0 for parameter values if szip is not in use for var. See [https://github.com/Unidata/netcdf-c/issues/1618]. * [Enhancement] Now allow parallel I/O with filters, for HDF5-1.10.3 and later. See [https://github.com/Unidata/netcdf-c/issues/1473]. * [Enhancement] Increased default size of cache buffer to 16 MB, from 4 MB. Increased number of slots to 4133. See [https://github.com/Unidata/netcdf-c/issues/1541]. diff --git a/configure.ac b/configure.ac index 0c4faec448..0d988d5993 100644 --- a/configure.ac +++ b/configure.ac @@ -208,6 +208,7 @@ fi AC_MSG_RESULT([$enable_netcdf_4]) # Does the user want to use HDF5? +# WARNING: enable_hdf5 does not imply that use_netcdf_4 is enabled AC_MSG_CHECKING([whether we should build with HDF5]) #enable_hdf5=$enable_netcdf_4 AC_ARG_ENABLE([hdf5], [AS_HELP_STRING([--disable-hdf5], @@ -1022,8 +1023,11 @@ if test "x$enable_netcdf_4" = xyes; then AC_DEFINE([USE_NETCDF4], [1], [if true, build netCDF-4]) fi +# Set defaults hdf5_parallel=no hdf5_supports_par_filters=no +enable_szlib=no + if test "x$enable_hdf5" = xyes; then AC_DEFINE([USE_HDF5], [1], [if true, use HDF5]) @@ -1082,16 +1086,12 @@ if test "x$enable_hdf5" = xyes; then # Check whether HDF5 was built with the SZLIB library. If so we # must be able to link to szip library. AC_MSG_CHECKING([whether szlib was used when building HDF5]) - enable_szlib=no if test "x$ac_cv_func_H5Z_SZIP" = xyes; then enable_szlib=yes AC_DEFINE([USE_SZIP], [1], [if true, compile in szip compression in netCDF-4 variables]) fi AC_MSG_RESULT([$enable_szlib]) - fi -AM_CONDITIONAL(HAS_PAR_FILTERS, [test x$hdf5_supports_par_filters = xyes ]) -AC_SUBST([HAS_PAR_FILTERS], [$hdf5_supports_par_filters]) # If the user wants hdf4 built in, check it out. if test "x$enable_hdf4" = xyes; then @@ -1302,6 +1302,7 @@ AM_CONDITIONAL(SHOW_DOXYGEN_TAG_LIST, [test x$enable_doxygen_tasks = xyes]) AM_CONDITIONAL(ENABLE_METADATA_PERF, [test x$enable_metadata_perf = xyes]) AM_CONDITIONAL(ENABLE_BYTERANGE, [test "x$enable_byterange" = xyes]) AM_CONDITIONAL(RELAX_COORD_BOUND, [test "xyes" = xyes]) +AM_CONDITIONAL(HAS_PAR_FILTERS, [test x$hdf5_supports_par_filters = xyes ]) # If the machine doesn't have a long long, and we want netCDF-4, then # we've got problems! @@ -1434,6 +1435,7 @@ AC_SUBST(HAS_JNA,[$enable_jna]) AC_SUBST(HAS_ERANGE_FILL,[$enable_erange_fill]) AC_SUBST(HAS_BYTERANGE,[$enable_byterange]) AC_SUBST(RELAX_COORD_BOUND,[yes]) +AC_SUBST([HAS_PAR_FILTERS], [$hdf5_supports_par_filters]) # Include some specifics for netcdf on windows. #AH_VERBATIM([_WIN32_STRICMP], diff --git a/debug/cf b/debug/cf index 962e56b220..ba3d402694 100644 --- a/debug/cf +++ b/debug/cf @@ -1,22 +1,23 @@ #!/bin/bash #NB=1 -DB=1 +#DB=1 #X=-x #ANSI=1 -#MEM=1 +#MEM=1 # export NC_VLEN_NOTEST=1 #NOTUIL=1 -FAST=1 +#FAST=1 #PROF=1 -#HDF5=1 -#DAP=1 +#NCZARR=1 +HDF5=1 +DAP=1 #SZIP=1 #HDF4=1 #PNETCDF=1 #PAR4=1 -#TESTSERVERS="localhost:8080,remotetest.unidata.ucar.edu" +#TESTSERVERS="localhost:8080,149.165.169.123:8080" if test $# != 0 ; then cmds=$@ @@ -31,11 +32,12 @@ CFLAGS="" #CFLAGS="-Wall -Wno-unused-but-set-variable -Wno-unused-variable -Wno-unused-parameter -Wno-char-subscripts -Wno-pointer-sign -Wno-format ${CFLAGS}" CFLAGS="-Wall -Wno-unused-parameter -Wno-char-subscripts -Wno-pointer-sign ${CFLAGS}" #CFLAGS="-Wall ${CFLAGS}" -#CFLAGS="-Wconversion" +#CFLAGS="$CFLAGS -Wconversion" +#CFLAGS="-Wall -Wunused-parameter -Wno-char-subscripts -Wno-pointer-sign ${CFLAGS}" if test "x$MEM" = x1 ; then -export NC_VLEN_NOTEST=1 -CFLAGS="-fsanitize=address ${CFLAGS}" + export NC_VLEN_NOTEST=1 + CFLAGS="-fsanitize=address ${CFLAGS}" fi if test "x$ANSI" = x1 ; then @@ -118,11 +120,12 @@ FLAGS="$FLAGS --enable-extreme-numbers" #FLAGS="$FLAGS --disable-testsets" #FLAGS="$FLAGS --disable-dap-remote-tests" #FLAGS="$FLAGS --enable-dap-auth-tests" -- requires a new remotetest server -FLAGS="$FLAGS --enable-doxygen --enable-internal-docs" +#FLAGS="$FLAGS --enable-doxygen --enable-internal-docs" FLAGS="$FLAGS --enable-logging" #FLAGS="$FLAGS --disable-diskless" FLAGS="$FLAGS --enable-mmap" FLAGS="$FLAGS --enable-byterange" +#FLAGS="$FLAGS --enable-atexit-finalize" #FLAGS="$FLAGS --with-udunits" #FLAGS="$FLAGS --with-libcf" #FLAGS="$FLAGS --enable-jna" @@ -130,6 +133,7 @@ FLAGS="$FLAGS --enable-byterange" #FLAGS="$FLAGS --disable-silent-rules" #FLAGS="$FLAGS --disable-filter-testing" #FLAGS="$FLAGS --enable-metadata-perf" +#FLAGS="$FLAGS --enable-extra-tests" #FLAGS="$FLAGS --with-ncproperties-extra=key1=value1,key2=value2" #FLAGS="$FLAGS --enable-valgrind" @@ -141,6 +145,10 @@ if test "x$PAR4" != x1 ; then FLAGS="$FLAGS --disable-parallel4" fi +if test "x$NCZARR" = x1 ; then +FLAGS="$FLAGS --enable-nczarr" +fi + if test "x$NOUTIL" = x1 ; then FLAGS="$FLAGS --disable-utilities" fi diff --git a/examples/C/filter_example.c b/examples/C/filter_example.c index 02e00569ea..8599feac5e 100644 --- a/examples/C/filter_example.c +++ b/examples/C/filter_example.c @@ -48,6 +48,7 @@ netcdf bzip2 { #include #include "netcdf.h" +#include "netcdf_filter.h" /* The HDF assigned id for bzip compression */ #define BZIP2_ID 307 @@ -169,7 +170,6 @@ test_bzip2(void) { int i; unsigned int level = BZIP2_LEVEL; - unsigned int id=0; size_t nparams = 0; printf("\n*** Testing API: bzip2 compression.\n"); @@ -204,8 +204,8 @@ test_bzip2(void) /* Read back the compression info and verify it */ level = 0; - CHECK(nc_inq_var_filter(ncid,varid,&id,&nparams,&level)); - if(id != BZIP2_ID || nparams != 1 || level != BZIP2_LEVEL) { + CHECK(nc_inq_var_filter_info(ncid,varid,BZIP2_ID,&nparams,&level)); + if(nparams != 1 || level != BZIP2_LEVEL) { printf("test_filter: filter def/inq mismatch\n"); return NC_EFILTER; } diff --git a/include/hdf5internal.h b/include/hdf5internal.h index 7149e5fa48..1093c67502 100644 --- a/include/hdf5internal.h +++ b/include/hdf5internal.h @@ -108,6 +108,9 @@ typedef struct NC_HDF5_TYPE_INFO hid_t native_hdf_typeid; } NC_HDF5_TYPE_INFO_T; +/* Forward */ +struct NC_FILTER_OBJ_HDF5; + /* Logging and debugging. */ void reportopenobjects(int log, hid_t); int hdf5_set_log_level(); @@ -171,11 +174,15 @@ int nc4_get_var_meta(NC_VAR_INFO_T *var); /* Define Filter API Function */ -int nc4_filter_action(int action, int formatx, int id, NC_FILTER_INFO* info); +int nc4_global_filter_action(int action, unsigned int id, struct NC_FILTER_OBJ_HDF5* infop); +int NC4_hdf5_addfilter(NC_VAR_INFO_T* var, int active, unsigned int id, size_t nparams, unsigned int* params); +int NC4_hdf5_remove_filter(NC_VAR_INFO_T* var, unsigned int filterid); + /* Support functions for provenance info (defined in nc4hdf.c) */ extern int NC4_hdf5get_libversion(unsigned*,unsigned*,unsigned*);/*libsrc4/nc4hdf.c*/ extern int NC4_hdf5get_superblock(struct NC_FILE_INFO*, int*);/*libsrc4/nc4hdf.c*/ extern int NC4_isnetcdf4(struct NC_FILE_INFO*); /*libsrc4/nc4hdf.c*/ +extern int nc4_find_default_chunksizes2(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var); #endif /* _HDF5INTERNAL_ */ diff --git a/include/nc4dispatch.h b/include/nc4dispatch.h index 4417d9f716..1403c80dc9 100644 --- a/include/nc4dispatch.h +++ b/include/nc4dispatch.h @@ -256,6 +256,9 @@ extern "C" { EXTERNL int NC4_show_metadata(int); + EXTERNL int + NC4_filter_actions(int, int, int, struct NC_Filterobject*); + #if defined(__cplusplus) } #endif diff --git a/include/nc4internal.h b/include/nc4internal.h index c28d017e5d..5d74a85959 100644 --- a/include/nc4internal.h +++ b/include/nc4internal.h @@ -16,7 +16,6 @@ #include #include #include -#include #include #include "ncdimscale.h" @@ -27,6 +26,7 @@ #include "netcdf_f.h" #include "netcdf_mem.h" +#include "netcdf_filter.h" #ifdef USE_PARALLEL #include "netcdf_par.h" #endif /* USE_PARALLEL */ @@ -126,6 +126,7 @@ typedef enum {NC_FALSE = 0, NC_TRUE = 1} nc_bool_t; /* Forward declarations. */ struct NC_GRP_INFO; struct NC_TYPE_INFO; +struct NC_FIlterobject; /** * This struct provides indexed Access to Meta-data objects. See the @@ -212,16 +213,12 @@ typedef struct NC_VAR_INFO int parallel_access; /**< Type of parallel access for I/O on variable (collective or independent) */ nc_bool_t dimscale; /**< True if var is a dimscale */ nc_bool_t *dimscale_attached; /**< Array of flags that are true if dimscale is attached for that dim index */ - nc_bool_t deflate; /**< True if var has deflate filter applied */ - int deflate_level; /**< If deflate is true, this is the deflate level, between 0 and 9. */ nc_bool_t shuffle; /**< True if var has shuffle filter applied */ nc_bool_t fletcher32; /**< True if var has fletcher32 filter applied */ size_t chunk_cache_size, chunk_cache_nelems; float chunk_cache_preemption; void *format_var_info; /**< Pointer to any binary format info. */ - unsigned int filterid; /**< ID for arbitrary filter. */ - size_t nparams; /**< nparams for arbitrary filter. */ - unsigned int *params; /**< Params for arbitrary filter. */ + NClist* filters; /**< List */ } NC_VAR_INFO_T; /** This is a struct to handle the field metadata from a user-defined @@ -452,4 +449,57 @@ int log_metadata_nc(NC_FILE_INFO_T *h5); /* Binary searcher for reserved attributes */ extern const NC_reservedatt *NC_findreserved(const char *name); +/**************************************************/ +/* Internal filter related structures */ + +/* Internal filter actions */ +#define NCFILTER_DEF 1 +#define NCFILTER_REMOVE 2 +#define NCFILTER_INQ 3 +#define NCFILTER_FILTERIDS 4 +#define NCFILTER_INFO 5 +#define NCFILTER_FREESPEC 6 +#define NCFILTER_CLIENT_REG 10 +#define NCFILTER_CLIENT_UNREG 11 +#define NCFILTER_CLIENT_INQ 12 + +typedef enum NC_FILTER_SORT { + NC_FILTER_SORT_SPEC=((int)1), + NC_FILTER_SORT_IDS=((int)2), + NC_FILTER_SORT_CLIENT=((int)3), +} NC_FILTER_SORT; + +/* Provide structs to pass args to filter_actions function for HDF5*/ + +typedef struct NC_FILTER_SPEC_HDF5 { + int active; /**< true iff HDF5 library was told to activate filter */ + unsigned int filterid; /**< ID for arbitrary filter. */ + size_t nparams; /**< nparams for arbitrary filter. */ + unsigned int* params; /**< Params for arbitrary filter. */ +} NC_FILTER_SPEC_HDF5; + +typedef struct NC_FILTERIDS_HDF5 { + size_t nfilters; /**< number of filters */ + unsigned int* filterids; /**< Filter ids. */ +} NC_FILTERIDS_HDF5; + +typedef struct NC_FILTER_CLIENT_HDF5 { + unsigned int id; + /* The filter info for hdf5 */ + /* Avoid needing hdf.h by using void* */ + void* info; +} NC_FILTER_CLIENT_HDF5; + +typedef struct NC_FILTER_OBJ_HDF5 { + NC_Filterobject hdr; /* So we can cast it */ + NC_FILTER_SORT sort; /* discriminate union */ + union { + NC_FILTER_SPEC_HDF5 spec; + NC_FILTERIDS_HDF5 ids; + NC_FILTER_CLIENT_HDF5 client; + } u; +} NC_FILTER_OBJ_HDF5; + +extern void NC4_freefilterspec(NC_FILTER_SPEC_HDF5* f); + #endif /* _NC4INTERNAL_ */ diff --git a/include/ncdispatch.h b/include/ncdispatch.h index a87d8e1732..5a4a612228 100644 --- a/include/ncdispatch.h +++ b/include/ncdispatch.h @@ -165,7 +165,6 @@ struct nc_vlen_t; struct NC; - int NC_create(const char *path, int cmode, size_t initialsz, int basepe, size_t *chunksizehintp, int useparallel, void *parameters, int *ncidp); diff --git a/include/netcdf.h b/include/netcdf.h index 0e7951e2e2..67a8d6186a 100644 --- a/include/netcdf.h +++ b/include/netcdf.h @@ -475,7 +475,13 @@ by the desired type. */ #define NC_ERCFILE (-133) /**< RC file failure */ #define NC_ENULLPAD (-134) /**< Header Bytes not Null-Byte padded */ #define NC_EINMEMORY (-135) /**< In-memory file error */ -#define NC4_LAST_ERROR (-136) /**< @internal All netCDF errors > this. */ +#define NC_ENOFILTER (-136) /**< Filter not defined on variable. */ + +#define NC4_LAST_ERROR (-137) /**< @internal All netCDF errors > this. */ + +/* Errors for all remote access methods(e.g. DAP and CDMREMOTE)*/ +#define NC_EURL (NC_EDAPURL) /**< Malformed URL */ +#define NC_ECONSTRAINT (NC_EDAPCONSTRAINT) /**< Malformed Constraint*/ /** @internal This is used in netCDF-4 files for dimensions without * coordinate vars. */ @@ -486,10 +492,6 @@ by the desired type. */ * size_t. Doh! */ #define NC_HAVE_NEW_CHUNKING_API 1 -/* Errors for all remote access methods(e.g. DAP and CDMREMOTE)*/ -#define NC_EURL (NC_EDAPURL) /**< Malformed URL */ -#define NC_ECONSTRAINT (NC_EDAPCONSTRAINT) /**< Malformed Constraint*/ - /* * The Interface */ @@ -900,7 +902,7 @@ nc_inq_var_endian(int ncid, int varid, int *endianp); EXTERNL int nc_def_var_filter(int ncid, int varid, unsigned int id, size_t nparams, const unsigned int* parms); -/* Learn about the filter on a variable */ +/* Learn about the first filter on a variable */ EXTERNL int nc_inq_var_filter(int ncid, int varid, unsigned int* idp, size_t* nparams, unsigned int* params); diff --git a/include/netcdf_dispatch.h b/include/netcdf_dispatch.h index aa42a62544..e34d8a9df5 100644 --- a/include/netcdf_dispatch.h +++ b/include/netcdf_dispatch.h @@ -23,6 +23,9 @@ #ifndef NETCDF_DISPATCH_H #define NETCDF_DISPATCH_H +/* Forward */ +struct NC_Filterobject; + /* This is the dispatch table, with a pointer to each netCDF * function. */ struct NC_Dispatch @@ -137,6 +140,10 @@ struct NC_Dispatch int (*set_var_chunk_cache)(int, int, size_t, size_t, float); int (*get_var_chunk_cache)(int ncid, int varid, size_t *sizep, size_t *nelemsp, float *preemptionp); + + /* Dispatch table Version 2 or later */ + /* Handle all filter related actions. */ + int (*filter_actions)(int ncid, int varid, int action, struct NC_Filterobject*); }; #if defined(__cplusplus) @@ -219,6 +226,8 @@ extern "C" { EXTERNL int NC_NOTNC4_inq_user_type(int, nc_type, char *, size_t *, nc_type *, size_t *, int *); EXTERNL int NC_NOTNC4_inq_typeid(int, const char *, nc_type *); + EXTERNL int NC_NOTNC4_filter_actions(int, int, int, struct NC_Filterobject*); + #if defined(__cplusplus) } #endif diff --git a/include/netcdf_filter.h b/include/netcdf_filter.h index ca4ae79e6e..2dccb8e6ec 100644 --- a/include/netcdf_filter.h +++ b/include/netcdf_filter.h @@ -14,45 +14,95 @@ /* API for libdispatch/dfilter.c */ /* Must match values in */ +#ifndef H5Z_FILTER_DEFLATE +#define H5Z_FILTER_DEFLATE 1 +#endif #ifndef H5Z_FILTER_SZIP #define H5Z_FILTER_SZIP 4 +#define H5_SZIP_ALLOW_K13_OPTION_MASK 1 +#define H5_SZIP_CHIP_OPTION_MASK 2 +#define H5_SZIP_EC_OPTION_MASK 4 +#define H5_SZIP_NN_OPTION_MASK 32 +#define H5_SZIP_MAX_PIXELS_PER_BLOCK 32 #endif -#define NC_SZIP_EC 4 /**< Selects entropy coding method for szip. */ -#define NC_SZIP_NN 32 /**< Selects nearest neighbor coding method for szip. */ - -/* Define the known filter formats */ -#define NC_FILTER_FORMAT_HDF5 1 /* Use the H5Z_class2_t format */ +#define H5_SZIP_ALL_MASKS (H5_SZIP_CHIP_OPTION_MASK|H5_SZIP_EC_OPTION_MASK|H5_SZIP_NN_OPTION_MASK) -/* Note that this structure can be extended - in the usual C way if the first field of the extended - struct is of type NC_FILTER_INFO -*/ -typedef struct NC_FILTER_INFO { - int version; /* Of this structure */ -# define NC_FILTER_INFO_VERSION 1 - int format; /* Controls actual type of this structure */ - int id; /* Must be unique WRT format */ - void* info; /* The filter info as defined by the format. - For format == NC_FILTER_FORMAT_HDF5, - this must conform to H5Z_class2_t in H5Zpublic.h; - Defined as void* to avoid specifics. - */ -} NC_FILTER_INFO; +/** The maximum allowed setting for pixels_per_block when calling nc_def_var_szip(). */ +#define NC_MAX_PIXELS_PER_BLOCK 32 #if defined(__cplusplus) extern "C" { #endif -/* Provide consistent filter spec parser */ -EXTERNL int NC_parsefilterspec(const char* spec, unsigned int* idp, size_t* nparamsp, unsigned int** paramsp); +/* Define the formats for NC_FILTER classes */ +#define NC_FILTER_FORMAT_HDF5 (NC_FORMATX_NC_HDF5) + +/* Define the sort for NC_FILTER classes */ +//#define NC_FILTER_SORT_SPEC ((int)1) /* Use for NC_Filterspec; Must match nc4internal.h */ + +/* Define a Header Object for all filter-related objects */ + +/* provide a common generic struct field */ +/* + format indicates e.g. HDF5 + sort indicates the "subclass" of the superclass +*/ +typedef struct NC_Filterobject {int format;} NC_Filterobject; + +/* Generic version of Filterspec */ +typedef struct NC_Filterspec { + NC_Filterobject hdr; /**< e.g. NC_FILTER_FORMAT_HDF5 */ +} NC_Filterspec; + +/**************************************************/ +/* HDF5 Specific filter functions */ + +/*Define a filter for a variable */ +EXTERNL int +nc_def_var_filter(int ncid, int varid, unsigned int id, size_t nparams, const unsigned int* parms); + +/* Learn about the first defined filter filter on a variable */ +EXTERNL int +nc_inq_var_filter(int ncid, int varid, unsigned int* idp, size_t* nparams, unsigned int* params); + +/* Support inquiry about all the filters associated with a variable */ +/* As is usual, it is expected that this will be called twice: + once to get the number of filters, and then a second time to read the ids */ +EXTERNL int nc_inq_var_filterids(int ncid, int varid, size_t* nfilters, unsigned int* filterids); + +/* Learn about the filter with specified id wrt a variable */ +EXTERNL int +nc_inq_var_filter_info(int ncid, int varid, unsigned int id, size_t* nparams, unsigned int* params); + +/* Remove filter from variable*/ +EXTERNL int nc_var_filter_remove(int ncid, int varid, unsigned int id); + +/* Support direct user defined filters; + last arg is void*, but is actually H5Z_class2_t*. + It is void* to avoid having to reference hdf.h. +*/ +EXTERNL int nc_filter_client_register(unsigned int id, void*/*H5Z_class2_t* */); +EXTERNL int nc_filter_client_unregister(unsigned int id); +EXTERNL int nc_filter_client_inq(unsigned int id, void*/*H5Z_class2_t* */); + +/* HDF5 specific filter info */ +typedef struct NC4_Filterspec { + NC_Filterspec hdr; + /* HDF5 specific extensions */ + unsigned int filterid; /**< ID for arbitrary filter. */ + size_t nparams; /**< nparams for arbitrary filter. */ + unsigned int* params; /**< Params for arbitrary filter. */ +} NC4_Filterspec; + +EXTERNL void NC4_filterfix8(unsigned char* mem, int decode); + +EXTERNL int NC_parsefilterlist(const char* listspec, int* formatp, size_t* nfilters, NC_Filterspec*** filtersp); +EXTERNL int NC_parsefilterspec(const char* txt, int format, NC_Filterspec** specp); -EXTERNL void NC_filterfix8(unsigned char* mem, int decode); +/* End HDF5 Specific Declarations */ -/* Support direct user defined filters */ -EXTERNL int nc_filter_register(NC_FILTER_INFO* filter_info); -EXTERNL int nc_filter_unregister(int format, int id); -EXTERNL int nc_filter_inq(int format, int id, NC_FILTER_INFO* filter_info); +/**************************************************/ #if defined(__cplusplus) } diff --git a/libdap2/ncd2dispatch.c b/libdap2/ncd2dispatch.c index 7593e98798..b7f5524eab 100644 --- a/libdap2/ncd2dispatch.c +++ b/libdap2/ncd2dispatch.c @@ -174,6 +174,8 @@ NCD2_def_var_filter, NCD2_set_var_chunk_cache, NCD2_get_var_chunk_cache, +NC_NOTNC4_filter_actions, + }; const NC_Dispatch* NCD2_dispatch_table = NULL; /* moved here from ddispatch.c */ diff --git a/libdap4/ncd4dispatch.c b/libdap4/ncd4dispatch.c index 780a53383e..effd978659 100644 --- a/libdap4/ncd4dispatch.c +++ b/libdap4/ncd4dispatch.c @@ -241,6 +241,14 @@ NCD4_set_var_chunk_cache(int ncid, int p2, size_t p3, size_t p4, float p5) return (NC_EPERM); } +struct NC_FILTER_ACTION; + +static int +NCD4_filter_actions(int ncid, int varid, int action, struct NC_Filterobject* spec) +{ + return (NC_EPERM); +} + /**************************************************/ /* Following functions basically return the netcdf-4 value WRT to the nc4id. @@ -861,6 +869,7 @@ NCD4_def_var_filter, NCD4_set_var_chunk_cache, NCD4_get_var_chunk_cache, +NCD4_filter_actions, }; diff --git a/libdispatch/derror.c b/libdispatch/derror.c index 4ee6adbc1f..0cc87e7798 100644 --- a/libdispatch/derror.c +++ b/libdispatch/derror.c @@ -258,7 +258,9 @@ const char *nc_strerror(int ncerr1) case NC_EDISKLESS: return "NetCDF: Error in using diskless access"; case NC_EFILTER: - return "NetCDF: Filter error: bad id or parameters or filter library non-existent"; + return "NetCDF: Filter error: bad id or parameters"; + case NC_ENOFILTER: + return "NetCDF: Filter error: filter not defined for variable"; case NC_ECANTEXTEND: return "NetCDF: Attempt to extend dataset during NC_INDEPENDENT I/O operation. Use nc_var_par_access to set mode NC_COLLECTIVE before extending variable."; case NC_EMPI: return "NetCDF: MPI operation failed."; diff --git a/libdispatch/dfilter.c b/libdispatch/dfilter.c index ed541333e6..225b88ec1c 100644 --- a/libdispatch/dfilter.c +++ b/libdispatch/dfilter.c @@ -13,39 +13,93 @@ #include "netcdf.h" #include "netcdf_filter.h" +#include "ncdispatch.h" +#include "nc4internal.h" #ifdef USE_HDF5 #include "hdf5internal.h" #endif /* -Common utilities related to filters. +Unified filter related code */ +#ifndef H5Z_FILTER_SZIP +/** ID of HDF SZIP filter. */ +#define H5Z_FILTER_SZIP 4 +#endif + +#define LPAREN '(' +#define RPAREN ')' +#define LBRACK '[' +#define RBRACK ']' + +#define NUMCHAR "0123456789" +#define NAMECHAR1 "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" +#define NAMECHARN (NAMECHAR1 NUMCHAR "_-") /* Forward */ static int gettype(const int q0, const int q1, int* unsignedp); +const struct LegalFormat { + const char* tag; + int format; +} legalformats[] = { + {"hdf5", NC_FILTER_FORMAT_HDF5}, + {NULL, 0}, +}; + +const struct FilterName { + const char* name; /* name or alias as assigned by HDF group*/ + unsigned int id; /* id as assigned by HDF group*/ +} known_filters[] = { +{"zip", 2}, /* Standard zlib compression */ +{"zlib", 2}, /* alias */ +{"deflate", 2}, /* alias */ +{"szip", 4}, /* Standard szip compression */ +{"bzip2", 307}, /* BZIP2 lossless compression used by PyTables */ +{"lzf", 32000}, /* LZF lossless compression used by H5Py project */ +{"blosc", 32001}, /* Blosc lossless compression used by PyTables */ +{"mafisc", 32002}, /* Modified LZMA compression filter, MAFISC (Multidimensional Adaptive Filtering Improved Scientific data Compression) */ +{"snappy", 32003}, /* Snappy lossless compression. */ +{"lz4", 32004}, /* LZ4 fast lossless compression algorithm */ +{"apax", 32005}, /* Samplify's APAX Numerical Encoding Technology */ +{"cbf", 32006}, /* All imgCIF/CBF compressions and decompressions, including Canonical, Packed, Packed Vesrsion 2, Byte Offset and Nibble Offset. */ +{"jpeg-xr", 32007}, /* Enables images to be compressed/decompressed with JPEG-XR compression */ +{"bitshuffle", 32008}, /* Extreme version of shuffle filter that shuffles data at bit level instead of byte level. */ +{"spdp", 32009}, /* SPDP fast lossless compression algorithm for single- and double-precision floating-point data. */ +{"lpc-rice", 32010}, /* LPC-Rice multi-threaded lossless compression */ +{"ccsds-123", 32011}, /* ESA CCSDS-123 multi-threaded compression filter */ +{"jpeg-ls", 32012}, /* CharLS JPEG-LS multi-threaded compression filter */ +{"zfp", 32013}, /* Rate, accuracy or precision bounded compression for floating-point arrays */ +{"fpzip", 32014}, /* Fast and Efficient Lossy or Lossless Compressor for Floating-Point Data */ +{"zstandard", 32015}, /* Real-time compression algorithm with wide range of compression / speed trade-off and fast decoder */ +{"b3d", 32016}, /* GPU based image compression method developed for light-microscopy applications */ +{"sz", 32017}, /* An error-bounded lossy compressor for scientific floating-point data */ +{"fcidecomp", 32018}, /* EUMETSAT CharLS compression filter for use with netCDF */ +{"user-defined", 32768}, /* First user-defined filter */ +{NULL,0} +}; + /**************************************************/ /* -Parse a filter spec string into a filter id + a vector -of unsigned ints. +Parse a filter spec string into a NC_FILTER_SPEC* -@param spec0 - a string containing the spec as a sequence of +@param txt - a string containing the spec as a sequence of constants separated by commas. -@param idp - store the parsed filter id here -@param nparamsp - store number of parsed filter params here -@param paramsp - store the vector of parsed filter params here -@return 1 if parse succeeded, 0 otherwise. +@param specp - store the parsed filter here -- caller frees +@return NC_NOERR if parse succeeded +@return NC_EINVAL otherwise */ EXTERNL int -NC_parsefilterspec(const char* spec, unsigned int* idp, size_t* nparamsp, unsigned int** paramsp) +NC_parsefilterspec(const char* txt, int format, NC_Filterspec** specp) { int stat = NC_NOERR; int sstat; /* for scanf */ char* p; - char* sdata = NULL; + char* sdata0 = NULL; /* what to free */ + char* sdata = NULL; /* sdata0 with leading prefix skipped */ unsigned int id; size_t count; /* no. of comma delimited params */ size_t nparams; /* final no. of unsigned ints */ @@ -54,8 +108,14 @@ NC_parsefilterspec(const char* spec, unsigned int* idp, size_t* nparamsp, unsign unsigned int* ulist = NULL; unsigned char mem[8]; - if(spec == NULL || strlen(spec) == 0) goto fail; - sdata = strdup(spec); + if(txt == NULL) goto fail; + len = strlen(txt); + if(len == 0) goto fail; + + sdata0 = (char*)calloc(1,len+1+1); + if(sdata0 == NULL) return 0; + memcpy(sdata0,txt,len); + sdata = sdata0; /* Count number of parameters + id and delimit */ p=sdata; @@ -72,11 +132,27 @@ NC_parsefilterspec(const char* spec, unsigned int* idp, size_t* nparamsp, unsign /* Extract the filter id */ p = sdata; - sstat = sscanf(p,"%u",&id); - if(sstat != 1) goto fail; + if(strchr(NAMECHAR1,*p) != NULL) { + const struct FilterName* candidate = known_filters; + char* q = p+1; + while(*q && strchr(NAMECHARN,*q) != NULL) { + q++; + } + if(*q) goto fail; /*name has bad char*/ + /* Lookup name */ + id = 0; /* => not found */ + for(;candidate->name;candidate++) { + if(strcasecmp(candidate->name,p)==0) {id = candidate->id; break;} + } + if(id == 0) goto fail; /* unknown name */ + } else if(strchr(NUMCHAR,*p) != NULL) { + sstat = sscanf(p,"%u",&id); + if(sstat != 1) goto fail; + } else goto fail; /* unparseable id */ + count--; + /* skip past the filter id */ p = p + strlen(p) + 1; - count--; /* Allocate the max needed space; *2 in case the params are all doubles */ ulist = (unsigned int*)malloc(sizeof(unsigned int)*(count)*2); @@ -146,7 +222,7 @@ NC_parsefilterspec(const char* spec, unsigned int* idp, size_t* nparamsp, unsign sstat = sscanf(p,"%lf",&vald); if(sstat != 1) goto fail; memcpy(mem,&vald,sizeof(mem)); - NC_filterfix8(mem,0); + NC4_filterfix8(mem,0); vector = (unsigned int*)mem; ulist[nparams++] = vector[0]; ulist[nparams++] = vector[1]; @@ -158,7 +234,7 @@ NC_parsefilterspec(const char* spec, unsigned int* idp, size_t* nparamsp, unsign sstat = sscanf(p,"%lld",(long long*)&val64u); if(sstat != 1) goto fail; memcpy(mem,&val64u,sizeof(mem)); - NC_filterfix8(mem,0); + NC4_filterfix8(mem,0); vector = (unsigned int*)&mem; ulist[nparams++] = vector[0]; ulist[nparams++] = vector[1]; @@ -169,21 +245,402 @@ NC_parsefilterspec(const char* spec, unsigned int* idp, size_t* nparamsp, unsign p = p + strlen(p) + 1; /* move to next param */ } /* Now return results */ - if(idp) *idp = id; - if(nparamsp) *nparamsp = nparams; - if(paramsp) { - *paramsp = ulist; - ulist = NULL; /* avoid duplicate free */ + if(specp != NULL) { + NC4_Filterspec* pfs = calloc(1,sizeof(NC4_Filterspec)); + if(pfs == NULL) {stat = NC_ENOMEM; goto done;} + pfs->hdr.hdr.format = format; + pfs->filterid = id; + pfs->nparams = nparams; + pfs->params = ulist; ulist = NULL; + *specp = (NC_Filterspec*)pfs; } + done: if(sdata) free(sdata); if(ulist) free(ulist); return stat; fail: - stat = NC_EFILTER; + stat = NC_EINVAL; goto done; } +/* +Parse a string containing multiple '|' separated filter specs. + +@param spec0 - a string containing the list of filter specs. +@param nspecsp - # of parsed specs +@param specsp - pointer to hold vector of parsed specs. Caller frees +@return NC_NOERR if parse succeeded +@return NC_EINVAL if bad parameters or parse failed +*/ + +EXTERNL int +NC_parsefilterlist(const char* txt0, int* formatp, size_t* nspecsp, NC_Filterspec*** vectorp) +{ + int stat = NC_NOERR; + int format = NC_FILTER_FORMAT_HDF5; /* default */ + size_t len = 0; + size_t nspecs = 0; + NC4_Filterspec** vector = NULL; + char* spec0 = NULL; /* with prefix */ + char* spec = NULL; /* without prefix */ + char* p = NULL; + char* q = NULL; + + if(txt0 == NULL) return NC_EINVAL; + /* Duplicate txt0 so we can modify it */ + len = strlen(txt0); + if((spec = calloc(1,len+1+1)) == NULL) return NC_ENOMEM; + memcpy(spec,txt0,len); /* Note double ending nul */ + spec0 = spec; /* Save for later free */ + + /* See if there is a prefix '[format]' tag */ + if(spec[0] == LBRACK) { + int found = 0; + const struct LegalFormat* candidates = legalformats; + p = spec + 1; + q = strchr(p,RBRACK); + if(q == NULL) {stat = NC_EINVAL; goto done;} + *q++ = '\0'; /* delimit tag */ + while(candidates->tag != NULL) { + if(strcasecmp(p,candidates->tag) == 0) { + found = 1; + format = candidates->format; + } + } + if(found == 0) {stat = NC_EINVAL; goto done;} + spec = q; /* skip tag wrt later processing */ + } + if(formatp) *formatp = format; + + if(format != NC_FILTER_FORMAT_HDF5) {stat = NC_EINVAL; goto done;} /* for now */ + + /* pass 1: count number of specs */ + p = spec; + nspecs = 0; + while(*p) { + q = strchr(p,'|'); + if(q == NULL) q = p + strlen(p); /* fake it */ + nspecs++; + p = q + 1; + } + if(nspecs > 0) { + int count = 0; + if((vector = (NC4_Filterspec**)malloc(sizeof(NC4_Filterspec*)*nspecs)) == NULL) + {stat = NC_ENOMEM; goto done;} + /* pass 2: parse */ + p = spec; + for(count=0;countparams) free(nfs->params); + nullfree(nfs); + } + free(vector); + } + return stat; +} + +EXTERNL void +NC4_filterfix8(unsigned char* mem, int decode) +{ +#ifdef WORDS_BIGENDIAN + if(decode) { /* Apply inverse of the encode case */ + byteswap4(mem); /* step 1: byte-swap each piece */ + byteswap4(mem+4); + byteswap8(mem); /* step 2: convert to little endian format */ + } else { /* encode */ + byteswap8(mem); /* step 1: convert to little endian format */ + byteswap4(mem); /* step 2: byte-swap each piece */ + byteswap4(mem+4); + } +#else /* Little endian */ + /* No action is necessary */ +#endif +} + + +/* Support direct user defined filters */ + +/* Use void* to avoid having to include hdf.h*/ +EXTERNL int +nc_filter_client_register(unsigned int id, void* info) +{ + int stat = NC_NOERR; + if(id == 0 ||info == NULL) + return NC_EINVAL; +#ifdef USE_HDF5 + NC_FILTER_OBJ_HDF5 client; + memset(&client,0,sizeof(client)); + client.hdr.format = NC_FILTER_FORMAT_HDF5; + client.sort = NC_FILTER_SORT_CLIENT; + client.u.client.id = id; + client.u.client.info = info; + /* Note use of a global function, not part of the dispatch table */ + stat = nc4_global_filter_action(NCFILTER_CLIENT_REG, id, &client); +#else + stat = NC_ENOTBUILT; +#endif + return stat; +} + +EXTERNL int +nc_filter_client_unregister(unsigned int id) +{ + int stat = NC_NOERR; +#ifdef USE_HDF5 + stat = nc4_global_filter_action(NCFILTER_CLIENT_UNREG, id, NULL); +#else + stat = NC_ENOTBUILT; +#endif + return stat; +} + +/* Use void* to avoid having to include hdf.h*/ +EXTERNL int +nc_filter_client_inq(unsigned int id, void* infop) +{ + int stat = NC_NOERR; +#ifdef USE_HDF5 + H5Z_class2_t* hct = (H5Z_class2_t*)infop; + NC_FILTER_OBJ_HDF5 client; + if(id == 0 ||infop == NULL) + return NC_EINVAL; + memset(&client,0,sizeof(client)); + client.hdr.format = NC_FILTER_FORMAT_HDF5; + client.sort = NC_FILTER_SORT_CLIENT; + client.u.client.id = id; + client.u.client.info = hct; + /* Note use of a global function, not part of the dispatch table */ + stat = nc4_global_filter_action(NCFILTER_CLIENT_INQ, id, &client); + if(stat == NC_NOERR) { + *hct = *(H5Z_class2_t*)client.u.client.info; + } +#else + stat = NC_ENOTBUILT; +#endif + return stat; +} + +/** +Find the set of filters (if any) associated with a variable. + +\param ncid NetCDF or group ID, from a previous call to nc_open(), +nc_create(), nc_def_grp(), or associated inquiry functions such as +nc_inq_ncid(). + +\param varid Variable ID +\param nfilters return no. of filters +\param ids return the filter ids (caller allocates) + +\returns ::NC_NOERR No error. +\returns ::NC_ENOTNC4 Not a netCDF-4 file. +\returns ::NC_EBADID Bad ncid. +\returns ::NC_ENOTVAR Invalid variable ID. +\returns ::NC_EINVAL Invalid arguments +\ingroup variables +\author Dennis Heimbigner +*/ +EXTERNL int +nc_inq_var_filterids(int ncid, int varid, size_t* nfiltersp, unsigned int* ids) +{ + NC* ncp; + int stat = NC_check_id(ncid,&ncp); + NC_FILTER_OBJ_HDF5 ncids; + + if(stat != NC_NOERR) return stat; + TRACE(nc_inq_var_filterids); + + memset(&ncids,0,sizeof(ncids)); + ncids.hdr.format = NC_FILTER_FORMAT_HDF5; + ncids.sort = NC_FILTER_SORT_IDS; + ncids.u.ids.nfilters = (nfiltersp?*nfiltersp:0); + ncids.u.ids.filterids = ids; + + if((stat = ncp->dispatch->filter_actions(ncid,varid, NCFILTER_FILTERIDS, (NC_Filterobject*)&ncids)) == NC_NOERR) { + if(nfiltersp) *nfiltersp = ncids.u.ids.nfilters; + } + return stat; + } + +/** +Find the the param info about filter (if any) +associated with a variable and with specified id. + +This is a wrapper for nc_inq_var_all(). + +\param ncid NetCDF or group ID, from a previous call to nc_open(), +nc_create(), nc_def_grp(), or associated inquiry functions such as +nc_inq_ncid(). + +\param varid Variable ID +\param id The filter id of interest +\param formatp (Out) Storage for the filter format +\param nparamsp (Out) Storage which will get the number of parameters to the filter +\param params (Out) Storage which will get associated parameters. +Note: the caller must allocate and free. + +\returns ::NC_NOERR No error. +\returns ::NC_ENOTNC4 Not a netCDF-4 file. +\returns ::NC_EBADID Bad ncid. +\returns ::NC_ENOTVAR Invalid variable ID. +\returns ::NC_ENOFILTER No filter defined. +\ingroup variables +\author Dennis Heimbigner +*/ +EXTERNL int +nc_inq_var_filter_info(int ncid, int varid, unsigned int id, size_t* nparamsp, unsigned int* params) +{ + NC* ncp; + int stat = NC_check_id(ncid,&ncp); + NC_FILTER_OBJ_HDF5 spec; + + if(stat != NC_NOERR) return stat; + TRACE(nc_inq_var_filter_info_hdf5); + + memset(&spec,0,sizeof(spec)); + spec.hdr.format = NC_FILTER_FORMAT_HDF5; + spec.sort = NC_FILTER_SORT_SPEC; + spec.u.spec.filterid = id; + spec.u.spec.nparams = (nparamsp?*nparamsp:0); + spec.u.spec.params = params; + + if((stat = ncp->dispatch->filter_actions(ncid,varid,NCFILTER_INFO,(NC_Filterobject*)&spec)) == NC_NOERR) { + if(nparamsp) *nparamsp = spec.u.spec.nparams; + } + return stat; +} + +/** +Find the first filter (if any) associated with a variable. + +\param ncid NetCDF or group ID, from a previous call to nc_open(), +nc_create(), nc_def_grp(), or associated inquiry functions such as +nc_inq_ncid(). + +\param varid Variable ID + +\param idp Storage which will get the filter id; a return value of zero means no filter + +\param nparamsp Storage which will get the number of parameters to the +filter + +\param params Storage which will get associated parameters. +Note: the caller must allocate and free. + +\returns ::NC_NOERR No error. +\returns ::NC_ENOTNC4 Not a netCDF-4 file. +\returns ::NC_EBADID Bad ncid. +\returns ::NC_ENOTVAR Invalid variable ID. +\returns ::NC_ENOFILTER No filter defined. +\ingroup variables +\author Dennis Heimbigner +*/ +EXTERNL int +nc_inq_var_filter(int ncid, int varid, unsigned int* idp, size_t* nparamsp, unsigned int* params) +{ + NC* ncp; + NC_FILTER_OBJ_HDF5 spec; + int stat = NC_check_id(ncid,&ncp); + + if(stat != NC_NOERR) return stat; + TRACE(nc_inq_var_filter); + + memset(&spec,0,sizeof(spec)); + spec.hdr.format = NC_FILTER_FORMAT_HDF5; + spec.sort = NC_FILTER_SORT_SPEC; + spec.u.spec.filterid = (idp?*idp:0); + spec.u.spec.nparams = (nparamsp?*nparamsp:0); + spec.u.spec.params = params; + + if((stat=ncp->dispatch->filter_actions(ncid,varid,NCFILTER_INQ,(NC_Filterobject*)&spec))) + return stat; + if(idp) *idp = spec.u.spec.filterid; + if(nparamsp) *nparamsp = spec.u.spec.nparams; + return stat; +} + +/** + Define a new variable hdf5 filter. + + @param ncid File and group ID. + @param varid Variable ID. + @param id + @param nparams Number of filter parameters. + @param parms Filter parameters. + + @return ::NC_NOERR No error. + @return ::NC_EBADID Bad ID. + @author Dennis Heimbigner +*/ + +EXTERNL int +nc_def_var_filter(int ncid, int varid, unsigned int id, size_t nparams, const unsigned int* params) +{ + NC* ncp; + NC_FILTER_OBJ_HDF5 spec; + int stat = NC_check_id(ncid,&ncp); + + if(stat != NC_NOERR) return stat; + TRACE(nc_def_var_filter_hdf5); + + memset(&spec,0,sizeof(spec)); + spec.hdr.format = NC_FILTER_FORMAT_HDF5; + spec.sort = NC_FILTER_SORT_SPEC; + spec.u.spec.filterid = id; + spec.u.spec.nparams = nparams; + spec.u.spec.params = (unsigned int*)params; /* discard const */ + return ncp->dispatch->filter_actions(ncid,varid,NCFILTER_DEF,(NC_Filterobject*)&spec); +} + +/** + Remove all filters with specified id from a variable + + @param ncid File and group ID. + @param varid Variable ID. + @param id filter to remove + + @return ::NC_NOERR No error. + @return ::NC_EBADID Bad ID. + @author Dennis Heimbigner +*/ +EXTERNL int +nc_var_filter_remove(int ncid, int varid, unsigned int id) +{ + NC* ncp; + int stat = NC_check_id(ncid,&ncp); + NC_FILTER_OBJ_HDF5 spec; + + if(stat != NC_NOERR) return stat; + TRACE(nc_var_filter_hdf5_remove); + + memset(&spec,0,sizeof(spec)); + spec.hdr.format = NC_FILTER_FORMAT_HDF5; + spec.sort = NC_FILTER_SORT_SPEC; + spec.u.spec.filterid = id; + return ncp->dispatch->filter_actions(ncid,varid,NCFILTER_REMOVE,(NC_Filterobject*)&spec); +} + +/**************************************************/ +/* Utilities */ + /* Look at q0 and q1) to determine type */ static int gettype(const int q0, const int q1, int* isunsignedp) @@ -249,79 +706,4 @@ byteswap4(unsigned char* mem) } #endif -EXTERNL void -NC_filterfix8(unsigned char* mem, int decode) -{ -#ifdef WORDS_BIGENDIAN - if(decode) { /* Apply inverse of the encode case */ - byteswap4(mem); /* step 1: byte-swap each piece */ - byteswap4(mem+4); - byteswap8(mem); /* step 2: convert to little endian format */ - } else { /* encode */ - byteswap8(mem); /* step 1: convert to little endian format */ - byteswap4(mem); /* step 2: byte-swap each piece */ - byteswap4(mem+4); - } -#else /* Little endian */ - /* No action is necessary */ -#endif -} - -/**************************************************/ -/* Support direct user defined filters */ - -EXTERNL int -nc_filter_register(NC_FILTER_INFO* filter) -{ - int stat = NC_NOERR; - if(filter == NULL) - return NC_EINVAL; - switch (filter->format) { - case NC_FILTER_FORMAT_HDF5: -#ifdef USE_HDF5 - stat = nc4_filter_action(FILTER_REG, filter->format, filter->id, filter); -#else - stat = NC_ENOTBUILT; -#endif - break; - default: - stat = NC_EINVAL; - } - return stat; -} - -EXTERNL int -nc_filter_unregister(int fformat, int id) -{ - int stat = NC_NOERR; - switch (fformat) { - case NC_FILTER_FORMAT_HDF5: -#ifdef USE_HDF5 - stat = nc4_filter_action(FILTER_UNREG, fformat, id, NULL); -#else - stat = NC_ENOTBUILT; -#endif - break; - default: - stat = NC_EINVAL; - } - return stat; -} -EXTERNL int -nc_filter_inq(int fformat, int id, NC_FILTER_INFO* filter_info) -{ - int stat = NC_NOERR; - switch (fformat) { - case NC_FILTER_FORMAT_HDF5: -#ifdef USE_HDF5 - stat = nc4_filter_action(FILTER_INQ, fformat, id, filter_info); -#else - stat = NC_ENOTBUILT; -#endif - break; - default: - stat = NC_EINVAL; - } - return stat; -} diff --git a/libdispatch/dnotnc4.c b/libdispatch/dnotnc4.c index bce2bddfc3..979a1a6b73 100644 --- a/libdispatch/dnotnc4.c +++ b/libdispatch/dnotnc4.c @@ -8,7 +8,10 @@ * @author Ed Hartnett */ +#include "netcdf.h" +#include "netcdf_filter.h" #include "ncdispatch.h" +#include "nc4internal.h" /** * @internal Not allowed for classic model. @@ -610,3 +613,19 @@ NC_NOTNC4_inq_typeid(int ncid, const char *name, nc_type *typeidp) { return NC_ENOTNC4; } + +/** + * @internal Carry out one of several filter actions + * + * @param ncid Containing group id + * @param varid Containing variable id + * @param action Action to perform + * + * @return ::NC_ENOTNC4 Not allowed for classic model. + * @author D. Heimbigner + */ +int +NC_NOTNC4_filter_actions(int ncid, int varid, int action, struct NC_Filterobject* spec) +{ + return NC_ENOTNC4; +} diff --git a/libdispatch/dutil.c b/libdispatch/dutil.c index 307bf6380d..05a644d82b 100644 --- a/libdispatch/dutil.c +++ b/libdispatch/dutil.c @@ -348,6 +348,6 @@ NC_testmode(const char* path, const char* tag) } done: - nclistfree(modelist); + nclistfreeall(modelist); return found; } diff --git a/libdispatch/dvar.c b/libdispatch/dvar.c index 1ed70e9f95..9d5ccb40c9 100644 --- a/libdispatch/dvar.c +++ b/libdispatch/dvar.c @@ -6,7 +6,11 @@ * order of functions in this file affects the doxygen documentation. */ +#include "config.h" +#include "netcdf.h" +#include "netcdf_filter.h" #include "ncdispatch.h" +#include "nc4internal.h" #include "netcdf_f.h" #include "nc4internal.h" @@ -666,37 +670,6 @@ nc_def_var_endian(int ncid, int varid, int endian) return ncp->dispatch->def_var_endian(ncid,varid,endian); } -/** - * Define a new variable filter. - * - * HDF5 filters are plug-in libraries which can modify data when it is - * being read or written. - * - * @note Variables must be chunked to use filters. Calling - * nc_def_var_filter() on a variable causes its storage to be changed - * to chunked; default chunksizes are selected. - * - * @param ncid File and group ID. - * @param varid Variable ID. - * @param id ID of the HDF5 filter. - * @param nparams Number of filter parameters. - * @param parms Filter parameters. - * - * @return ::NC_NOERR No error. - * @return ::NC_EBADID Bad ID. - * @return ::NC_EFILTER Filter error. - * @author Dennis Heimbigner - */ -int -nc_def_var_filter(int ncid, int varid, unsigned int id, - size_t nparams, const unsigned int* parms) -{ - NC* ncp; - int stat = NC_check_id(ncid,&ncp); - if(stat != NC_NOERR) return stat; - return ncp->dispatch->def_var_filter(ncid,varid,id,nparams,parms); -} - /** * Set szip compression settings on a variable. Szip is an * implementation of the extended-Rice lossless compression algorithm; diff --git a/libdispatch/dvarinq.c b/libdispatch/dvarinq.c index 6685d07228..ce4787ff97 100644 --- a/libdispatch/dvarinq.c +++ b/libdispatch/dvarinq.c @@ -6,16 +6,14 @@ Functions for inquiring about variables. */ #include "config.h" +#include "netcdf.h" +#include "netcdf_filter.h" #include "ncdispatch.h" +#include "nc4internal.h" #ifdef USE_HDF5 #include #endif /* USE_HDF5 */ -#ifndef H5Z_FILTER_SZIP -/** ID of HDF SZIP filter. */ -#define H5Z_FILTER_SZIP 4 -#endif - /** \name Learning about Variables Functions to learn about the variables in a file. */ @@ -284,13 +282,34 @@ variable, the deflate_level will be written here. \ref ignored_if_null. \returns ::NC_ENOTVAR Invalid variable ID. */ int -nc_inq_var_deflate(int ncid, int varid, int *shufflep, int *deflatep, - int *deflate_levelp) +nc_inq_var_deflate(int ncid, int varid, int *shufflep, int *deflatep, int *deflate_levelp) { NC* ncp; + size_t nparams; + unsigned int params[4]; + int deflating = 0; + int stat = NC_check_id(ncid,&ncp); if(stat != NC_NOERR) return stat; TRACE(nc_inq_var_deflate); + + /* Verify id and nparams */ + stat = nc_inq_var_filter_info(ncid,varid,H5Z_FILTER_DEFLATE,&nparams,params); + switch (stat) { + case NC_ENOFILTER: deflating = 0; stat = NC_NOERR; break; + case NC_NOERR: deflating = 1; break; + default: return stat; + } + if(deflatep) *deflatep = deflating; + if(deflating) { + if(nparams != 1) + return NC_EFILTER; /* bad # params */ + /* Param[0] should be level */ + if(deflate_levelp) *deflate_levelp = (int)params[0]; + } + /* also get the shuffle state */ + if(!shufflep) + return NC_NOERR; return ncp->dispatch->inq_var_all( ncid, varid, NULL, /*name*/ @@ -299,15 +318,15 @@ nc_inq_var_deflate(int ncid, int varid, int *shufflep, int *deflatep, NULL, /*dimidsp*/ NULL, /*nattsp*/ shufflep, /*shufflep*/ - deflatep, /*deflatep*/ - deflate_levelp, /*deflatelevelp*/ + NULL, /*deflatep*/ + NULL, /*deflatelevelp*/ NULL, /*fletcher32p*/ NULL, /*contiguousp*/ NULL, /*chunksizep*/ NULL, /*nofillp*/ NULL, /*fillvaluep*/ NULL, /*endianp*/ - NULL,NULL,NULL + NULL, NULL, NULL ); } @@ -459,10 +478,12 @@ nc_inq_var_fill(int ncid, int varid, int *no_fill, void *fill_valuep) { NC* ncp; int stat = NC_check_id(ncid,&ncp); + if(stat != NC_NOERR) return stat; TRACE(nc_inq_var_fill); + return ncp->dispatch->inq_var_all( - ncid, varid, + ncid,varid, NULL, /*name*/ NULL, /*xtypep*/ NULL, /*ndimsp*/ @@ -607,66 +628,14 @@ nc_inq_unlimdims(int ncid, int *nunlimdimsp, int *unlimdimidsp) #endif } -/** -Find the filter (if any) associated with a variable. - -This is a wrapper for nc_inq_var_all(). - -\param ncid NetCDF or group ID, from a previous call to nc_open(), -nc_create(), nc_def_grp(), or associated inquiry functions such as -nc_inq_ncid(). - -\param varid Variable ID - -\param idp Storage which will get the filter id; a return value of zero means no filter - -\param nparamsp Storage which will get the number of parameters to the -filter - -\param params Storage which will get associated parameters. -Note: the caller must allocate and free. - -\returns ::NC_NOERR No error. -\returns ::NC_ENOTNC4 Not a netCDF-4 file. -\returns ::NC_EBADID Bad ncid. -\returns ::NC_ENOTVAR Invalid variable ID. -\returns ::NC_EFILTER No filter defined. -\ingroup variables -\author Dennis Heimbigner -*/ -int -nc_inq_var_filter(int ncid, int varid, unsigned int* idp, size_t* nparamsp, unsigned int* params) -{ - NC* ncp; - int stat = NC_check_id(ncid,&ncp); - if(stat != NC_NOERR) return stat; - TRACE(nc_inq_var_filter); - return ncp->dispatch->inq_var_all( - ncid, varid, - NULL, /*name*/ - NULL, /*xtypep*/ - NULL, /*ndimsp*/ - NULL, /*dimidsp*/ - NULL, /*nattsp*/ - NULL, /*shufflep*/ - NULL, /*deflatep*/ - NULL, /*deflatelevelp*/ - NULL, /*fletcher32p*/ - NULL, /*contiguousp*/ - NULL, /*chunksizep*/ - NULL, /*nofillp*/ - NULL, /*fillvaluep*/ - NULL, /*endianp*/ - idp, nparamsp, params); -} - /** \ingroup variables Learn the szip settings of a variable. This function returns the szip settings for a variable. To turn on szip compression, use nc_def_var_szip(). Szip compression is only -available if HDF5 was built with szip support. +available if HDF5 was built with szip support. The nc_def_var_filter +function may also be used to set szip compression. If a variable is not using szip, then a zero will be passed back for both options_maskp and pixels_per_blockp. @@ -683,7 +652,7 @@ nc_inq_ncid(). \param options_maskp The szip options mask will be copied to this pointer. Note that the HDF5 layer adds to the options_mask, so this -value will be different from the value used when setting szip +value may be different from the value used when setting szip compression, however the bit set when setting szip compression will still be set in the options_maskp and can be checked for. If zero is returned, szip is not in use for this variable.\ref ignored_if_null. @@ -700,12 +669,12 @@ szip is not in use for this variable. \ref ignored_if_null. \returns ::NC_EFILTER Filter error. \author Ed Hartnett, Dennis Heimbigner + */ int nc_inq_var_szip(int ncid, int varid, int *options_maskp, int *pixels_per_blockp) { NC* ncp; - unsigned int id; size_t nparams; unsigned int params[4]; @@ -713,75 +682,28 @@ nc_inq_var_szip(int ncid, int varid, int *options_maskp, int *pixels_per_blockp) if(stat != NC_NOERR) return stat; TRACE(nc_inq_var_szip); - /* Verify id and nparams */ - stat = ncp->dispatch->inq_var_all( - ncid, varid, - NULL, /*name*/ - NULL, /*xtypep*/ - NULL, /*ndimsp*/ - NULL, /*dimidsp*/ - NULL, /*nattsp*/ - NULL, /*shufflep*/ - NULL, /*deflatep*/ - NULL, /*deflatelevelp*/ - NULL, /*fletcher32p*/ - NULL, /*contiguousp*/ - NULL, /*chunksizep*/ - NULL, /*nofillp*/ - NULL, /*fillvaluep*/ - NULL, /*endianp*/ - &id, - &nparams, - NULL - ); - if(stat != NC_NOERR) return stat; - - /* Warning: the szip filter internally expands the set of - * parameters. This means the user submits 2, but once the filter - * is applied to the dataset, there are 4. How may we get back from - * inq_var_all() depends on whether or not the dataset has been - * created. */ - if(id == H5Z_FILTER_SZIP && (nparams != 4 && nparams !=2)) - return NC_EFILTER; /* not szip or bad # params */ - - /* If the szip filter is not in use, return 0 for both - * parameters. */ - if(id != H5Z_FILTER_SZIP) - { - if (options_maskp) - *options_maskp = 0; - if (pixels_per_blockp) - *pixels_per_blockp = 0; - return NC_NOERR; + /* Verify id and nparams */ + stat = nc_inq_var_filter_info(ncid,varid,H5Z_FILTER_SZIP,&nparams,params); + switch (stat) { + case NC_NOERR: + if(nparams != 2) + return NC_EFILTER; /* bad # params */ + break; + case NC_ENOFILTER: + /* If the szip filter is not in use, return 0 for both parameters. */ + params[0] = 0; + params[1] = 0; + stat = NC_NOERR; + break; + default: + return stat; } - /* Get params */ - stat = ncp->dispatch->inq_var_all( - ncid, varid, - NULL, /*name*/ - NULL, /*xtypep*/ - NULL, /*ndimsp*/ - NULL, /*dimidsp*/ - NULL, /*nattsp*/ - NULL, /*shufflep*/ - NULL, /*deflatep*/ - NULL, /*deflatelevelp*/ - NULL, /*fletcher32p*/ - NULL, /*contiguousp*/ - NULL, /*chunksizep*/ - NULL, /*nofillp*/ - NULL, /*fillvaluep*/ - NULL, /*endianp*/ - &id, - &nparams, - params - ); - if(stat != NC_NOERR) return stat; - /* Param[0] should be options_mask with possibly some other flags set, + /* Param[0] should be options_mask Param[1] should be pixels_per_block */ if(options_maskp) *options_maskp = (int)params[0]; if(pixels_per_blockp) *pixels_per_blockp = (int)params[1]; - return NC_NOERR; + return stat; } /*! @@ -832,13 +754,14 @@ to store filter parameters. \internal \ingroup variables */ +#if 0 int NC_inq_var_all(int ncid, int varid, char *name, nc_type *xtypep, int *ndimsp, int *dimidsp, int *nattsp, int *shufflep, int *deflatep, int *deflate_levelp, int *fletcher32p, int *contiguousp, size_t *chunksizesp, int *no_fill, void *fill_valuep, int *endiannessp, - unsigned int* idp, size_t* nparamsp, unsigned int* params + unsigned int* unused1, size_t* unused2, unsigned int* unused3 ) { NC* ncp; @@ -851,7 +774,8 @@ NC_inq_var_all(int ncid, int varid, char *name, nc_type *xtypep, contiguousp, chunksizesp, no_fill, fill_valuep, endiannessp, - idp,nparamsp,params); + NULL, NULL, NULL); } +#endif /*! \} */ /* End of named group ...*/ diff --git a/libhdf4/hdf4dispatch.c b/libhdf4/hdf4dispatch.c index d2b5437cd8..b93b1bc9ae 100644 --- a/libhdf4/hdf4dispatch.c +++ b/libhdf4/hdf4dispatch.c @@ -100,7 +100,9 @@ static const NC_Dispatch HDF4_dispatcher = { NC_NOTNC4_def_var_endian, NC_NOTNC4_def_var_filter, NC_NOTNC4_set_var_chunk_cache, - NC_NOTNC4_get_var_chunk_cache + NC_NOTNC4_get_var_chunk_cache, + + NC_NOTNC4_filter_actions, }; const NC_Dispatch *HDF4_dispatch_table = NULL; diff --git a/libhdf5/CMakeLists.txt b/libhdf5/CMakeLists.txt index f3c7bbc35f..d23ff8e2b2 100644 --- a/libhdf5/CMakeLists.txt +++ b/libhdf5/CMakeLists.txt @@ -8,7 +8,8 @@ # The source files for the HDF5 dispatch layer. SET(libnchdf5_SOURCES nc4hdf.c nc4info.c hdf5file.c hdf5attr.c hdf5dim.c hdf5grp.c hdf5type.c hdf5internal.c hdf5create.c hdf5open.c -hdf5var.c nc4mem.c nc4memcb.c hdf5cache.c hdf5dispatch.c) +hdf5var.c nc4mem.c nc4memcb.c hdf5cache.c hdf5dispatch.c hdf5filter.c +hdf5debug.c) IF(ENABLE_BYTERANGE) SET(libnchdf5_SOURCES ${libnchdf5_SOURCES} H5FDhttp.c) diff --git a/libhdf5/Makefile.am b/libhdf5/Makefile.am index 88433b0172..b04938c2d7 100644 --- a/libhdf5/Makefile.am +++ b/libhdf5/Makefile.am @@ -15,7 +15,8 @@ noinst_LTLIBRARIES = libnchdf5.la # The source files. libnchdf5_la_SOURCES = nc4hdf.c nc4info.c hdf5file.c hdf5attr.c \ hdf5dim.c hdf5grp.c hdf5type.c hdf5internal.c hdf5create.c hdf5open.c \ -hdf5var.c nc4mem.c nc4memcb.c hdf5cache.c hdf5dispatch.c +hdf5var.c nc4mem.c nc4memcb.c hdf5cache.c hdf5dispatch.c hdf5filter.c \ +hdf5debug.c hdf5debug.h if ENABLE_BYTERANGE libnchdf5_la_SOURCES += H5FDhttp.c H5FDhttp.h diff --git a/libhdf5/hdf5debug.c b/libhdf5/hdf5debug.c new file mode 100644 index 0000000000..88bd69faea --- /dev/null +++ b/libhdf5/hdf5debug.c @@ -0,0 +1,47 @@ +/********************************************************************* + * Copyright 2018, UCAR/Unidata + * See netcdf/COPYRIGHT file for copying and redistribution conditions. + *********************************************************************/ +#include "config.h" +#include +#include +#if !defined _WIN32 && !defined __CYGWIN__ +#include +#endif + +#include "hdf5debug.h" + +#ifdef H5CATCH + +#define STSIZE 1000 + +static void* stacktrace[STSIZE]; + +int +nch5breakpoint(int err) +{ +#if !defined _WIN32 && !defined __CYGWIN__ + int count = 0; + char** trace = NULL; + int i; + + count = backtrace(stacktrace,STSIZE); + trace = backtrace_symbols(stacktrace, STSIZE); + fprintf(stderr,"backtrace:\n"); + for(i=0;i +#include "hdf5internal.h" +#include "hdf5debug.h" + +#define HAVE_H5_DEFLATE + +/* Mnemonic */ +#define FILTERACTIVE 1 + + +/* WARNING: GLOBAL VARIABLE */ + +/* Define list of registered filters */ +static NClist* NC4_registeredfilters = NULL; /** List */ + +/**************************************************/ +/* Filter registration support */ + +static int +filterlookup(unsigned int id) +{ + int i; + if(NC4_registeredfilters == NULL) + NC4_registeredfilters = nclistnew(); + for(i=0;iid == id) return i; /* return position */ + } + return -1; +} + +static void +reclaiminfo(NC_FILTER_CLIENT_HDF5* info) +{ + nullfree(info); +} + +static int +filterremove(int pos) +{ + NC_FILTER_CLIENT_HDF5* info = NULL; + if(NC4_registeredfilters == NULL) + return THROW(NC_EINVAL); + if(pos < 0 || pos >= nclistlength(NC4_registeredfilters)) + return THROW(NC_EINVAL); + info = nclistget(NC4_registeredfilters,pos); + reclaiminfo(info); + nclistremove(NC4_registeredfilters,pos); + return NC_NOERR; +} + +static NC_FILTER_CLIENT_HDF5* +dupfilterinfo(NC_FILTER_CLIENT_HDF5* info) +{ + NC_FILTER_CLIENT_HDF5* dup = NULL; + if(info == NULL) goto fail; + if((dup = calloc(1,sizeof(NC_FILTER_CLIENT_HDF5))) == NULL) goto fail; + *dup = *info; + return dup; +fail: + reclaiminfo(dup); + return NULL; +} + +int +NC4_hdf5_addfilter(NC_VAR_INFO_T* var, int active, unsigned int id, size_t nparams, unsigned int* inparams) +{ + int stat = NC_NOERR; + NC_FILTER_SPEC_HDF5* fi = NULL; + unsigned int* params = NULL; + + if(var->filters == NULL) { + if((var->filters = nclistnew())==NULL) return THROW(NC_ENOMEM); + } + + if(nparams > 0 && inparams == NULL) + return THROW(NC_EINVAL); + if(inparams != NULL) { + if((params = malloc(sizeof(unsigned int)*nparams)) == NULL) + return THROW(NC_ENOMEM); + memcpy(params,inparams,sizeof(unsigned int)*nparams); + } + + if((fi = calloc(1,sizeof(NC_FILTER_SPEC_HDF5))) == NULL) + {nullfree(params); return THROW(NC_ENOMEM);} + + fi->active = active; + fi->filterid = id; + fi->nparams = nparams; + fi->params = params; + nclistpush(var->filters,fi); + return THROW(stat); +} + +int +nc4_global_filter_action(int op, unsigned int id, NC_FILTER_OBJ_HDF5* infop) +{ + int stat = NC_NOERR; + H5Z_class2_t* h5filterinfo = NULL; + herr_t herr; + int pos = -1; + NC_FILTER_CLIENT_HDF5* dup = NULL; + NC_FILTER_CLIENT_HDF5* elem = NULL; + NC_FILTER_CLIENT_HDF5 ncf; + + switch (op) { + case NCFILTER_CLIENT_REG: /* Ignore id argument */ + if(infop == NULL) {stat = NC_EINVAL; goto done;} + assert(NC_FILTER_FORMAT_HDF5 == infop->hdr.format); + assert(NC_FILTER_SORT_CLIENT == infop->sort); + elem = (NC_FILTER_CLIENT_HDF5*)&infop->u.client; + h5filterinfo = elem->info; + /* Another sanity check */ + if(id != h5filterinfo->id) + {stat = NC_EINVAL; goto done;} + /* See if this filter is already defined */ + if((pos = filterlookup(id)) >= 0) + {stat = NC_ENAMEINUSE; goto done;} /* Already defined */ + if((herr = H5Zregister(h5filterinfo)) < 0) + {stat = NC_EFILTER; goto done;} + /* Save a copy of the passed in info */ + ncf.id = id; + ncf.info = elem->info; + if((dup=dupfilterinfo(&ncf)) == NULL) + {stat = NC_ENOMEM; goto done;} + nclistpush(NC4_registeredfilters,dup); + break; + case NCFILTER_CLIENT_UNREG: + if(id <= 0) + {stat = NC_ENOTNC4; goto done;} + /* See if this filter is already defined */ + if((pos = filterlookup(id)) < 0) + {stat = NC_ENOFILTER; goto done;} /* Not defined */ + if((herr = H5Zunregister(id)) < 0) + {stat = NC_EFILTER; goto done;} + if((stat=filterremove(pos))) goto done; + break; + case NCFILTER_CLIENT_INQ: + if(infop == NULL) goto done; + /* Look up the id in our local table */ + if((pos = filterlookup(id)) < 0) + {stat = NC_ENOFILTER; goto done;} /* Not defined */ + elem = (NC_FILTER_CLIENT_HDF5*)nclistget(NC4_registeredfilters,pos); + if(elem == NULL) {stat = NC_EINTERNAL; goto done;} + if(infop != NULL) { + infop->u.client = *elem; + } + break; + default: + {stat = NC_EINTERNAL; goto done;} + } +done: + return THROW(stat); +} + +/** + * @internal Define filter settings. Called by nc_def_var_filter(). + * + * @param ncid File ID. + * @param varid Variable ID. + * @param id Filter ID + * @param nparams Number of parameters for filter. + * @param parms Filter parameters. + * + * @returns ::NC_NOERR for success + * @returns ::NC_EBADID Bad ncid. + * @returns ::NC_ENOTVAR Invalid variable ID. + * @returns ::NC_ENOTNC4 Attempting netcdf-4 operation on file that is + * not netCDF-4/HDF5. + * @returns ::NC_ELATEDEF Too late to change settings for this variable. + * @returns ::NC_EFILTER Filter error. + * @returns ::NC_EINVAL Invalid input + * @author Dennis Heimbigner + */ +int +NC4_filter_actions(int ncid, int varid, int op, NC_Filterobject* args) +{ + int stat = NC_NOERR; + NC_GRP_INFO_T *grp = NULL; + NC_FILE_INFO_T *h5 = NULL; + NC_VAR_INFO_T *var = NULL; + NC_FILTER_OBJ_HDF5* obj = (NC_FILTER_OBJ_HDF5*)args; + unsigned int id = 0; + size_t nparams = 0; + unsigned int* idp = NULL; + size_t* nparamsp = NULL; + size_t* nfiltersp = NULL; + unsigned int* params = NULL; + size_t nfilters = 0; + unsigned int* filterids = NULL; + + LOG((2, "%s: ncid 0x%x varid %d op=%d", __func__, ncid, varid, op)); + + if(args == NULL) return THROW(NC_EINVAL); + if(args->format != NC_FILTER_FORMAT_HDF5) return THROW(NC_EFILTER); + + /* Find info for this file and group and var, and set pointer to each. */ + if ((stat = nc4_hdf5_find_grp_h5_var(ncid, varid, &h5, &grp, &var))) + return THROW(stat); + + assert(h5 && var && var->hdr.id == varid); + + nfilters = nclistlength(var->filters); + + switch (op) { + case NCFILTER_DEF: { + if(obj->sort != NC_FILTER_SORT_SPEC) return THROW(NC_EFILTER); + /* If the HDF5 dataset has already been created, then it is too + * late to set all the extra stuff. */ + if (!(h5->flags & NC_INDEF)) return THROW(NC_EINDEFINE); + if (!var->ndims) return NC_NOERR; /* For scalars, ignore */ + if (var->created) + return THROW(NC_ELATEDEF); + /* Can't turn on parallel and szip before HDF5 1.10.2. */ +#ifdef USE_PARALLEL +#ifndef HDF5_SUPPORTS_PAR_FILTERS + if (h5->parallel == NC_TRUE) + return THROW(NC_EINVAL); +#endif /* HDF5_SUPPORTS_PAR_FILTERS */ +#endif /* USE_PARALLEL */ + id = obj->u.spec.filterid; + nparams = obj->u.spec.nparams; + params = obj->u.spec.params; +#ifdef HAVE_H5_DEFLATE + if(id == H5Z_FILTER_DEFLATE) { + int k; + int level; + if(nparams != 1) + return THROW(NC_EFILTER); /* incorrect no. of parameters */ + level = (int)params[0]; + if (level < NC_MIN_DEFLATE_LEVEL || + level > NC_MAX_DEFLATE_LEVEL) + return THROW(NC_EINVAL); + /* If szip compression is already applied, return error. */ + for(k=0;kfilters);k++) { + NC_FILTER_SPEC_HDF5* f = nclistget(var->filters,k); + if (f->filterid == H5Z_FILTER_SZIP) + return THROW(NC_EINVAL); + } + } +#else /*!HAVE_H5_DEFLATE*/ + if(id == H5Z_FILTER_DEFLATE) + return THROW(NC_EFILTER); /* Not allowed */ +#endif +#ifdef HAVE_H5Z_SZIP + if(id == H5Z_FILTER_SZIP) { /* Do error checking */ + int k; + if(nparams != 2) + return THROW(NC_EFILTER); /* incorrect no. of parameters */ + /* Pixels per block must be an even number, < 32. */ + if (params[1] % 2 || params[1] > NC_MAX_PIXELS_PER_BLOCK) + return THROW(NC_EINVAL); + /* If zlib compression is already applied, return error. */ + for(k=0;kfilters);k++) { + NC_FILTER_SPEC_HDF5* f = nclistget(var->filters,k); + if (f->filterid == H5Z_FILTER_DEFLATE) + return THROW(NC_EINVAL); + } + } +#else /*!HAVE_H5Z_SZIP*/ + if(id == H5Z_FILTER_SZIP) + return THROW(NC_EFILTER); /* Not allowed */ +#endif + /* Filter => chunking */ + var->contiguous = NC_FALSE; + /* Determine default chunksizes for this variable unless already specified */ + if(var->chunksizes && !var->chunksizes[0]) { + /* Should this throw error? */ + if((stat = nc4_find_default_chunksizes2(grp, var))) + goto done; + /* Adjust the cache. */ + if ((stat = nc4_adjust_var_cache(grp, var))) + goto done; + } +#ifdef HAVE_H5Z_SZIP + if(id == H5Z_FILTER_SZIP) { /* szip X chunking error checking */ + /* For szip, the pixels_per_block parameter must not be greater + * than the number of elements in a chunk of data. */ + size_t num_elem = 1; + int d; + for (d = 0; d < var->ndims; d++) + num_elem *= var->dim[d]->len; + /* Pixels per block must be <= number of elements. */ + if (params[1] > num_elem) + return THROW(NC_EINVAL); + } +#endif + if((stat = NC4_hdf5_addfilter(var,!FILTERACTIVE,id,nparams,params))) + goto done; +#ifdef USE_PARALLEL +#ifndef HDF5_SUPPORTS_PAR_FILTERS +/* Switch to collective access. HDF5 requires collevtive access + * for filter use with parallel I/O. */ + if (h5->parallel) + var->parallel_access = NC_COLLECTIVE; +#endif /* HDF5_SUPPORTS_PAR_FILTERS */ +#endif /* USE_PARALLEL */ + } break; + case NCFILTER_INQ: { + if (!var->ndims) return THROW(NC_ENOFILTER); /* For scalars, fail */ + if(obj->sort != NC_FILTER_SORT_SPEC) return THROW(NC_EFILTER); + idp = &obj->u.spec.filterid; + nparamsp = &obj->u.spec.nparams; + params = obj->u.spec.params; + if(nfilters > 0) { + /* Return info about var->filters[0] */ + NC_FILTER_SPEC_HDF5* f = (NC_FILTER_SPEC_HDF5*)nclistget(var->filters,0); + if(idp) *idp = f->filterid; + if(nparamsp) { + *nparamsp = (f->params == NULL ? 0 : f->nparams); + if(params && f->params != NULL && f->nparams > 0) + memcpy(params,f->params,f->nparams*sizeof(unsigned int)); + } + } else {stat = NC_ENOFILTER; goto done;} + } break; + case NCFILTER_FILTERIDS: { + if(obj->sort != NC_FILTER_SORT_IDS) return THROW(NC_EFILTER); + nfiltersp = &obj->u.ids.nfilters; + filterids = obj->u.ids.filterids; + if(nfiltersp) *nfiltersp = nfilters; + if(filterids) filterids[0] = 0; + if(nfilters > 0 && filterids != NULL) { + int k; + for(k=0;kfilters,k); + filterids[k] = f->filterid; + } + } + } break; + case NCFILTER_INFO: { + int k,found; + if(obj->sort != NC_FILTER_SORT_SPEC) return THROW(NC_EFILTER); + id = obj->u.spec.filterid; + for(found=0,k=0;kfilters,k); + if(f->filterid == id) { + obj->u.spec.nparams = f->nparams; + if(obj->u.spec.params != NULL && f->params != NULL && f->nparams > 0) + memcpy(obj->u.spec.params,f->params,f->nparams*sizeof(unsigned int)); + found = 1; + break; + } + } + if(!found) {stat = NC_ENOFILTER; goto done;} + } break; + case NCFILTER_REMOVE: { + int k; + if (!(h5->flags & NC_INDEF)) return THROW(NC_EINDEFINE); + if(obj->sort != NC_FILTER_SORT_SPEC) return THROW(NC_EFILTER); + id = obj->u.spec.filterid; + /* Walk backwards */ + for(k=nfilters-1;k>=0;k--) { + NC_FILTER_SPEC_HDF5* f = (NC_FILTER_SPEC_HDF5*)nclistget(var->filters,k); + if(f->filterid == id) { + if(f->active) { + /* Remove from variable */ + if((stat = NC4_hdf5_remove_filter(var,id))) {stat = NC_ENOFILTER; goto done;} + } + nclistremove(var->filters,k); + NC4_freefilterspec(f); + } + } + } break; + default: + {stat = NC_EINTERNAL; goto done;} + } + +done: + return THROW(stat); +} + +void +NC4_freefilterspec(NC_FILTER_SPEC_HDF5* f) +{ + if(f) { + if(f->params != NULL) {free(f->params);} + free(f); + } +} diff --git a/libhdf5/hdf5open.c b/libhdf5/hdf5open.c index 52f8b6d5f7..73777c855b 100644 --- a/libhdf5/hdf5open.c +++ b/libhdf5/hdf5open.c @@ -18,6 +18,9 @@ #include "H5FDhttp.h" #endif +/*Nemonic */ +#define FILTERACTIVE 1 + #define NUM_TYPES 12 /**< Number of netCDF atomic types. */ #define CD_NELEMS_ZLIB 1 /**< Number of parameters needed for ZLIB filter. */ @@ -915,8 +918,8 @@ NC4_open(const char *path, int mode, int basepe, size_t *chunksizehintp, /** * @internal Find out what filters are applied to this HDF5 dataset, - * fletcher32, deflate, and/or shuffle. All other filters are just - * dumped The possible values of + * fletcher32, deflate, and/or shuffle. All other filters are + * captured. * * @param propid ID of HDF5 var creation properties list. * @param var Pointer to NC_VAR_INFO_T for this variable. @@ -933,6 +936,7 @@ static int get_filter_info(hid_t propid, NC_VAR_INFO_T *var) unsigned int cd_values_zip[CD_NELEMS_ZLIB]; size_t cd_nelems = CD_NELEMS_ZLIB; int f; + int stat = NC_NOERR; assert(var); @@ -955,46 +959,56 @@ static int get_filter_info(hid_t propid, NC_VAR_INFO_T *var) break; case H5Z_FILTER_DEFLATE: - var->deflate = NC_TRUE; if (cd_nelems != CD_NELEMS_ZLIB || cd_values_zip[0] > NC_MAX_DEFLATE_LEVEL) return NC_EHDFERR; - var->deflate_level = cd_values_zip[0]; + if((stat = NC4_hdf5_addfilter(var,FILTERACTIVE,filter,cd_nelems,cd_values_zip))) + return stat; break; - case H5Z_FILTER_SZIP: + case H5Z_FILTER_SZIP: { /* Szip is tricky because the filter code expands the set of parameters from 2 to 4 - and changes some of the parameter values */ - var->filterid = filter; - if(cd_nelems == 0) - var->params = NULL; - else { + and changes some of the parameter values; try to compensate */ + if(cd_nelems == 0) { + if((stat = NC4_hdf5_addfilter(var,FILTERACTIVE,filter,0,NULL))) + return stat; + } else { /* We have to re-read the parameters based on actual nparams, which in the case of szip, differs from users original nparams */ - var->params = (unsigned int*)calloc(1,sizeof(unsigned int)*cd_nelems); - if(var->params == NULL) + unsigned int* realparams = (unsigned int*)calloc(1,sizeof(unsigned int)*cd_nelems); + if(realparams == NULL) return NC_ENOMEM; if((filter = H5Pget_filter2(propid, f, NULL, &cd_nelems, - var->params, 0, NULL, NULL)) < 0) + realparams, 0, NULL, NULL)) < 0) return NC_EHDFERR; /* fix up the parameters and the #params */ - var->nparams = cd_nelems; + if(cd_nelems != 4) + return NC_EHDFERR; + cd_nelems = 2; /* ignore last two */ + /* Fix up changed params */ + realparams[0] &= (H5_SZIP_ALL_MASKS); + /* Save info */ + stat = NC4_hdf5_addfilter(var,FILTERACTIVE,filter,cd_nelems,realparams); + nullfree(realparams); + if(stat) return stat; + } - break; + } break; default: - var->filterid = filter; - var->nparams = cd_nelems; - if(cd_nelems == 0) - var->params = NULL; - else { + if(cd_nelems == 0) { + if((stat = NC4_hdf5_addfilter(var,FILTERACTIVE,filter,0,NULL))) return stat; + } else { /* We have to re-read the parameters based on actual nparams */ - var->params = (unsigned int*)calloc(1,sizeof(unsigned int)*var->nparams); - if(var->params == NULL) + unsigned int* realparams = (unsigned int*)calloc(1,sizeof(unsigned int)*cd_nelems); + if(realparams == NULL) return NC_ENOMEM; if((filter = H5Pget_filter2(propid, f, NULL, &cd_nelems, - var->params, 0, NULL, NULL)) < 0) + realparams, 0, NULL, NULL)) < 0) return NC_EHDFERR; + stat = NC4_hdf5_addfilter(var,FILTERACTIVE,filter,cd_nelems,realparams); + nullfree(realparams); + if(stat) return stat; } break; } diff --git a/libhdf5/hdf5type.c b/libhdf5/hdf5type.c index 37a3dfd7a0..8c15e15b70 100644 --- a/libhdf5/hdf5type.c +++ b/libhdf5/hdf5type.c @@ -108,7 +108,7 @@ NC4_inq_type_equal(int ncid1, nc_type typeid1, int ncid2, * @return ::NC_EBADTYPE Type not found. * @author Ed Hartnett */ -extern int +EXTERNL int NC4_inq_typeid(int ncid, const char *name, nc_type *typeidp) { NC_GRP_INFO_T *grp; diff --git a/libhdf5/hdf5var.c b/libhdf5/hdf5var.c index c015176c26..4c88dc69ba 100644 --- a/libhdf5/hdf5var.c +++ b/libhdf5/hdf5var.c @@ -1,4 +1,3 @@ - /* Copyright 2003-2019, University Corporation for Atmospheric * Research. See COPYRIGHT file for copying and redistribution * conditions.*/ @@ -15,6 +14,9 @@ #endif #include /* For pow() used below. */ +#include "netcdf.h" +#include "netcdf_filter.h" + /** @internal Default size for unlimited dim chunksize. */ #define DEFAULT_1D_UNLIM_SIZE (4096) @@ -25,13 +27,6 @@ /** Number of bytes in 64 KB. */ #define SIXTY_FOUR_KB (65536) -/** Number of parameters needed when turning on szip filter. */ -#define NUM_SZIP_PARAM 2 - -/** The maximum allowed setting for pixels_per_block when calling - * nc_def_var_szip(). */ -#define NC_MAX_PIXELS_PER_BLOCK 32 - #ifdef LOGGING /** * Report the chunksizes selected for a variable. @@ -155,7 +150,7 @@ check_chunksizes(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var, const size_t *chunksize * @returns ::NC_ENOTVAR Invalid variable ID. * @author Ed Hartnett, Dennis Heimbigner */ -static int +int nc4_find_default_chunksizes2(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var) { int d; @@ -637,22 +632,17 @@ NC4_def_var(int ncid, const char *name, nc_type xtype, int ndims, * @author Ed Hartnett */ static int -nc_def_var_extra(int ncid, int varid, int *shuffle, int *deflate, - int *deflate_level, int *fletcher32, int *contiguous, +nc_def_var_extra(int ncid, int varid, int *shuffle, int *unused1, + int *unused2, int *fletcher32, int *contiguous, const size_t *chunksizes, int *no_fill, const void *fill_value, int *endianness) { NC_GRP_INFO_T *grp; NC_FILE_INFO_T *h5; NC_VAR_INFO_T *var; - int option_mask; int d; int retval; - /* All or none of these will be provided. */ - assert((deflate && deflate_level && shuffle) || - (!deflate && !deflate_level && !shuffle)); - LOG((2, "%s: ncid 0x%x varid %d", __func__, ncid, varid)); /* Find info for this file and group, and set pointer to each. */ @@ -669,12 +659,11 @@ nc_def_var_extra(int ncid, int varid, int *shuffle, int *deflate, return NC_ENOTVAR; assert(var && var->hdr.id == varid); - /* Can't turn on parallel and deflate/fletcher32/szip/shuffle * before HDF5 1.10.3. */ #ifndef HDF5_SUPPORTS_PAR_FILTERS if (h5->parallel == NC_TRUE) - if (deflate || fletcher32 || shuffle) + if (nclistlength(var->filters) > 0 || fletcher32 || shuffle) return NC_EINVAL; #endif @@ -683,6 +672,7 @@ nc_def_var_extra(int ncid, int varid, int *shuffle, int *deflate, if (var->created) return NC_ELATEDEF; +#if 0 /* Check compression options. */ if (deflate && !deflate_level) return NC_EINVAL; @@ -712,6 +702,7 @@ nc_def_var_extra(int ncid, int varid, int *shuffle, int *deflate, var->deflate_level = *deflate_level; LOG((3, "%s: *deflate_level %d", __func__, *deflate_level)); } +#endif /* Shuffle filter? */ if (shuffle) @@ -732,9 +723,9 @@ nc_def_var_extra(int ncid, int varid, int *shuffle, int *deflate, * parallel I/O writes, then switch to collective access. HDF5 * requires collevtive access for filter use with parallel * I/O. */ - if (deflate || shuffle || fletcher32) + if (shuffle || fletcher32) { - if (h5->parallel && (var->deflate || var->shuffle || var->fletcher32)) + if (h5->parallel && (nclistlength(var->filters) > 0 || var->shuffle || var->fletcher32)) var->parallel_access = NC_COLLECTIVE; } #endif /* USE_PARALLEL */ @@ -747,7 +738,7 @@ nc_def_var_extra(int ncid, int varid, int *shuffle, int *deflate, * no filters in use for this data. */ if (*contiguous) { - if (var->deflate || var->fletcher32 || var->shuffle) + if (nclistlength(var->filters) > 0 || var->fletcher32 || var->shuffle) return NC_EINVAL; for (d = 0; d < var->ndims; d++) @@ -803,7 +794,7 @@ nc_def_var_extra(int ncid, int varid, int *shuffle, int *deflate, /* Is this a variable with a chunksize greater than the current * cache size? */ - if (!var->contiguous && (deflate || contiguous)) + if (!var->contiguous && (contiguous)) { /* Determine default chunksizes for this variable (do nothing * for scalar vars). */ @@ -911,8 +902,17 @@ int NC4_def_var_deflate(int ncid, int varid, int shuffle, int deflate, int deflate_level) { - return nc_def_var_extra(ncid, varid, &shuffle, &deflate, - &deflate_level, NULL, NULL, NULL, NULL, NULL, NULL); + int stat = NC_NOERR; + unsigned int level = (unsigned int)deflate_level; + /* Set shuffle first */ + if((stat = nc_def_var_extra(ncid, varid, &shuffle, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL))) goto done; + if(deflate) + stat = nc_def_var_filter(ncid, varid, H5Z_FILTER_DEFLATE,1,&level); + else + stat = nc_var_filter_remove(ncid, varid, H5Z_FILTER_DEFLATE); + if(stat) goto done; +done: + return stat; } /** @@ -1077,141 +1077,27 @@ NC4_def_var_endian(int ncid, int varid, int endianness) NULL, NULL, NULL, &endianness); } -/** - * @internal Define filter settings. Called by nc_def_var_filter(). - * - * @param ncid File ID. - * @param varid Variable ID. - * @param id Filter ID - * @param nparams Number of parameters for filter. - * @param parms Filter parameters. - * - * @returns ::NC_NOERR for success - * @returns ::NC_EBADID Bad ncid. - * @returns ::NC_ENOTVAR Invalid variable ID. - * @returns ::NC_ENOTNC4 Attempting netcdf-4 operation on file that is - * not netCDF-4/HDF5. - * @returns ::NC_ELATEDEF Too late to change settings for this variable. - * @returns ::NC_EFILTER Filter error. - * @returns ::NC_EINVAL Invalid input - * @author Dennis Heimbigner, Ed Hartnett - */ int NC4_def_var_filter(int ncid, int varid, unsigned int id, size_t nparams, - const unsigned int* parms) + const unsigned int* params) { int retval = NC_NOERR; NC *nc; - NC_GRP_INFO_T *grp; - NC_FILE_INFO_T *h5; - NC_VAR_INFO_T *var; + NC_FILTER_OBJ_HDF5 spec; LOG((2, "%s: ncid 0x%x varid %d", __func__, ncid, varid)); - /* Find info for this file and group, and set pointer to each. */ - if ((retval = nc4_find_nc_grp_h5(ncid, &nc, &grp, &h5))) - return retval; - - assert(nc && grp && h5); - - /* Find the var. */ - var = (NC_VAR_INFO_T*)ncindexith(grp->vars,varid); - if(!var) - return NC_ENOTVAR; - assert(var->hdr.id == varid); + if((retval = NC_check_id(ncid,&nc))) return retval; + assert(nc); - /* If the HDF5 dataset has already been created, then it is too - * late to set all the extra stuff. */ - if (var->created) - return NC_ELATEDEF; + memset(&spec,0,sizeof(spec)); + spec.hdr.format = NC_FILTER_FORMAT_HDF5; + spec.sort = NC_FILTER_SORT_SPEC; + spec.u.spec.filterid = id; + spec.u.spec.nparams = nparams; + spec.u.spec.params = (unsigned int*)params; /* Need to remove const */ - /* Can't turn on parallel and szip before HDF5 1.10.3. */ -#ifdef USE_PARALLEL -#ifndef HDF5_SUPPORTS_PAR_FILTERS - if (h5->parallel == NC_TRUE) - return NC_EINVAL; -#endif /* HDF5_SUPPORTS_PAR_FILTERS */ - - /* Switch to collective access. HDF5 requires collevtive access - * for filter use with parallel I/O. */ - if (h5->parallel) - var->parallel_access = NC_COLLECTIVE; -#endif /* USE_PARALLEL */ - -#ifdef HAVE_H5Z_SZIP - /* We have special handling for the szip filter. */ - if (id == H5Z_FILTER_SZIP) - { - if (nparams != 2) - return NC_EFILTER; /* incorrect no. of parameters */ - - /* If zlib compression is already applied, return error. */ - if (var->deflate) - return NC_EINVAL; - - /* Pixels per block must be an even number, < 32. */ - if (parms[1] % 2 || parms[1] > NC_MAX_PIXELS_PER_BLOCK) - return NC_EINVAL; - } -#else /*!HAVE_H5Z_SZIP*/ - if (id == H5Z_FILTER_SZIP) - return NC_EFILTER; /* HDF5 was not built with szip. */ -#endif /*!HAVE_H5Z_SZIP*/ - -#if 0 - { - unsigned int fcfg = 0; - herr_t herr = H5Zget_filter_info(id,&fcfg); - if(herr < 0) - return NC_EFILTER; - if((H5Z_FILTER_CONFIG_ENCODE_ENABLED & fcfg) == 0 - || (H5Z_FILTER_CONFIG_DECODE_ENABLED & fcfg) == 0) - return NC_EFILTER; - } -#endif /*0*/ - - /* Filters can only be applied to chunked datasets in HDF5. */ - var->contiguous = NC_FALSE; - - /* Determine default chunksizes for this variable unless already specified */ - if(var->chunksizes && !var->chunksizes[0]) { - if((retval = nc4_find_default_chunksizes2(grp, var))) - return retval; - /* Adjust the cache. */ - if ((retval = nc4_adjust_var_cache(grp, var))) - return retval; - } - -#ifdef HAVE_H5Z_SZIP - /* For szip, the pixels_per_block parameter must not be greater - * than the number of elements in a chunk of data. */ - if (id == H5Z_FILTER_SZIP) - { - size_t num_elem = 1; - int d; - - for (d = 0; d < var->ndims; d++) - num_elem *= var->dim[d]->len; - - /* Pixels per block must be <= number of elements. */ - if (parms[1] > num_elem) - return NC_EINVAL; - } -#endif /*!HAVE_H5Z_SZIP*/ - - /* Remember the filter info for later. It will be applied when the - * dataset for this variable is created. */ - var->filterid = id; - var->nparams = nparams; - var->params = NULL; - if(parms != NULL) { - var->params = (unsigned int*)calloc(nparams,sizeof(unsigned int)); - if(var->params == NULL) return NC_ENOMEM; - memcpy(var->params,parms,sizeof(unsigned int)*var->nparams); - } - - - return NC_NOERR; + return nc->dispatch->filter_actions(ncid,varid,NCFILTER_DEF,(NC_Filterobject*)&spec); } /** @@ -2315,10 +2201,10 @@ NC4_get_vars(int ncid, int varid, const size_t *startp, const size_t *countp, int NC4_HDF5_inq_var_all(int ncid, int varid, char *name, nc_type *xtypep, int *ndimsp, int *dimidsp, int *nattsp, - int *shufflep, int *deflatep, int *deflate_levelp, + int *shufflep, int *unused4, int *unused5, int *fletcher32p, int *contiguousp, size_t *chunksizesp, int *no_fill, void *fill_valuep, int *endiannessp, - unsigned int *idp, size_t *nparamsp, unsigned int *params) + unsigned int *unused1, size_t *unused2, unsigned int *unused3) { NC_FILE_INFO_T *h5; NC_GRP_INFO_T *grp; @@ -2337,9 +2223,9 @@ NC4_HDF5_inq_var_all(int ncid, int varid, char *name, nc_type *xtypep, /* Now that lazy atts have been read, use the libsrc4 function to * get the answers. */ return NC4_inq_var_all(ncid, varid, name, xtypep, ndimsp, dimidsp, nattsp, - shufflep, deflatep, deflate_levelp, fletcher32p, + shufflep, unused4, unused5, fletcher32p, contiguousp, chunksizesp, no_fill, fill_valuep, - endiannessp, idp, nparamsp, params); + endiannessp, unused1, unused2, unused3); } /** diff --git a/libhdf5/nc4hdf.c b/libhdf5/nc4hdf.c index 5042d5bf00..79b1654f47 100644 --- a/libhdf5/nc4hdf.c +++ b/libhdf5/nc4hdf.c @@ -15,6 +15,9 @@ */ #include "config.h" +#include "netcdf.h" +#include "nc4internal.h" +#include "ncdispatch.h" #include "hdf5internal.h" #include @@ -25,11 +28,6 @@ #define NC_HDF5_MAX_NAME 1024 /**< @internal Max size of HDF5 name. */ -/* WARNING: GLOBAL VARIABLE */ - -/* Define list of registered filters */ -static NClist* filters = NULL; - /** * @internal Flag attributes in a linked list as dirty. * @@ -908,34 +906,42 @@ var_create_dataset(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var, nc_bool_t write_dimid } /* If the user wants to compress the data, using either zlib - * (a.k.a deflate) or szip, set that up now. Szip can be turned on - * either directly with nc_def_var_szip(), or using - * nc_def_var_filter() with H5Z_FILTER_SZIP as the id. If the user + * (a.k.a deflate) or szip, or another filter, set that up now. + * Szip and zip can be turned on + * either directly with nc_def_var_szip/deflate(), or using + * nc_def_var_filter(). If the user * has specified a filter, it will be applied here. */ - if (var->deflate) - { - /* Turn on zlip compression. */ - if (H5Pset_deflate(plistid, var->deflate_level) < 0) - BAIL(NC_EHDFERR); - } - else if(var->filterid) - { - /* Since szip is a built-in filter for HDF5, it is activated - * with a special function. */ - if (var->filterid == H5Z_FILTER_SZIP) - { - if (var->nparams != 2) - BAIL(NC_EFILTER); - if (H5Pset_szip(plistid, (int)var->params[0], (int)var->params[1]) < 0) - BAIL(NC_EFILTER); - } - else - { - /* There is a filter that is not szip. Apply it. */ - if (H5Pset_filter(plistid, var->filterid, H5Z_FLAG_MANDATORY, var->nparams, - var->params) < 0) - BAIL(NC_EFILTER); - } + if(var->filters != NULL) { + int j; + for(j=0;jfilters);j++) { + NC_FILTER_SPEC_HDF5* fi = (NC_FILTER_SPEC_HDF5*)nclistget(var->filters,j); + size_t nparams; + unsigned int* params; + nparams = fi->nparams; + params = fi->params; + if(fi->filterid == H5Z_FILTER_DEFLATE) {/* Handle zip case here */ + unsigned level; + if(nparams != 1) + BAIL(NC_EFILTER); + level = (int)params[0]; + if(H5Pset_deflate(plistid, level) < 0) + BAIL(NC_EFILTER); + } else if(fi->filterid == H5Z_FILTER_SZIP) {/* Handle szip case here */ + int options_mask; + int bits_per_pixel; + if(nparams != 2) + BAIL(NC_EFILTER); + options_mask = (int)params[0]; + bits_per_pixel = (int)params[1]; + if(H5Pset_szip(plistid, options_mask, bits_per_pixel) < 0) + BAIL(NC_EFILTER); + } else { + herr_t code = H5Pset_filter(plistid, (unsigned int)fi->filterid, H5Z_FLAG_MANDATORY, nparams, params); + if(code < 0) { + BAIL(NC_EFILTER); + } + } + } } /* If the user wants to fletcher error correction, set that up now. */ @@ -962,7 +968,7 @@ var_create_dataset(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var, nc_bool_t write_dimid /* If there are no unlimited dims, and no filters, and the user * has not specified chunksizes, use contiguous variable for * better performance. */ - if (!var->shuffle && !var->deflate && !var->filterid && !var->fletcher32 && + if (!var->shuffle && !var->fletcher32 && nclistlength(var->filters) == 0 && (var->chunksizes == NULL || !var->chunksizes[0]) && !unlimdim) var->contiguous = NC_TRUE; @@ -2542,7 +2548,7 @@ NC4_hdf5get_superblock(struct NC_FILE_INFO* h5, int* idp) return stat; } -static int NC4_get_strict_att(NC_FILE_INFO_T*); +static int NC4_strict_att_exists(NC_FILE_INFO_T*); static int NC4_walk(hid_t, int*); /** @@ -2574,11 +2580,12 @@ NC4_isnetcdf4(struct NC_FILE_INFO* h5) { int stat; int isnc4 = 0; + int exists; int count; /* Look for NC3_STRICT_ATT_NAME */ - isnc4 = NC4_get_strict_att(h5); - if(isnc4 > 0) + exists = NC4_strict_att_exists(h5); + if(exists) goto done; /* attribute did not exist */ /* => last resort: walk the HDF5 file looking for markers */ @@ -2595,26 +2602,26 @@ NC4_isnetcdf4(struct NC_FILE_INFO* h5) } /** - * @internal Get the NC3 strict attribute. + * @internal See if the NC3 strict attribute exists. * * @param h5 Pointer to HDF5 file info struct. * - * @returns NC_NOERR No error. + * @returns 1 if error || exists; 0 otherwise * @author Dennis Heimbigner. */ static int -NC4_get_strict_att(NC_FILE_INFO_T *h5) +NC4_strict_att_exists(NC_FILE_INFO_T *h5) { hid_t grpid = -1; - hid_t attid = -1; - + htri_t attr_exists; + /* Get root group ID. */ grpid = ((NC_HDF5_GRP_INFO_T *)(h5->root_grp->format_grp_info))->hdf_grpid; - /* Try to extract the NC3_STRICT_ATT_NAME attribute */ - attid = H5Aopen_name(grpid, NC3_STRICT_ATT_NAME); - H5Aclose(attid); - return attid; + /* See if the NC3_STRICT_ATT_NAME attribute exists */ + if ((attr_exists = H5Aexists(grpid, NC3_STRICT_ATT_NAME)) < 0) + return 1; + return (attr_exists?1:0); } /** @@ -2683,121 +2690,22 @@ NC4_walk(hid_t gid, int* countp) return ncstat; } - -/**************************************************/ -/* Filter registration support */ - -static int -filterlookup(int id) -{ - int i; - if(filters == NULL) - filters = nclistnew(); - for(i=0;iid == id) return i; /* return position */ - } - return -1; -} - -static void -reclaiminfo(NC_FILTER_INFO* info) -{ - if(info != NULL) - nullfree(info->info); - nullfree(info); -} - -static int -filterremove(int pos) -{ - NC_FILTER_INFO* info = NULL; - if(filters == NULL) - filters = nclistnew(); - if(pos < 0 || pos >= nclistlength(filters)) - return NC_EINVAL; - info = nclistget(filters,pos); - reclaiminfo(info); - nclistremove(filters,pos); - return NC_NOERR; -} - -static NC_FILTER_INFO* -dupfilterinfo(NC_FILTER_INFO* info) -{ - NC_FILTER_INFO* dup = NULL; - if(info == NULL) goto fail; - if(info->info == NULL) goto fail; - if((dup = calloc(1,sizeof(NC_FILTER_INFO))) == NULL) goto fail; - *dup = *info; - if((dup->info = calloc(1,sizeof(H5Z_class2_t))) == NULL) goto fail; - { - H5Z_class2_t* h5dup = (H5Z_class2_t*)dup->info; - H5Z_class2_t* h5info = (H5Z_class2_t*)info->info; - *h5dup = *h5info; - } - return dup; -fail: - reclaiminfo(dup); - return NULL; -} - int -nc4_filter_action(int op, int format, int id, NC_FILTER_INFO* info) +NC4_hdf5_remove_filter(NC_VAR_INFO_T* var, unsigned int filterid) { int stat = NC_NOERR; - H5Z_class2_t* h5filterinfo = NULL; - herr_t herr; - int pos = -1; - NC_FILTER_INFO* dup = NULL; - - if(format != NC_FILTER_FORMAT_HDF5) - {stat = NC_ENOTNC4; goto done;} - - switch (op) { - case FILTER_REG: /* Ignore id argument */ - if(info == NULL || info->info == NULL) - {stat = NC_EINVAL; goto done;} - if(info->version != NC_FILTER_INFO_VERSION - || info->format != NC_FILTER_FORMAT_HDF5) - {stat = NC_ENOTNC4; goto done;} - h5filterinfo = info->info; - /* Another sanity check */ - if(info->id != h5filterinfo->id) - {stat = NC_EINVAL; goto done;} - /* See if this filter is already defined */ - if((pos = filterlookup(id)) >= 0) - {stat = NC_ENAMEINUSE; goto done;} /* Already defined */ - if((herr = H5Zregister(h5filterinfo)) < 0) - {stat = NC_EFILTER; goto done;} - /* Save a copy of the passed in info */ - if((dup = dupfilterinfo(info)) == NULL) - {stat = NC_ENOMEM; goto done;} - nclistpush(filters,dup); - break; - case FILTER_UNREG: - if(id <= 0) - {stat = NC_ENOTNC4; goto done;} - /* See if this filter is already defined */ - if((pos = filterlookup(id)) < 0) - {stat = NC_EFILTER; goto done;} /* Not defined */ - if((herr = H5Zunregister(id)) < 0) - {stat = NC_EFILTER; goto done;} - if((stat=filterremove(pos))) goto done; - break; - case FILTER_INQ: - if(id <= 0) - {stat = NC_ENOTNC4; goto done;} - /* Look up the id in our local table */ - if((pos = filterlookup(id)) < 0) - {stat = NC_EFILTER; goto done;} /* Not defined */ - if(info != NULL) { - *info = *((NC_FILTER_INFO*)nclistget(filters,pos)); - } - break; - default: - {stat = NC_EINTERNAL; goto done;} - } + NC_HDF5_VAR_INFO_T *hdf5_var; + hid_t propid = -1; + herr_t herr = -1; + H5Z_filter_t hft; + + hdf5_var = (NC_HDF5_VAR_INFO_T *)var->format_var_info; + if ((propid = H5Dget_create_plist(hdf5_var->hdf_datasetid)) < 0) + {stat = NC_EHDFERR; goto done;} + + hft = filterid; + if((herr = H5Premove_filter(propid,hft)) < 0) + {stat = NC_EHDFERR; goto done;} done: return stat; -} +} diff --git a/libsrc/nc3dispatch.c b/libsrc/nc3dispatch.c index 165caff4d6..1fadc25e54 100644 --- a/libsrc/nc3dispatch.c +++ b/libsrc/nc3dispatch.c @@ -164,6 +164,7 @@ NC3_def_var_filter, NC3_set_var_chunk_cache, NC3_get_var_chunk_cache, +NC_NOTNC4_filter_actions, }; const NC_Dispatch* NC3_dispatch_table = NULL; /*!< NC3 Dispatch table, moved here from ddispatch.c */ diff --git a/libsrc4/CMakeLists.txt b/libsrc4/CMakeLists.txt index 59aabbb45a..30888447a5 100644 --- a/libsrc4/CMakeLists.txt +++ b/libsrc4/CMakeLists.txt @@ -6,7 +6,9 @@ # See netcdf-c/COPYRIGHT file for more info. # Process these files with m4. -SET(libsrc4_SOURCES nc4dispatch.c nc4attr.c nc4dim.c nc4grp.c nc4type.c nc4var.c ncfunc.c nc4internal.c ncindex.c ) +SET(libsrc4_SOURCES nc4dispatch.c nc4attr.c nc4dim.c nc4grp.c +nc4internal.c nc4type.c nc4var.c ncfunc.c error4.c nc4printer.c +ncindex.c) IF(LOGGING) SET(libsrc4_SOURCES ${libsrc4_SOURCES} error4.c) diff --git a/libsrc4/Makefile.am b/libsrc4/Makefile.am index 1a3d6c842f..4e4444f132 100644 --- a/libsrc4/Makefile.am +++ b/libsrc4/Makefile.am @@ -14,4 +14,5 @@ libnetcdf4_la_SOURCES = nc4dispatch.c nc4attr.c nc4dim.c nc4grp.c \ nc4internal.c nc4type.c nc4var.c ncfunc.c error4.c nc4printer.c \ ncindex.c + EXTRA_DIST = CMakeLists.txt diff --git a/libsrc4/nc4dispatch.c b/libsrc4/nc4dispatch.c index e3a162ae9e..474d6aee38 100644 --- a/libsrc4/nc4dispatch.c +++ b/libsrc4/nc4dispatch.c @@ -11,6 +11,7 @@ #include "config.h" #include +#include "netcdf.h" #include "nc4internal.h" #include "nc4dispatch.h" #include "nc.h" diff --git a/libsrc4/nc4internal.c b/libsrc4/nc4internal.c index 2a7dff9c09..ab23190bca 100644 --- a/libsrc4/nc4internal.c +++ b/libsrc4/nc4internal.c @@ -16,6 +16,8 @@ * @author Ed Hartnett, Dennis Heimbigner, Ward Fisher */ #include "config.h" +#include "netcdf.h" +#include "netcdf_filter.h" #include "nc4internal.h" #include "nc.h" /* from libsrc */ #include "ncdispatch.h" /* from libdispatch */ @@ -26,6 +28,8 @@ size_t nc4_chunk_cache_size = CHUNK_CACHE_SIZE; /**< Default chunk ca size_t nc4_chunk_cache_nelems = CHUNK_CACHE_NELEMS; /**< Default chunk cache number of elements. */ float nc4_chunk_cache_preemption = CHUNK_CACHE_PREEMPTION; /**< Default chunk cache preemption. */ +static void freefilterlist(NClist* filters); + #ifdef LOGGING /* This is the severity level of messages which will be logged. Use severity 0 for errors, 1 for important log messages, 2 for less @@ -1335,9 +1339,8 @@ var_free(NC_VAR_INFO_T *var) if (var->dimscale_attached) free(var->dimscale_attached); - /* Release parameter information. */ - if (var->params) - free(var->params); + /* Release filter information. */ + freefilterlist(var->filters); /* Delete any format-specific info. */ if (var->format_var_info) @@ -1454,9 +1457,11 @@ nc4_rec_grp_del(NC_GRP_INFO_T *grp) ncindexfree(grp->att); /* Delete all vars. */ - for (i = 0; i < ncindexsize(grp->vars); i++) - if ((retval = var_free((NC_VAR_INFO_T *)ncindexith(grp->vars, i)))) + for (i = 0; i < ncindexsize(grp->vars); i++) { + NC_VAR_INFO_T* v = (NC_VAR_INFO_T *)ncindexith(grp->vars, i); + if ((retval = var_free(v))) return retval; + } ncindexfree(grp->vars); /* Delete all dims, and free the list of dims. */ @@ -1809,3 +1814,15 @@ NC4_show_metadata(int ncid) #endif /*LOGGING*/ return retval; } + +static void +freefilterlist(NClist* filters) +{ + int i; + if(filters == NULL) return; + for(i=0;ideflate; - if (deflate_levelp) - *deflate_levelp = var->deflate_level; if (shufflep) *shufflep = (int)var->shuffle; if (fletcher32p) *fletcher32p = (int)var->fletcher32; - if (idp) - *idp = var->filterid; - if (nparamsp) - *nparamsp = (var->params == NULL ? 0 : var->nparams); - if (params && var->params != NULL) - memcpy(params,var->params,var->nparams*sizeof(unsigned int)); + if (deflatep) + return NC_EFILTER; + + if (idp) { +#if 0 + NC* nc = h5->controller; + NC_FILTER_ACTION action; + action.action = NCFILTER_INQ_FILTER; + action.format = NC_FORMATX_NC_HDF5; + action.id = (idp)?*idp:0; + action.nelems = (nparamsp)?*nparamsp:0; + action.elems = params; + if((retval = nc->dispatch->filter_actions(ncid,varid,&action)) == NC_NOERR) { + if(idp) *idp = action.id; + if(nparamsp) *nparamsp = action.nelems; + } + return retval; +#else + return NC_EFILTER; +#endif + } /* Fill value stuff. */ if (no_fill) diff --git a/libsrcp/ncpdispatch.c b/libsrcp/ncpdispatch.c index 375817b650..9d3714bc46 100644 --- a/libsrcp/ncpdispatch.c +++ b/libsrcp/ncpdispatch.c @@ -1460,6 +1460,7 @@ NC_NOTNC4_def_var_filter, NC_NOTNC4_set_var_chunk_cache, NC_NOTNC4_get_var_chunk_cache, +NC_NOTNC4_filter_actions, }; const NC_Dispatch *NCP_dispatch_table = NULL; /* moved here from ddispatch.c */ diff --git a/nc_test4/CMakeLists.txt b/nc_test4/CMakeLists.txt index 48cc85587d..d51ebfada1 100644 --- a/nc_test4/CMakeLists.txt +++ b/nc_test4/CMakeLists.txt @@ -33,6 +33,8 @@ IF(ENABLE_FILTER_TESTING) build_bin_test(test_filter) build_bin_test(test_filter_misc) build_bin_test(test_filter_reg) + build_bin_test(tst_multifilter) + build_bin_test(test_filter_order) ADD_SH_TEST(nc_test4 tst_filter) SET(NC4_TESTS ${NC4_TESTS} tst_filterparser test_filter_reg) ENDIF(ENABLE_FILTER_TESTING) diff --git a/nc_test4/Makefile.am b/nc_test4/Makefile.am index f135d4b0cd..f799654b1f 100644 --- a/nc_test4/Makefile.am +++ b/nc_test4/Makefile.am @@ -19,7 +19,6 @@ include $(top_srcdir)/lib_flags.am TEST_EXTENSIONS = .sh # Link to our assembled library. -AM_LDFLAGS += ${top_builddir}/liblib/libnetcdf.la LDADD = ${top_builddir}/liblib/libnetcdf.la # These are netCDF-4 C test programs which are built and run. @@ -73,8 +72,9 @@ endif # Filter Tests (requires ncdump and ncgen) if ENABLE_FILTER_TESTING extradir = -extra_PROGRAMS = test_filter test_filter_misc +extra_PROGRAMS = test_filter test_filter_misc test_filter_order check_PROGRAMS += test_filter_reg +check_PROGRAMS += tst_multifilter TESTS += tst_filter.sh test_filter_reg endif endif # BUILD_UTILITIES @@ -95,7 +95,8 @@ run_empty_vlen_test.sh ref_hdf5_compat1.nc ref_hdf5_compat2.nc \ ref_hdf5_compat3.nc tst_misc.sh tdset.h5 tst_szip.sh ref_szip.h5 \ ref_szip.cdl tst_filter.sh bzip2.cdl ref_filtered.cdl \ ref_unfiltered.cdl ref_bzip2.c findplugin.in ref_unfilteredvv.cdl \ -ref_filteredvv.cdl +ref_filteredvv.cdl ref_multi.cdl ref_filter_order.txt \ +ref_ncgenF.cdl ref_nccopyF.cdl CLEANFILES = tst_mpi_parallel.bin cdm_sea_soundings.nc bm_chunking.nc \ tst_floats_1D.cdl floats_1D_3.nc floats_1D.cdl tst_*.nc \ diff --git a/nc_test4/ref_bzip2.c b/nc_test4/ref_bzip2.c index ccf5515d60..416f1c792d 100644 --- a/nc_test4/ref_bzip2.c +++ b/nc_test4/ref_bzip2.c @@ -4,7 +4,7 @@ static size_t var_chunksizes[4] = {4, 4, 4, 4} ; -static unsigned int var_filterparams[1] = {9U} ; +static unsigned int var_0_filterparams[1] = {9U} ; void check_err(const int stat, const int line, const char *file) { @@ -72,7 +72,7 @@ main() {/* create bzip2.nc */ check_err(stat,__LINE__,__FILE__); stat = nc_def_var_fill(bzip2_grp, var_id, NC_NOFILL, NULL); check_err(stat,__LINE__,__FILE__); - stat = nc_def_var_filter(bzip2_grp, var_id, 307, 1, var_filterparams); + stat = nc_def_var_filter(bzip2_grp, var_id, 307, 1, var_0_filterparams); check_err(stat,__LINE__,__FILE__); /* leave define mode */ diff --git a/nc_test4/ref_filter_order.txt b/nc_test4/ref_filter_order.txt new file mode 100644 index 0000000000..64d8436d21 --- /dev/null +++ b/nc_test4/ref_filter_order.txt @@ -0,0 +1,17 @@ +test1: filter order. +set var filter 0 +test: nparams=1: params= 0 +dimsizes=4,4,4,4 +chunksizes=4,4,4,4 +set var filter 1 +test: nparams=1: params= 1 +dimsizes=4,4,4,4 +chunksizes=4,4,4,4 +test1: compression. +Apply filter: 0 +Apply filter: 1 +test1: decompression. +Apply filter: 1 +Apply filter: 0 +data comparison: |array|=256 +no data errors diff --git a/nc_test4/ref_multi.cdl b/nc_test4/ref_multi.cdl new file mode 100644 index 0000000000..4e6d50c72f --- /dev/null +++ b/nc_test4/ref_multi.cdl @@ -0,0 +1,16 @@ +netcdf multifilter { +dimensions: + dim0 = 4 ; + dim1 = 4 ; + dim2 = 4 ; + dim3 = 4 ; +variables: + float var(dim0, dim1, dim2, dim3) ; + var:_Storage = "chunked" ; + var:_ChunkSizes = 4, 4, 4, 4 ; + var:_Filter = "307,9|1,2|40000" ; + var:_NoFill = "true" ; + +// global attributes: + :_Format = "netCDF-4" ; +} diff --git a/nc_test4/ref_nccopyF.cdl b/nc_test4/ref_nccopyF.cdl new file mode 100644 index 0000000000..f064c15549 --- /dev/null +++ b/nc_test4/ref_nccopyF.cdl @@ -0,0 +1,21 @@ +netcdf nccopyF { +dimensions: + dim0 = 4 ; + dim1 = 4 ; + dim2 = 4 ; + dim3 = 4 ; + +// global attributes: + :_Format = "netCDF-4" ; + +group: g { + variables: + float var(dim0, dim1, dim2, dim3) ; + var:_Storage = "chunked" ; + var:_ChunkSizes = 4, 4, 4, 4 ; + var:_Filter = "307,4|40000" ; + var:_NoFill = "true" ; + + // group attributes: + } // group g +} diff --git a/nc_test4/ref_ncgenF.cdl b/nc_test4/ref_ncgenF.cdl new file mode 100644 index 0000000000..a9c5dfb363 --- /dev/null +++ b/nc_test4/ref_ncgenF.cdl @@ -0,0 +1,21 @@ +netcdf multifiltered { +dimensions: + dim0 = 4 ; + dim1 = 4 ; + dim2 = 4 ; + dim3 = 4 ; + +// global attributes: + :_Format = "netCDF-4" ; + +group: g { + variables: + float var(dim0, dim1, dim2, dim3) ; + var:_Storage = "chunked" ; + var:_ChunkSizes = 4, 4, 4, 4 ; + var:_Filter = "307,4|4,32,32" ; + var:_NoFill = "true" ; + + // group attributes: + } // group g +} diff --git a/nc_test4/test_filter_misc.c b/nc_test4/test_filter_misc.c index 0b7ff21f5d..95b1d1821c 100644 --- a/nc_test4/test_filter_misc.c +++ b/nc_test4/test_filter_misc.c @@ -309,7 +309,7 @@ insert(int index, void* src, size_t size) void* dst = &baseline[index]; if(size == 8) { memcpy(src8,src,size); - NC_filterfix8(src8,0); + NC4_filterfix8(src8,0); src = src8; } memcpy(dst,src,size); diff --git a/nc_test4/test_filter_order.c b/nc_test4/test_filter_order.c new file mode 100644 index 0000000000..01dc17cad1 --- /dev/null +++ b/nc_test4/test_filter_order.c @@ -0,0 +1,609 @@ +/* + Copyright 2018, UCAR/Unidata + See COPYRIGHT file for copying and redistribution conditions. +*/ + +#include "config.h" +#include +#include +#include +#include +#include + +#include +/* Older versions of the hdf library may define H5PL_type_t here */ +#include + +#include "netcdf.h" +#include "netcdf_filter.h" + +#undef TESTODDSIZE + +#undef DEBUG + +#define FILTER_ID 32768 + +#define MAXERRS 8 + +#define MAXPARAMS 32 + +#define MAXDIMS 8 + +#define TESTFILE "testfilter_reg.nc" + +#define NPARAMS 1 +#define PARAMVAL 17 + +#define NDIMS 4 +static size_t dimsize[NDIMS] = {4,4,4,4}; +static size_t chunksize[NDIMS] = {4,4,4,4}; + +static size_t ndims = NDIMS; + +static size_t totalproduct = 1; /* x-product over max dims */ +static size_t actualproduct = 1; /* x-product over actualdims */ +static size_t chunkproduct = 1; /* x-product over actual chunks */ + +static int nerrs = 0; + +static int ncid, varid; +static int dimids[MAXDIMS]; +static size_t odom[MAXDIMS]; +static float* array = NULL; +static float* expected = NULL; + +static size_t nparams = 0; +static unsigned int params[MAXPARAMS]; +static unsigned int baseline[2][NPARAMS] = {{0},{1}}; + +static struct Base { + unsigned int id; + H5Z_class2_t* info; +} baseinfo; + +static const H5Z_class2_t H5Z_REG[2]; + +/* Forward */ +static int filter_test1(void); +static void init(int argc, char** argv); +static void reset(void); +static void odom_reset(void); +static int odom_more(void); +static int odom_next(void); +static int odom_offset(void); +static float expectedvalue(void); +static void verifyparams(int); + +#define ERRR do { \ +fflush(stdout); /* Make sure our stdout is synced with stderr. */ \ +fprintf(stderr, "Sorry! Unexpected result, %s, line: %d\n", \ + __FILE__, __LINE__); \ +nerrs++;\ +} while (0) + +static int +check(int err,int line) +{ + if(err != NC_NOERR) { + fprintf(stderr,"fail (%d): %s\n",line,nc_strerror(err)); + } + return NC_NOERR; +} + +static void +report(const char* msg, int lineno) +{ + fprintf(stderr,"fail: line=%d %s\n",lineno,msg); + exit(1); +} + +#define CHECK(x) check(x,__LINE__) +#define REPORT(x) report(x,__LINE__) + +static int +verifychunks(void) +{ + int i; + int store = -1; + size_t localchunks[MAXDIMS]; + memset(localchunks,0,sizeof(localchunks)); + CHECK(nc_inq_var_chunking(ncid, varid, &store, localchunks)); + if(store != NC_CHUNKED) { + fprintf(stderr,"bad chunk store\n"); + return 0; + } + for(i=0;iid != base->id) + {stat = NC_EINVAL; fprintf(stderr,"verifyfilterinfo: id mismatch\n");} +#ifdef USE_HDF5 + H5Z_class2_t* h5info = info->info; + H5Z_class2_t* h5base = base->info; + if(info->id != base->id) + {stat = NC_EINVAL; fprintf(stderr,"verifyfilterinfo: H5Z_class_t: id mismatch\n");} + if(h5info->encoder_present != h5base->encoder_present) + {stat = NC_EINVAL; fprintf(stderr,"verifyfilterinfo: H5Z_class_t: encoder_present mismatch\n");} + if(h5info->decoder_present != h5base->decoder_present) + {stat = NC_EINVAL; fprintf(stderr,"verifyfilterinfo: H5Z_class_t: decoder_present mismatch\n");} + if(h5info->decoder_present != h5base->decoder_present) + {stat = NC_EINVAL; fprintf(stderr,"verifyfilterinfo: H5Z_class_t: decoder_present mismatch\n");} + if(strcmp(h5info->name,h5base->name) != 0) + {stat = NC_EINVAL; fprintf(stderr,"verifyfilterinfo: H5Z_class_t: name mismatch\n");} + if(h5info->can_apply != h5base->can_apply) + {stat = NC_EINVAL; fprintf(stderr,"verifyfilterinfo: H5Z_class_t: can_apply mismatch\n");} + if(h5info->set_local != h5base->set_local) + {stat = NC_EINVAL; fprintf(stderr,"verifyfilterinfo: H5Z_class_t: set_local mismatch\n");} + if(h5info->filter != h5base->filter) + {stat = NC_EINVAL; fprintf(stderr,"verifyfilterinfo: H5Z_class_t: filter mismatch\n");} +#else + stat = NC_ENOTBUILT; fprintf(stderr,"Unknown format\n")} +#endif + return stat; +} + +static void +registerfilters(void) +{ + struct Base inqinfo; + + /* Register filter 0 */ + baseinfo.id = FILTER_ID; + baseinfo.info = (H5Z_class2_t*)&H5Z_REG[0]; + CHECK(nc_filter_client_register(baseinfo.id,baseinfo.info)); + /* Verify by inquiry */ + memset(&inqinfo,0,sizeof(struct Base)); + inqinfo.id = FILTER_ID; + if((inqinfo.info = (H5Z_class2_t*)calloc(1,sizeof(H5Z_class2_t))) == NULL) + CHECK(NC_ENOMEM); + CHECK((nc_filter_client_inq(inqinfo.id,(void*)inqinfo.info))); + CHECK((verifyfilterinfo(&inqinfo,&baseinfo))); + nullfree(inqinfo.info); + + /* Register filter 1 */ + baseinfo.id = FILTER_ID+1; + baseinfo.info = (H5Z_class2_t*)&H5Z_REG[1]; + CHECK(nc_filter_client_register(baseinfo.id,baseinfo.info)); + /* Verify by inquiry */ + memset(&inqinfo,0,sizeof(struct Base)); + inqinfo.id = FILTER_ID+1; + if((inqinfo.info = (H5Z_class2_t*)calloc(1,sizeof(H5Z_class2_t))) == NULL) + CHECK(NC_ENOMEM); + CHECK((nc_filter_client_inq(inqinfo.id,(void*)inqinfo.info))); + CHECK((verifyfilterinfo(&inqinfo,&baseinfo))); + nullfree(inqinfo.info); +} + +static void +setvarfilter(int index) +{ + /* NOOP the params */ + CHECK(nc_def_var_filter(ncid,varid,FILTER_ID+index,NPARAMS,baseline[index])); + verifyparams(index); +} + +static void +verifyparams(int index) +{ + int i; + CHECK(nc_inq_var_filter_info(ncid,varid,FILTER_ID+index,&nparams,params)); + if(nparams != NPARAMS) REPORT("nparams mismatch"); + for(i=0;i= MAXERRS) + break; + } + } + } else + { + odom_reset(); + while(odom_more()) { + int offset = odom_offset(); + float expect = expectedvalue(); + if(array[offset] != expect) { + fprintf(stderr,"data mismatch: array[%d]=%f expected=%f\n", + offset,array[offset],expect); + errs++; + if(errs >= MAXERRS) + break; + } + odom_next(); + } + } + + if(errs == 0) + printf("no data errors\n"); + return (errs == 0); +} + +static void +showparameters(void) +{ + int i; + printf("test: nparams=%ld: params=",(unsigned long)nparams); + for(i=0;i=0;i--) { + odom[i] += 1; + if(odom[i] < dimsize[i]) break; + if(i == 0) return 0; /* leave the 0th entry if it overflows*/ + odom[i] = 0; /* reset this position*/ + } + return 1; +} + +static int +odom_offset(void) +{ + int i; + int offset = 0; + for(i=0;i 0?1:0); +} + +/**************************************************/ +/* In-line filter code */ + +#define H5Z_FILTER_REG0 FILTER_ID +#define H5Z_FILTER_REG1 (FILTER_ID+1) + +#ifndef DLL_EXPORT +#define DLL_EXPORT +#endif + +static int paramcheck(size_t nparams, const unsigned int* params); + +/* Forward */ +static int paramcheck(size_t nparams, const unsigned int* params); + +/* Make this explicit */ +/* + * The "can_apply" callback returns positive a valid combination, zero for an + * invalid combination and negative for an error. + */ +static htri_t +H5Z_reg_can_apply(hid_t dcpl_id, hid_t type_id, hid_t space_id) +{ + return 1; /* Assume it can always apply */ +} + +/* +As a filter, it is the identity function, +passing input to output unchanged. +*/ + +size_t +H5Z_filter_reg(unsigned int flags, size_t cd_nelmts, + const unsigned int cd_values[], size_t nbytes, + size_t *buf_size, void **buf) +{ + void* newbuf; + + if(cd_nelmts == 0) + goto fail; + + if(!paramcheck(cd_nelmts,cd_values)) + goto fail; + + printf("Apply filter: %u\n",cd_values[0]); + + if (flags & H5Z_FLAG_REVERSE) { + /* Replace buffer */ +#ifdef HAVE_H5ALLOCATE_MEMORY + newbuf = H5allocate_memory(*buf_size,0); +#else + newbuf = malloc(*buf_size); +#endif + if(newbuf == NULL) abort(); + if(*buf != NULL) { + memcpy(newbuf,*buf,*buf_size); + /* reclaim old buffer */ +#ifdef HAVE_H5FREE_MEMORY + H5free_memory(*buf); +#else + free(*buf); +#endif + } + *buf = newbuf; + + } else { + + /* Replace buffer */ +#ifdef HAVE_H5ALLOCATE_MEMORY + newbuf = H5allocate_memory(*buf_size,0); +#else + newbuf = malloc(*buf_size); +#endif + if(newbuf == NULL) abort(); + if(*buf != NULL) { + memcpy(newbuf,*buf,*buf_size); + /* reclaim old buffer */ +#ifdef HAVE_H5FREE_MEMORY + H5free_memory(*buf); +#else + free(*buf); +#endif + } + *buf = newbuf; + + } + + return *buf_size; + +fail: + return 0; +} + +static int +paramcheck(size_t nparams, const unsigned int* params) +{ + if(nparams != 1) { + fprintf(stderr,"Incorrect parameter count: need=1 sent=%ld\n",(unsigned long)nparams); + goto fail; + } + + return 1; + +fail: + return 0; +} + +static const H5Z_class2_t H5Z_REG[2] = { +{ + H5Z_CLASS_T_VERS, /* H5Z_class_t version */ + (H5Z_filter_t)(H5Z_FILTER_REG0), /* Filter id number */ + 1, /* encoder_present flag (set to true) */ + 1, /* decoder_present flag (set to true) */ + "registered0", /* Filter name for debugging */ + (H5Z_can_apply_func_t)H5Z_reg_can_apply, /* The "can apply" callback */ + NULL, /* The "set local" callback */ + (H5Z_func_t)H5Z_filter_reg, /* The actual filter function */ +}, +{ + H5Z_CLASS_T_VERS, /* H5Z_class_t version */ + (H5Z_filter_t)(H5Z_FILTER_REG1), /* Filter id number */ + 1, /* encoder_present flag (set to true) */ + 1, /* decoder_present flag (set to true) */ + "registered1", /* Filter name for debugging */ + (H5Z_can_apply_func_t)H5Z_reg_can_apply, /* The "can apply" callback */ + NULL, /* The "set local" callback */ + (H5Z_func_t)H5Z_filter_reg, /* The actual filter function */ +} +}; diff --git a/nc_test4/test_filter_reg.c b/nc_test4/test_filter_reg.c index 8794d19979..93820bfc51 100644 --- a/nc_test4/test_filter_reg.c +++ b/nc_test4/test_filter_reg.c @@ -57,7 +57,10 @@ static size_t nparams = 0; static unsigned int params[MAXPARAMS]; static unsigned int baseline[NPARAMS] = {PARAMVAL}; -static NC_FILTER_INFO baseinfo; +static struct Base { + unsigned int id; + H5Z_class2_t* info; +} baseinfo; static const H5Z_class2_t H5Z_REG[1]; @@ -84,6 +87,7 @@ check(int err,int line) { if(err != NC_NOERR) { fprintf(stderr,"fail (%d): %s\n",line,nc_strerror(err)); + nerrs++; } return NC_NOERR; } @@ -137,21 +141,14 @@ create(void) } static int -verifyfilterinfo(NC_FILTER_INFO* info, NC_FILTER_INFO* base) +verifyfilterinfo(struct Base* info, struct Base* base) { int stat = NC_NOERR; - if(info->version != base->version) - {stat = NC_EINVAL; fprintf(stderr,"verifyfilterinfo: version mismatch\n");} - if(info->format != base->format) - {stat = NC_EINVAL; fprintf(stderr,"verifyfilterinfo: format mismatch\n");} if(info->id != base->id) {stat = NC_EINVAL; fprintf(stderr,"verifyfilterinfo: id mismatch\n");} - if(info->format == NC_FILTER_FORMAT_HDF5) { #ifdef USE_HDF5 H5Z_class2_t* h5info = (H5Z_class2_t*)info->info; H5Z_class2_t* h5base = (H5Z_class2_t*)base->info; - if(h5info->version != h5base->version) - {stat = NC_EINVAL; fprintf(stderr,"verifyfilterinfo: H5Z_class_t: version mismatch\n");} if(h5info->id != h5base->id) {stat = NC_EINVAL; fprintf(stderr,"verifyfilterinfo: H5Z_class_t: id mismatch\n");} if(h5info->encoder_present != h5base->encoder_present) @@ -171,46 +168,51 @@ verifyfilterinfo(NC_FILTER_INFO* info, NC_FILTER_INFO* base) #else stat = NC_ENOTBUILT; fprintf(stderr,"Unknown format\n")} #endif - } else - {stat = NC_EINVAL; fprintf(stderr,"Unknown format\n");} - return stat; } static void registerfilter(void) { - NC_FILTER_INFO inqinfo; + struct Base inqinfo; - baseinfo.version = NC_FILTER_INFO_VERSION; - baseinfo.format = NC_FILTER_FORMAT_HDF5; baseinfo.id = FILTER_ID; - baseinfo.info = (void*)&H5Z_REG[0]; - CHECK(nc_filter_register(&baseinfo)); + baseinfo.info = (H5Z_class2_t*)&H5Z_REG[0]; + CHECK(nc_filter_client_register(baseinfo.id,baseinfo.info)); /* Verify by inquiry */ - memset(&inqinfo,0,sizeof(NC_FILTER_INFO)); - CHECK((nc_filter_inq(NC_FILTER_FORMAT_HDF5, FILTER_ID, &inqinfo))); + memset(&inqinfo,0,sizeof(struct Base)); + inqinfo.id = FILTER_ID; + inqinfo.info = calloc(1,sizeof(H5Z_class2_t)); + if(inqinfo.info == NULL) CHECK(NC_ENOMEM); + CHECK((nc_filter_client_inq(inqinfo.id,(void*)inqinfo.info))); CHECK((verifyfilterinfo(&inqinfo,&baseinfo))); + nullfree(inqinfo.info); } static void unregisterfilter(void) { int stat = NC_NOERR; - NC_FILTER_INFO inqinfo; + struct Base inqinfo; /* Verify that the filter info is still good */ - memset(&inqinfo,0,sizeof(NC_FILTER_INFO)); - CHECK((stat = nc_filter_inq(NC_FILTER_FORMAT_HDF5, FILTER_ID, &inqinfo))); + memset(&inqinfo,0,sizeof(struct Base)); + inqinfo.id = FILTER_ID; + inqinfo.info = calloc(1,sizeof(H5Z_class2_t)); + if(inqinfo.info == NULL) CHECK(NC_ENOMEM); + CHECK((stat=nc_filter_client_inq(inqinfo.id,(void*)inqinfo.info))); CHECK((verifyfilterinfo(&inqinfo,&baseinfo))); /* Unregister */ - CHECK((stat = nc_filter_unregister(NC_FILTER_FORMAT_HDF5, FILTER_ID))); + inqinfo.id = FILTER_ID; + CHECK((stat = nc_filter_client_unregister(inqinfo.id))); /* Inq again to verify unregistered */ - stat = nc_filter_inq(NC_FILTER_FORMAT_HDF5, FILTER_ID, NULL); - if(stat != NC_EFILTER) { + inqinfo.id = FILTER_ID; + stat=nc_filter_client_inq(inqinfo.id,(void*)inqinfo.info); + if(stat != NC_ENOFILTER) { fprintf(stderr,"unregister: failed\n"); CHECK(NC_EFILTER); } + nullfree(inqinfo.info); } static void diff --git a/nc_test4/test_szip.c b/nc_test4/test_szip.c index b9a26af043..55ebbf9843 100644 --- a/nc_test4/test_szip.c +++ b/nc_test4/test_szip.c @@ -80,12 +80,7 @@ main(void) */ szip_params[0] = H5_SZIP_NN_OPTION_MASK; szip_params[1] = H5_SZIP_MAX_PIXELS_PER_BLOCK_IN; - { int stat = nc_def_var_filter(ncid, varid, HDF5_FILTER_SZIP, 2, szip_params); - if(stat) { - fprintf(stderr,"XXX: %d %s\b",stat,nc_strerror(stat)); - ERR; - } - } + if(nc_def_var_filter(ncid, varid, HDF5_FILTER_SZIP, 2, szip_params)) ERR; if(nc_inq_var_szip(ncid, varid, &options_mask_in, &pixels_per_block_in)) ERR; if(!(options_mask_in & H5_SZIP_NN_OPTION_MASK)) ERR; #endif diff --git a/nc_test4/tst_filter.sh b/nc_test4/tst_filter.sh index 548c0bb95a..a7f213e3e4 100755 --- a/nc_test4/tst_filter.sh +++ b/nc_test4/tst_filter.sh @@ -10,6 +10,8 @@ NCP=1 UNK=1 NGC=1 MISC=1 +MULTI=1 +ORDER=1 # Load the findplugins function . ${builddir}/findplugin.sh @@ -19,10 +21,10 @@ echo "findplugin.sh loaded" # These attributes might be platform dependent sclean() { cat $1 \ + | sed -e '/:_IsNetcdf4/d' \ | sed -e '/:_Endianness/d' \ | sed -e '/_NCProperties/d' \ | sed -e '/_SuperblockVersion/d' \ - | sed -e '/_IsNetcdf4/d' \ | cat > $2 } @@ -51,6 +53,14 @@ export HDF5_PLUGIN_PATH if ! test -f ${BZIP2PATH} ; then echo "Unable to locate ${BZIP2PATH}"; exit 1; fi if ! test -f ${MISCPATH} ; then echo "Unable to locate ${MISCPATH}"; exit 1; fi +# See if we have szip +HAVE_SZIP=0 +if test -f ${TOPBUILDDIR}/libnetcdf.settings ; then + if grep "SZIP Support:[ ]*yes" <${TOPBUILDDIR}/libnetcdf.settings ; then + HAVE_SZIP=1 + fi +fi + # Execute the specified tests if test "x$API" = x1 ; then @@ -117,8 +127,8 @@ sclean ./tst_filtervv.txt ./filteredvv.dump diff -b -w ${srcdir}/ref_filteredvv.cdl ./filteredvv.dump echo " *** Pass: nccopy '*' filter" -echo " *** Testing 'v|v' filter application" -${NCCOPY} -M0 -F "var1|/g/var2,307,9,4" unfilteredvv.nc filteredvbar.nc +echo " *** Testing 'v&v' filter application" +${NCCOPY} -M0 -F "var1&/g/var2,307,9,4" unfilteredvv.nc filteredvbar.nc ${NCDUMP} -n filteredvv -s filteredvbar.nc > ./tst_filtervbar.txt # Remove irrelevant -s output sclean ./tst_filtervbar.txt ./filteredvbar.dump @@ -135,19 +145,19 @@ test -s tst_filter2.txt echo " *** Pass: pass-thru of filters" echo " *** Testing -F none" -rm -f ./tst_filter.txt ./tst_filter2.txt ./tst_filter.nc -${NCCOPY} -M0 -F none ./filtered.nc ./tst_filter.nc -${NCDUMP} -s tst_filter.nc > ./tst_filter.txt -sed -e '/_Filter/p' -e d < ./tst_filter.txt >./tst_filter2.txt -test ! -s tst_filter2.txt +rm -f ./tst_none.txt ./tst_none2.txt ./tst_none.nc +${NCCOPY} -M0 -F none ./filtered.nc ./tst_none.nc +${NCDUMP} -hs tst_none.nc > ./tst_none.txt +sed -e '/_Filter/p' -e d < ./tst_none.txt >./tst_none2.txt +test ! -s tst_none2.txt echo " *** Pass: -F none" echo " *** Testing -F var,none " -rm -f ./tst_filter.txt ./tst_filter.nc -${NCCOPY} -M0 -F "/g/var,none" ./filtered.nc ./tst_filter.nc -${NCDUMP} -s tst_filter.nc > ./tst_filter.txt -sed -e '/_Filter/p' -e d < ./tst_filter.txt >tst_filter2.txt -test ! -s tst_filter2.txt +rm -f ./tst_vnone.txt ./tst_vnone.nc tst_vnone2.txt +${NCCOPY} -M0 -F "/g/var,none" ./filtered.nc ./tst_vnone.nc +${NCDUMP} -s tst_vnone.nc > ./tst_vnone.txt +sed -e '/_Filter/p' -e d < ./tst_vnone.txt >tst_vnone2.txt +test ! -s tst_vnone2.txt echo " *** Pass: -F var,none" echo "*** Pass: all nccopy filter tests" @@ -183,13 +193,54 @@ diff -b -w ${srcdir}/ref_bzip2.c ./test_bzip2.c echo "*** Pass: ncgen dynamic filter" fi -#cleanup -rm -f ./bzip*.nc ./unfiltered.nc ./filtered.nc ./tst_filter.txt ./tst_filter2.txt *.dump bzip*hdr.* -rm -f ./test_bzip2.c -rm -f ./testmisc.nc -rm -f ./tst_filter2.nc -rm -f ./unfilteredvv.nc ./filteredvv.nc ./filteredvbar.nc -rm -f ./tst_filtervv.txt ./tst_filtervbar.txt +if test "x$MULTI" = x1 ; then +echo "*** Testing multiple filters" +rm -f ./multifilter.nc ./multi.txt ./smulti.cdl +rm -f nccopyF.cdl nccopyF.nc ncgenF.cdl ncgenF.nc +${execdir}/tst_multifilter +${NCDUMP} -hs ./multifilter.nc >./multi.cdl +# Remove irrelevant -s output +sclean ./multi.cdl ./smulti.cdl +diff -b -w ${srcdir}/ref_multi.cdl ./smulti.cdl +echo "*** nccopy -F with multiple filters" +if ! test -f unfiltered.nc ; then + ${NCGEN} -4 -lb -o unfiltered.nc ${srcdir}/ref_unfiltered.cdl +fi +${NCCOPY} "-F/g/var,307,4|40000" unfiltered.nc nccopyF.nc +${NCDUMP} -hs nccopyF.nc > ./nccopyF.cdl +sclean nccopyF.cdl nccopyFs.cdl +diff -b -w ${srcdir}/ref_nccopyF.cdl ./nccopyFs.cdl +echo "*** ncgen with multiple filters" +${NCGEN} -4 -lb -o ncgenF.nc ${srcdir}/ref_nccopyF.cdl +# Need to fix name using -n +${NCDUMP} -hs -n nccopyF ncgenF.nc > ./ncgenF.cdl +sclean ncgenF.cdl ncgenFs.cdl +diff -b -w ${srcdir}/ref_nccopyF.cdl ./ncgenFs.cdl +echo "*** Pass: multiple filters" +fi + +if test "x$MULTI" = x1 ; then +echo "*** Testing multiple filter order of invocation" +rm -f filterorder.txt +${execdir}/test_filter_order >filterorder.txt +diff -b -w ${srcdir}/ref_filter_order.txt filterorder.txt +fi + echo "*** Pass: all selected tests passed" +#cleanup +rm -f testmisc.nc +rm -f unfiltered.nc unfilteredvv.nc filtered.nc filtered.dump +rm -f filteredvv.nc tst_filtervv.txt filteredvv.dump +rm -f filteredvbar.nc tst_filtervbar.txt filteredvbar.dump +rm -f tst_filter2.nc tst_filter2.txt +rm -f tst_none.nc tst_none.txt tst_none2.txt testfilter_reg.nc +rm -f tst_vnone.nc tst_vnone.txt tst_vnone2.txt +rm -f bzip2.nc bzip2.dump tst_filter.txt bzip2x.dump +rm -f test_bzip2.c +rm -f multifilter.nc multi.cdl smulti.cdl +rm -f nccopyF.nc nccopyF.cdl ncgenF.nc ncgenF.cdl +rm -f filterorder.txt +rm -f ncgenFs.cdl nccopyFs.cdl exit 0 + diff --git a/nc_test4/tst_filterparser.c b/nc_test4/tst_filterparser.c index 2176c480cf..e28dd2b815 100644 --- a/nc_test4/tst_filterparser.c +++ b/nc_test4/tst_filterparser.c @@ -115,7 +115,7 @@ params (BE): static unsigned int baseline[MAXPARAMS]; /* Expected */ static const char* spec = -"32768, -17b, 23ub, -25S, 27US, 77, 93U, 2147483647, -2147483648, 4294967295U, 789f, -9223372036854775807L, 18446744073709551615UL, 12345678.12345678d"; +"user-defined, -17b, 23ub, -25S, 27US, 77, 93U, 2147483647, -2147483648, 4294967295U, 789f, -9223372036854775807L, 18446744073709551615UL, 12345678.12345678d"; /* Define the type strings for each spec entry */ static const char* spectype[] = {"i", "b", "ub", "s", "us", "i", "ui", "i", "i", "ui", "f", "ll", "ull", "d"}; @@ -187,67 +187,67 @@ int main(int argc, char **argv) { int stat = 0; - unsigned int id = 0; - size_t i,nparams = 0; - unsigned int* params = NULL; + size_t i; /* Provide for 8-byte values */ long long basell; unsigned long long baseull; float basef; double based; + NC4_Filterspec* pfs = NULL; printf("\nTesting filter parser.\n"); buildbaseline(); /* Build our comparison vector */ - stat = NC_parsefilterspec(spec,&id,&nparams,¶ms); + stat = NC_parsefilterspec(spec,NC_FILTER_FORMAT_HDF5,(NC_Filterspec**)&pfs); if(stat) { fprintf(stderr,"NC_parsefilterspec failed\n"); exit(1); } - if(id != PARAMS_ID) - fprintf(stderr,"mismatch: id: expected=%u actual=%u\n",PARAMS_ID,id); + + if(pfs->filterid != PARAMS_ID) + fprintf(stderr,"mismatch: id: expected=%u actual=%u\n",(unsigned int)PARAMS_ID,pfs->filterid); /* Do all the 32 bit tests */ for(i=0;i<=8;i++) { - if(baseline[i] != params[i]) - mismatch(i,params,spectype[i]); + if(baseline[i] != pfs->params[i]) + mismatch(i,pfs->params,spectype[i]); } /* float */ - uf.ui = params[9]; + uf.ui = pfs->params[9]; memcpy(&basef,&baseline[9],4); if(uf.f != basef) - mismatch(9,params,"uf.f"); + mismatch(9,pfs->params,"uf.f"); /* signed long long */ - ul.ui[0] = params[10]; - ul.ui[1] = params[11]; - NC_filterfix8((unsigned char*)&ul.ll,1); + ul.ui[0] = pfs->params[10]; + ul.ui[1] = pfs->params[11]; + NC4_filterfix8((unsigned char*)&ul.ll,1); memcpy(&basell,&baseline[10],8); if(ul.ll != basell) - mismatch2(10,params,"ul.ll"); + mismatch2(10,pfs->params,"ul.ll"); /* unsigned long long */ - ul.ui[0] = params[12]; - ul.ui[1] = params[13]; - NC_filterfix8((unsigned char*)&ul.ull,1); + ul.ui[0] = pfs->params[12]; + ul.ui[1] = pfs->params[13]; + NC4_filterfix8((unsigned char*)&ul.ull,1); memcpy(&baseull,&baseline[12],8); if(ul.ull != baseull) - mismatch2(12,params,"ul.ull"); + mismatch2(12,pfs->params,"ul.ull"); /* double */ - ud.ui[0] = params[14]; - ud.ui[1] = params[15]; - NC_filterfix8((unsigned char*)&ud.d,1); + ud.ui[0] = pfs->params[14]; + ud.ui[1] = pfs->params[15]; + NC4_filterfix8((unsigned char*)&ud.d,1); memcpy(&based,&baseline[14],8); if(ud.d != based) - mismatch2(14,params,"ud.d"); + mismatch2(14,pfs->params,"ud.d"); if (!nerrs) printf("SUCCESS!!\n"); - if(params) free(params); + if(pfs->params) free(pfs->params); return (nerrs > 0 ? 1 : 0); } @@ -284,151 +284,6 @@ gettype(const int q0, const int q1, int* isunsignedp) return type; } -static int -parsefilterspec(const char* spec, unsigned int* idp, size_t* nparamsp, unsigned int** paramsp) -{ - int stat = NC_NOERR; - int sstat; /* for scanf */ - char* p; - char* sdata = NULL; - unsigned int id; - size_t count; /* no. of comma delimited params */ - size_t nparams; /* final no. of unsigned ints */ - size_t len; - int i; - unsigned int* ulist = NULL; - unsigned char mem[8]; - - if(spec == NULL || strlen(spec) == 0) goto fail; - sdata = strdup(spec); - - /* Count number of parameters + id and delimit */ - p=sdata; - for(count=0;;count++) { - char* q = strchr(p,','); - if(q == NULL) break; - *q++ = '\0'; - p = q; - } - count++; /* for final piece */ - - if(count == 0) - goto fail; /* no id and no parameters */ - - /* Extract the filter id */ - p = sdata; - sstat = sscanf(p,"%u",&id); - if(sstat != 1) goto fail; - /* skip past the filter id */ - p = p + strlen(p) + 1; - count--; - - /* Allocate the max needed space; *2 in case the params are all doubles */ - ulist = (unsigned int*)malloc(sizeof(unsigned int)*(count)*2); - if(ulist == NULL) goto fail; - - /* walk and convert */ - nparams = 0; /* actual count */ - for(i=0;i 2 => we might have a two letter tag */ - q = (p + len) - 2; - type = gettype(*q,*(q+1),&isunsigned); - break; - } - - /* Now parse */ - switch (type) { - case 'b': - case 's': - case 'i': - /* special case for a positive integer;for back compatibility.*/ - if(!isnegative) - sstat = sscanf(p,"%u",&val32u); - else - sstat = sscanf(p,"%d",(int*)&val32u); - if(sstat != 1) goto fail; - switch(type) { - case 'b': val32u = (val32u & 0xFF); break; - case 's': val32u = (val32u & 0xFFFF); break; - } - ulist[nparams++] = val32u; - break; - - case 'f': - sstat = sscanf(p,"%lf",&vald); - if(sstat != 1) goto fail; - valf = (float)vald; - ulist[nparams++] = *(unsigned int*)&valf; - break; - - /* The following are 8-byte values, so we must swap pieces if this - is a little endian machine */ - case 'd': - sstat = sscanf(p,"%lf",&vald); - if(sstat != 1) goto fail; - memcpy(mem,&vald,sizeof(mem)); - NC_filterfix8(mem,0); - vector = (unsigned int*)mem; - ulist[nparams++] = vector[0]; - ulist[nparams++] = vector[1]; - break; - case 'l': /* long long */ - if(isunsigned) - sstat = sscanf(p,"%llu",&val64u); - else - sstat = sscanf(p,"%lld",(long long*)&val64u); - if(sstat != 1) goto fail; - memcpy(mem,&val64u,sizeof(mem)); - NC_filterfix8(mem,0); - vector = (unsigned int*)&mem; - ulist[nparams++] = vector[0]; - ulist[nparams++] = vector[1]; - break; - default: - goto fail; - } - p = p + strlen(p) + 1; /* move to next param */ - } - /* Now return results */ - if(idp) *idp = id; - if(nparamsp) *nparamsp = nparams; - if(paramsp) { - *paramsp = ulist; - ulist = NULL; /* avoid duplicate free */ - } -done: - if(sdata) free(sdata); - if(ulist) free(ulist); - return stat; -fail: - stat = NC_EFILTER; - goto done; -} - #ifdef WORDS_BIGENDIAN /* Byte swap an 8-byte integer in place */ static void @@ -464,8 +319,9 @@ byteswap4(unsigned char* mem) #endif +#if 0 static void -NC_filterfix8(unsigned char* mem, int decode) +NC4_filterfix8(unsigned char* mem, int decode) { #ifdef WORDS_BIGENDIAN if(decode) { /* Apply inverse of the encode case */ @@ -481,6 +337,7 @@ NC_filterfix8(unsigned char* mem, int decode) /* No action is necessary */ #endif } +#endif #endif /*DEBUG*/ diff --git a/nc_test4/tst_multifilter.c b/nc_test4/tst_multifilter.c new file mode 100644 index 0000000000..92f8a4f71b --- /dev/null +++ b/nc_test4/tst_multifilter.c @@ -0,0 +1,316 @@ +/* + Copyright 2018, UCAR/Unidata + See COPYRIGHT file for copying and redistribution conditions. +*/ + +/*! \file +Test support for multiple filters per variable +*/ + +#include "config.h" +#include +#include +#include + +#include +#include "netcdf.h" +#include "netcdf_filter.h" + +/* The HDF assigned id for bzip compression */ +#define BZIP2_ID 307 +/* The compression level used in this example */ +#define BZIP2_LEVEL 9 + +#define DEFLATE_LEVEL 2 + +#define NOOP_ID 40000 + +#define NFILTERS 3 + +#define TESTFILE "multifilter.nc" + +/* Point at which we give up */ +#define MAXERRS 8 + +#define NDIMS 4 +#define DIMSIZE 4 +#define CHUNKSIZE 4 /* Note: not the total size of the chunk, but size wrt a dim*/ + +static size_t dimsize = DIMSIZE; +static size_t chunksize = CHUNKSIZE; +static size_t actualdims = NDIMS; + +static size_t actualproduct = 1; /* x-product over dim sizes */ +static size_t chunkproduct = 1; /* x-product over chunksizes */ + +static size_t dims[NDIMS]; +static size_t chunks[NDIMS]; + +static int nerrs = 0; + +static int ncid, varid; +static int dimids[NDIMS]; +static float* array = NULL; +static float* expected = NULL; + +/* Forward */ +static void init(int argc, char** argv); +static int test_multi(void); +static int verifychunks(void); + +#define ERRR do { \ +fflush(stdout); /* Make sure our stdout is synced with stderr. */ \ +fprintf(stderr, "Sorry! Unexpected result, %s, line: %d\n", \ + __FILE__, __LINE__); \ +nerrs++;\ +} while (0) + +static int +check(int err,int line) +{ + if(err != NC_NOERR) { + fprintf(stderr,"fail (%d): %s\n",line,nc_strerror(err)); + fflush(stderr); + exit(1); + } + return NC_NOERR; +} + +#define CHECK(x) check(x,__LINE__) + +/* +Read the chunking information about the variable +and verify that it is as expected. +*/ + +static int +verifychunks(void) +{ + int i; + int store = -1; + size_t chunksizes[NDIMS]; + memset(chunksizes,0,sizeof(chunksizes)); + CHECK(nc_inq_var_chunking(ncid, varid, &store, chunksizes)); + /* Storate must be chunked, not contiguous */ + if(store != NC_CHUNKED) { + fprintf(stderr,"bad chunk store\n"); + return NC_ESTORAGE; + } + /* Chunk sizes must match our predefined set */ + for(i=0;i= MAXERRS) + break; + } + } + if(errs == 0) + printf("no data errors\n"); + if(actualproduct <= 1) + return NC_EBADDIM; + return (errs == 0 ? NC_NOERR: NC_EINVAL); +} + +int +verifyfilters(int ncid, int varid) +{ + size_t nfilters, nparams; + unsigned int filterids[NFILTERS]; + unsigned int params[2]; + + /* Read back the compression info and verify it */ + CHECK(nc_inq_var_filterids(ncid,varid,&nfilters,filterids)); + if(nfilters != NFILTERS) { + fprintf(stderr,"Fail: nfilters mismatch: expected=%d actual=%u\n",NFILTERS,(unsigned)nfilters); + return NC_EINVAL; + } + if(filterids[0] != BZIP2_ID + || filterids[1] != H5Z_FILTER_DEFLATE + || filterids[2] != NOOP_ID + ) { + fprintf(stderr,"Fail: filter id mismatch: actual/expected={%u/%u,%u/%u,%u/%u}\n", + filterids[0],BZIP2_ID, + filterids[1],H5Z_FILTER_DEFLATE, + filterids[2],NOOP_ID); + return NC_EINVAL; + } + /* Get level for each filter */ + CHECK(nc_inq_var_filter_info(ncid,varid,BZIP2_ID,&nparams,params)); + if(nparams != 1) { + fprintf(stderr,"Fail: nparams mismatch: id=%u expected=1 actual=%u\n",filterids[0],(unsigned)nparams); + return NC_EINVAL; + } + if(params[0] != BZIP2_LEVEL) { + fprintf(stderr,"Fail: parameter mismatch: expected=%u actual=%u\n",BZIP2_LEVEL,params[0]); + return NC_EINVAL; + } + CHECK(nc_inq_var_filter_info(ncid,varid,H5Z_FILTER_DEFLATE,&nparams,params)); + if(nparams != 1) { + fprintf(stderr,"Fail: nparams mismatch: id=%u expected=1 actual=%u\n",filterids[1],(unsigned)nparams); + return NC_EINVAL; + } + if(params[0] != DEFLATE_LEVEL) { + fprintf(stderr,"Fail: parameter mismatch: expected=%u actual=%u\n",BZIP2_LEVEL,params[0]); + return NC_EINVAL; + } + CHECK(nc_inq_var_filter_info(ncid,varid,NOOP_ID,&nparams,params)); + if(nparams != 0) { + fprintf(stderr,"Fail: parameter mismatch: id=%u nparams: expected=0 actual=%u\n",NOOP_ID,(unsigned)nparams); + return NC_EINVAL; + } + return NC_NOERR; +} + +/* +Create the file, write it, then re-read for comparison. +*/ +static int +test_multi(void) +{ + int i; + unsigned int params[2]; + + printf("\n*** Testing Multi-filter application: filter set = bzip2 deflate noop"); + printf("\n"); + + /* Clear the data array */ + memset(array,0,sizeof(float)*actualproduct); + + /* Create a file */ + CHECK(nc_create(TESTFILE, NC_NETCDF4|NC_CLOBBER, &ncid)); + + /* Do not use fill for this file */ + CHECK(nc_set_fill(ncid, NC_NOFILL, NULL)); + + /* Define the dimensions */ + for(i=0;i 0?1:0); +} + diff --git a/nc_test4/tst_vars3.c b/nc_test4/tst_vars3.c index b2ee6921ea..634bc3a0e5 100644 --- a/nc_test4/tst_vars3.c +++ b/nc_test4/tst_vars3.c @@ -9,6 +9,12 @@ #include #include "err_macros.h" #include "netcdf.h" +#include "netcdf_filter.h" + +#define NC_SZIP_EC_BPP_IN 32 /**< @internal bits per pixel input. */ +#define NC_SZIP_EC_BPP_OUT 64 /**< @internal bits per pixel output. */ +#define NC_SZIP_NN 32 /**< @internal SZIP NN option mask. */ +#define NC_SZIP_EC 4 /**< @internal SZIP EC option mask. */ #define FILE_NAME "tst_vars3.nc" #define NDIMS1 1 @@ -29,8 +35,6 @@ #define VAR_NAME "var1" #define NUM_PARAMS_IN 2 #define NUM_PARAMS_OUT 4 -#define NC_SZIP_EC_BPP_IN 32 /**< @internal bits per pixel input. */ -#define NC_SZIP_EC_BPP_OUT 64 /**< @internal bits per pixel output. */ int main(int argc, char **argv) @@ -454,10 +458,11 @@ main(int argc, char **argv) /* Also check using nc_inq_var_filter */ if (nc_inq_var_filter(ncid, varid, &filterid, &nparams, params_out)) ERR; - if (filterid != H5_FILTER_SZIP || nparams != 4) ERR; + if (filterid != H5_FILTER_SZIP || nparams != 2) ERR; /* According to H5Zszip, the mapping should be as follows */ - if(params_out[0] != options_mask) ERR; - if(params[1] != pixels_per_block) ERR; + if (!(options_mask & NC_SZIP_NN)) ERR; + if (pixels_per_block != NC_SZIP_EC_BPP_IN && pixels_per_block != NC_SZIP_EC_BPP_OUT) + ERR; if (nc_close(ncid)) ERR; } SUMMARIZE_ERR; @@ -504,7 +509,7 @@ main(int argc, char **argv) /* Also check using nc_inq_var_filter */ if (nc_inq_var_filter(ncid, varid, &filterid, &nparams, params_out)) ERR; - if (filterid != H5_FILTER_SZIP || nparams != 4) ERR; + if (filterid != H5_FILTER_SZIP || nparams != 2) ERR; /* According to H5Zszip, the mapping should be as follows */ if(params_out[0] != options_mask) ERR; if(params_out[1] != pixels_per_block) ERR; @@ -523,8 +528,7 @@ main(int argc, char **argv) #define V_MEDIUM "medium_var" #define V_LARGE "large_var" #define NUM_PARAMS 2 -#define NC_SZIP_NN 32 /**< @internal SZIP NN option mask. */ -#define NC_SZIP_EC 4 /**< @internal SZIP EC option mask. */ + int ncid; int nvars, ndims, ngatts, unlimdimid; int ndims_in, natts_in, dimids_in; @@ -696,7 +700,8 @@ main(int argc, char **argv) /* Open the file and check. */ if (nc_open(FILE_NAME, NC_WRITE, &ncid)) ERR; - if (nc_inq_var_szip(ncid, varid, &option_mask_in, &pixels_per_block_in)) ERR; + if (nc_inq_var_szip(ncid, varid, &option_mask_in, &pixels_per_block_in)) + ERR; if (p < FIRST_VALID) { if (option_mask_in) ERR; @@ -707,13 +712,13 @@ main(int argc, char **argv) } /* Also check using nc_inq_var_filter */ - if (nc_inq_var_filter(ncid, varid, &filterid, &nparams, params_in)) ERR; - if (p < FIRST_VALID) - { + ret = nc_inq_var_filter(ncid, varid, &filterid, &nparams, params_in); + if (p < FIRST_VALID) { + if(ret != NC_ENOFILTER) ERR; } else { - if (filterid != H5_FILTER_SZIP || nparams != 4) ERR; + if (filterid != H5_FILTER_SZIP || nparams != 2) ERR; /* According to H5Zszip, the mapping should be as follows */ if(params_in[0] != option_mask_in) ERR; if(params_in[1] != pixels_per_block_in) ERR; @@ -745,7 +750,7 @@ main(int argc, char **argv) params[1] = NC_SZIP_EC_BPP_IN; /* pixels_per_block */ if (nc_def_var_chunking(ncid, varid, NC_CHUNKED, NULL)) ERR; if (nc_def_var_filter(ncid, varid, H5_FILTER_SZIP, NUM_PARAMS_IN, - params) != NC_EFILTER) ERR; + params) != NC_EFILTER) ERR; if (nc_def_var_szip(ncid, varid, NC_SZIP_NN, NC_SZIP_EC_BPP_IN) != NC_EFILTER) ERR; if (nc_close(ncid)) ERR; diff --git a/ncdump/nccopy.c b/ncdump/nccopy.c index d1c005d41e..b433e43920 100644 --- a/ncdump/nccopy.c +++ b/ncdump/nccopy.c @@ -63,16 +63,19 @@ typedef struct VarID { #define MAX_FILTER_PARAMS 256 struct FilterSpec { - char* fqn; + char* fqn; /* Of variable */ int nofilter; /* 1=> do not apply any filters to this variable */ - unsigned int filterid; - size_t nparams; - unsigned int* params; + NC4_Filterspec pfs; }; static List* filterspecs = NULL; static int suppressfilters = 0; /* 1 => do not apply any output filters unless specified */ +/* Forward declaration, because copy_type, copy_vlen_type call each other */ +static int copy_type(int igrp, nc_type typeid, int ogrp); +static void freespeclist(List* specs); +static void freefilterlist(size_t,NC4_Filterspec**); + #endif /* table of formats for legal -k values */ @@ -259,10 +262,10 @@ parsevarlist(char* vars, List* vlist) goto done; } - /* Walk delimitng on '|' separators */ + /* Walk delimitng on '&' separators */ for(q=vars;*q;q++) { if(*q == '\\') q++; - else if(*q == '|') {*q = '\0'; nvars++;} + else if(*q == '&') {*q = '\0'; nvars++;} /* else continue */ } nvars++; /*for last var*/ @@ -281,14 +284,13 @@ parsefilterspec(const char* optarg0, List* speclist) { int stat = NC_NOERR; char* optarg = NULL; - unsigned int* params = NULL; - size_t nparams; - unsigned int id; char* p = NULL; char* remainder = NULL; List* vlist = NULL; int i; int isnone = 0; + size_t nfilters = 0; + NC4_Filterspec** filters = NULL; if(optarg0 == NULL || strlen(optarg0) == 0 || speclist == NULL) return 0; optarg = strdup(optarg0); @@ -301,74 +303,101 @@ parsefilterspec(const char* optarg0, List* speclist) else if(*p == '\\') p++; /* else continue */ } - /* Parse the variable list */ if((vlist = listnew()) == NULL) {stat = NC_ENOMEM; goto done;} if((stat=parsevarlist(optarg,vlist))) goto done; if(strcasecmp(remainder,"none") != 0) { + int format; /* Collect the id+parameters */ - if((stat=NC_parsefilterspec(remainder,&id,&nparams,¶ms))) goto done; - } else + if((stat=NC_parsefilterlist(remainder,&format,&nfilters,(NC_Filterspec***)&filters))) goto done; + if(format != NC_FILTER_FORMAT_HDF5) + {stat = NC_EFILTER; goto done;} + } else { isnone = 1; + if(nfilters == 0) { + /* Add a fake filter */ + NC4_Filterspec* nilspec = (NC4_Filterspec*)calloc(1,sizeof(NC4_Filterspec)); + if(nilspec == NULL) {stat = NC_ENOMEM; goto done;} + nfilters = 1; + filters = calloc(1,sizeof(NC4_Filterspec**)); + if(filters == NULL) {free(nilspec); stat = NC_ENOMEM; goto done;} + filters[0] = nilspec; nilspec = NULL; + } + } /* Construct a spec entry for each element in vlist */ for(i=0;ifqn = malloc(vlen+1+1); /* make room for nul and possible prefix '/' */ - if(spec->fqn == NULL) {stat = NC_ENOMEM; goto done;} - spec->fqn[0] = '\0'; /* for strlcat */ - if(strcmp(var,"*") != 0 && var[0] != '/') strlcat(spec->fqn,"/",vlen+2); - strlcat(spec->fqn,var,vlen+2); - if(isnone) - spec->nofilter = 1; - else { - spec->filterid = id; - spec->nparams = nparams; - /* Duplicate the params */ - spec->params = malloc(nparams*sizeof(unsigned int)); - if(spec->params == NULL) {stat = NC_ENOMEM; goto done;} - memcpy(spec->params,params,nparams*sizeof(unsigned int)); + for(k=0;kfqn = malloc(vlen+1+1); /* make room for nul and possible prefix '/' */ + if(spec->fqn == NULL) {stat = NC_ENOMEM; goto done;} + spec->fqn[0] = '\0'; /* for strlcat */ + if(strcmp(var,"*") != 0 && var[0] != '/') strlcat(spec->fqn,"/",vlen+2); + strlcat(spec->fqn,var,vlen+2); + if(isnone) + spec->nofilter = 1; + else { + spec->pfs = *nsf; + if(nsf->nparams != 0) { + /* Duplicate the params */ + spec->pfs.params = malloc(spec->pfs.nparams*sizeof(unsigned int)); + if(spec->pfs.params == NULL) {stat = NC_ENOMEM; goto done;} + memcpy(spec->pfs.params,nsf->params,spec->pfs.nparams*sizeof(unsigned int)); + } else + spec->pfs.params = NULL; + } + listpush(speclist,spec); + spec = NULL; } - listpush(speclist,spec); - spec = NULL; } - + done: - if(params) free(params); + freefilterlist(nfilters,filters); if(vlist) listfreeall(vlist); if(optarg) free(optarg); return stat; } -static struct FilterSpec* -filterspecforvar(const char* ofqn) +static int +hasfilterspecsforvar(const char* ofqn) { int i; - struct FilterSpec* star = NULL; - struct FilterSpec* match = NULL; - /* See if any output filter spec is defined for this output variable */ - /* Name specific overrides '*' */ + /* See which output filter specs are defined for this output variable */ for(i=0;ifqn,"*")==0) - star = spec; /* save */ - if(strcmp(spec->fqn,ofqn)==0) { - match = spec; - break;; - } + if(strcmp(spec->fqn,"*")==0) return 1; + if(strcmp(spec->fqn,ofqn)==0) return 1; } - if(match) return match; - if(star) return star; - return NULL; + return 0; } +static List* +filterspecsforvar(const char* ofqn) +{ + int i; + List* list = listnew(); + /* See which output filter specs are defined for this output variable */ + for(i=0;ifqn,"*")==0) { + /* Add to the list */ + listpush(list,spec); + } else if(strcmp(spec->fqn,ofqn)==0) { + /* Add to the list */ + listpush(list,spec); + } + } + return list; +} /* Return size of chunk in bytes for a variable varid in a group igrp, or 0 if * layout is contiguous */ @@ -481,9 +510,6 @@ inq_var_chunking_params(int igrp, int ivarid, int ogrp, int ovarid, return stat; } -/* Forward declaration, because copy_type, copy_vlen_type call each other */ -static int copy_type(int igrp, nc_type typeid, int ogrp); - /* * copy a user-defined variable length type in the group igrp to the * group ogrp @@ -739,10 +765,11 @@ copy_var_filter(int igrp, int varid, int ogrp, int o_varid, int inkind, int outk VarID vid = {igrp,varid}; VarID ovid = {ogrp,o_varid}; /* handle filter parameters, copying from input, overriding with command-line options */ - struct FilterSpec* ospec = NULL; + List* ospecs = NULL; + List* inspecs = NULL; + List* actualspecs = NULL; struct FilterSpec inspec; - struct FilterSpec nospec; - struct FilterSpec* actualspec = NULL; + struct FilterSpec* tmp = NULL; char* ofqn = NULL; int inputdefined, outputdefined, unfiltered; int innc4 = (inkind == NC_FORMAT_NETCDF4 || inkind == NC_FORMAT_NETCDF4_CLASSIC); @@ -755,81 +782,112 @@ copy_var_filter(int igrp, int varid, int ogrp, int o_varid, int inkind, int outk if((stat = computeFQN(ovid,&ofqn))) goto done; /* Clear the in and out specs */ - memset(&inspec,0,sizeof(inspec)); - memset(&nospec,0,sizeof(nospec)); - nospec.nofilter = 1; - actualspec = NULL; - ospec = NULL; + inspecs = listnew(); + ospecs = NULL; + actualspecs = NULL; - /* Is there a filter on the output variable */ + /* Is there one or more filters on the output variable */ outputdefined = 0; /* default is no filter defined */ /* Only bother to look if output is netcdf-4 variant */ if(outnc4) { /* See if any output filter spec is defined for this output variable */ - ospec = filterspecforvar(ofqn); - if(ospec != NULL) + ospecs = filterspecsforvar(ofqn); + if(listlength(ospecs) > 0) outputdefined = 1; } - /* Is there a filter on the input variable */ + /* Is there already a filter on the input variable */ inputdefined = 0; /* default is no filter defined */ /* Only bother to look if input is netcdf-4 variant */ if(innc4) { - stat=nc_inq_var_filter(vid.grpid,vid.varid,&inspec.filterid,&inspec.nparams,NULL); - if(stat && stat != NC_EFILTER) + size_t nfilters; + unsigned int* ids = NULL; + int k; + if((stat = nc_inq_var_filterids(vid.grpid,vid.varid,&nfilters,NULL))) + goto done; + if(nfilters > 0) ids = malloc(nfilters*sizeof(unsigned int)); + if((stat = nc_inq_var_filterids(vid.grpid,vid.varid,&nfilters,ids))) + goto done; + memset(&inspec,0,sizeof(inspec)); + for(k=0;k 0) {/* input has a filter */ - inspec.params = (unsigned int*)malloc(sizeof(unsigned int)*inspec.nparams); - if((stat=nc_inq_var_filter(vid.grpid,vid.varid,&inspec.filterid,&inspec.nparams,inspec.params))) - goto done; - inputdefined = 1; - } + if(inspec.pfs.nparams > 0) { + inspec.pfs.params = (unsigned int*)malloc(sizeof(unsigned int)*inspec.pfs.nparams); + if((stat=nc_inq_var_filter_info(vid.grpid,vid.varid,inspec.pfs.filterid,NULL,inspec.pfs.params))) + goto done; + } + tmp = malloc(sizeof(struct FilterSpec)); + *tmp = inspec; + memset(&inspec,0,sizeof(inspec)); /*reset*/ + listpush(inspecs,tmp); + inputdefined = 1; + } + nullfree(ids); } /* Rules for choosing output filter are as follows: - global output input Actual Output - suppress filter filter filter + global output input Actual Output + suppress filter(s) filter(s) filter ----------------------------------------------------------- 1 true undefined NA unfiltered 2 true 'none' NA unfiltered - 3 true defined NA use output filter - 4 false undefined defined use input filter + 3 true defined NA use output filter(s) + 4 false undefined defined use input filter(s) 5 false 'none' NA unfiltered - 6 false defined NA use output filter + 6 false defined NA use output filter(s) 7 false undefined undefined unfiltered + 8 false defined defined use output filter(s) */ unfiltered = 0; - if(suppressfilters && !outputdefined) /* row 1 */ unfiltered = 1; - else if(suppressfilters && outputdefined && ospec->nofilter) /* row 2 */ - unfiltered = 1; - else if(suppressfilters && outputdefined) /* row 3 */ - actualspec = ospec; + else if(suppressfilters && outputdefined) { /* row 2 */ + int k; + /* Walk the set of filters to apply to see if "none" was specified */ + for(k=0;knofilter) {unfiltered = 1; break;} + } + } else if(suppressfilters && outputdefined) /* row 3 */ + actualspecs = ospecs; else if(!suppressfilters && !outputdefined && inputdefined) /* row 4 */ - actualspec = &inspec; - else if(!suppressfilters && outputdefined && ospec->nofilter) /* row 5 */ - unfiltered = 1; - else if(!suppressfilters && outputdefined) /* row 6 */ - actualspec = ospec; - else if(!suppressfilters && !outputdefined && !inputdefined) /* row 7 */ + actualspecs = inspecs; + else if(!suppressfilters && outputdefined) { /* row 5 & 6*/ + int k; + /* Walk the set of filters to apply to see if "none" was specified */ + for(k=0;knofilter) {unfiltered = 1; break;} + } + if(!unfiltered) actualspecs = ospecs; + } else if(!suppressfilters && !outputdefined && !inputdefined) /* row 7 */ unfiltered = 1; + else if(!suppressfilters && outputdefined && inputdefined) /* row 8 */ + actualspecs = ospecs; /* Apply actual filter spec if any */ if(!unfiltered) { - if((stat=nc_def_var_filter(ovid.grpid,ovid.varid, - actualspec->filterid, - actualspec->nparams, - actualspec->params))) + /* add all the actual filters */ + int k; + for(k=0;kpfs.filterid, + actual->pfs.nparams, + actual->pfs.params))) goto done; + } } done: /* Cleanup */ if(ofqn != NULL) free(ofqn); - if(inspec.fqn) free(inspec.fqn); - if(inspec.params) free(inspec.params); + freespeclist(inspecs); inspecs = NULL; + listfree(ospecs); ospecs = NULL; /* Contents are also in filterspecs */ /* Note we do not clean actualspec because it is a copy of in|out spec */ return stat; } @@ -976,7 +1034,7 @@ copy_chunking(int igrp, int i_varid, int ogrp, int o_varid, int ndims, int inkin ovid.grpid = ogrp; ovid.varid = o_varid; if((stat=computeFQN(ovid,&ofqn))) goto done; - if(option_deflate_level >= 0 || filterspecforvar(ofqn) != NULL) + if(option_deflate_level >= 0 || hasfilterspecsforvar(ofqn)) ocontig = 0; /* Apply the chunking, if any */ @@ -2100,7 +2158,7 @@ usage(void) [-h n] set size in bytes of chunk_cache for chunked variables\n\ [-e n] set number of elements that chunk_cache can hold\n\ [-r] read whole input file into diskless file on open (classic or 64-bit offset or cdf5 formats only)\n\ - [-F filterspec] specify the compression algorithm to apply to an output variable.\n\ + [-F filterspec] specify a compression algorithm to apply to an output variable (may be repeated).\n\ [-Ln] set log level to n (>= 0); ignored if logging isn't enabled.\n\ [-Mn] set minimum chunk size to n bytes (n >= 0)\n\ infile name of netCDF input file\n\ @@ -2328,15 +2386,39 @@ main(int argc, char**argv) exitcode = EXIT_FAILURE; #ifdef USE_NETCDF4 - { int i; - /* Clean up */ - for(i=0;ifqn) free(spec->fqn); - if(spec->params) free(spec->params); - } - } + /* Clean up */ + freespeclist(filterspecs); + filterspecs = NULL; #endif /*USE_NETCDF4*/ exit(exitcode); } + +#ifdef USE_NETCDF4 +static void +freespeclist(List* specs) +{ + int i; + for(i=0;ifqn) free(spec->fqn); + if(spec->pfs.params) free(spec->pfs.params); + free(spec); + } + listfree(specs); +} + +static void +freefilterlist(size_t nfilters, NC4_Filterspec** filters) +{ + int i; + if(filters == NULL) return; + for(i=0;iparams) free(pfs->params); + free(pfs); + } + free(filters); +} +#endif + diff --git a/ncdump/ncdump.c b/ncdump/ncdump.c index 8d334c0a6f..bbf13d68a0 100644 --- a/ncdump/ncdump.c +++ b/ncdump/ncdump.c @@ -9,6 +9,7 @@ Research/Unidata. See \ref copyright file for more info. */ #include #endif #ifdef _MSC_VER /* Microsoft Compilers */ + #include #endif #ifdef HAVE_UNISTD_H @@ -36,6 +37,7 @@ int optind; #include "netcdf.h" #include "netcdf_mem.h" +#include "netcdf_filter.h" #include "netcdf_aux.h" #include "utils.h" #include "nccomps.h" @@ -48,7 +50,6 @@ int optind; #include "cdl.h" #include "nclog.h" #include "ncwinpath.h" -#include "netcdf_aux.h" #include "nclist.h" #include "nc_provenance.h" @@ -1009,15 +1010,62 @@ pr_att_specials( } } - /*_Deflate, _Shuffle */ + /* _Filter (including deflate and shuffle) */ { - int shuffle=NC_NOSHUFFLE, deflate=0, deflate_level=0; - NC_CHECK( nc_inq_var_deflate(ncid, varid, &shuffle, - &deflate, &deflate_level) ); - if(deflate != 0) { - pr_att_name(ncid, varp->name, NC_ATT_DEFLATE); - printf(" = %d ;\n", deflate_level); + size_t nparams, nfilters, nbytes; + int shuffle=NC_NOSHUFFLE; + unsigned int* filterids = NULL; + unsigned int* params = NULL; + int usedeflateatt = 0; + + /* Get applicable filter ids */ + NC_CHECK(nc_inq_var_filterids(ncid, varid, &nfilters, NULL)); + /* Get set of filters for this variable */ + if(nfilters > 0) { + filterids = (unsigned int*)malloc(sizeof(unsigned int)*nfilters); + if(filterids == NULL) NC_CHECK(NC_ENOMEM); + } else + filterids = NULL; + NC_CHECK(nc_inq_var_filterids(ncid, varid, &nfilters, filterids)); + if(nfilters > 0) { + int k; + int pratt = 0; + for(k=0;k 0) { + params = (unsigned int*)calloc(1,sizeof(unsigned int)*nparams); + NC_CHECK(nc_inq_var_filter_info(ncid, varid, filterids[k], &nbytes, params)); + } else + params = NULL; + /* Use _Deflate if the first filter is zip */ + if(k == 0 && filterids[k] == H5Z_FILTER_DEFLATE) { + pr_att_name(ncid, varp->name, NC_ATT_DEFLATE); + printf(" = %d", (int)params[0]); + pratt = 1; + usedeflateatt = 1; + nullfree(params); params = NULL; + continue; + } + if(pratt || k == 0) { + pr_att_name(ncid,varp->name,NC_ATT_FILTER); + printf(" = \""); + pratt = 0; + } + if(k > 0) printf("|"); + printf("%u",filterids[k]); + if(nparams > 0) { + int i; + for(i=0;iname, NC_ATT_SHUFFLE); printf(" = \"true\" ;\n"); @@ -1054,28 +1102,6 @@ pr_att_specials( printf(" ;\n"); } } - /* _Filter */ - { - unsigned int id; - size_t nparams; - unsigned int* params = NULL; - if(nc_inq_var_filter(ncid, varid, &id, &nparams, NULL) == NC_NOERR - && id > 0) { - if(nparams > 0) { - params = (unsigned int*)calloc(1,sizeof(unsigned int)*nparams); - NC_CHECK(nc_inq_var_filter(ncid, varid, &id, &nparams, params)); - } - pr_att_name(ncid,varp->name,NC_ATT_FILTER); - printf(" = \"%u",id); - if(nparams > 0) { - int i; - for(i=0;iflags & _FILTER_FLAG) { - /* Special check for alternate way to specify _Deflate */ - if(special->_FilterID == ZIP_ID) { - unsigned int level; - if(special->nparams == 0 || special->_FilterParams == NULL) - level = 9; /* default */ - else - level = special->_FilterParams[0]; - if(level > 9) - derror("Illegal deflate level"); - else { - stat = nc_def_var_deflate(var->container->nc_id, - var->nc_id, - (special->_Shuffle == 1?1:0), - (level > 0?1:0), - level); - } - } else { + int k; + for(k=0;knfilters;k++) { + NC4_Filterspec* nfs = special->_Filters[k]; stat = nc_def_var_filter(var->container->nc_id, var->nc_id, - special->_FilterID, - special->nparams, - special->_FilterParams + nfs->filterid, + nfs->nparams, + nfs->params ); } check_err(stat,__LINE__,"ncgen"); diff --git a/ncgen/genc.c b/ncgen/genc.c index 5afba81358..7655567ad9 100644 --- a/ncgen/genc.c +++ b/ncgen/genc.c @@ -113,21 +113,28 @@ genc_netcdf(void) codeline("} ;"); } if(special->flags & _FILTER_FLAG) { - int i; - unsigned int* params = special->_FilterParams; - if(special->nparams == 0 || params == NULL) continue; - bbClear(tmp); - for(i=0;inparams;i++) { - bbprintf(tmp,"%s%luU", + int k; + for(k=0;knfilters;k++) { + int i; + size_t nparams; + unsigned int* params; + NC4_Filterspec* nfs = (NC4_Filterspec*)special->_Filters[k]; + if(nfs->nparams == 0 || nfs->params == NULL) continue; + bbClear(tmp); + nparams = nfs->nparams; + params = nfs->params; + for(i=0;inparams); - codedump(stmt); - codedump(tmp); - codeline("} ;"); - } - bbFree(tmp); + } + bbFree(tmp); } codeline(""); } @@ -505,46 +512,27 @@ genc_definespecialattributes(Symbol* vsym) codelined(1,"check_err(stat,__LINE__,__FILE__);"); } if(special->flags & _FILTER_FLAG) { - /* Special check for alternate way to specify _Deflate */ - if(special->_FilterID == ZIP_ID) { - unsigned int level; - if(special->nparams == 0 || special->_FilterParams == NULL) - level = 9; /* default */ - else - level = special->_FilterParams[0]; - if(level > 9) - derror("Illegal deflate level"); - else { - bbprintf0(stmt, - " stat = nc_def_var_deflate(%s, %s, %s, %d, %d);\n", - groupncid(vsym->container), - varncid(vsym), - (special->_Shuffle == 1?"NC_SHUFFLE":"NC_NOSHUFFLE"), - 1, - level); - codedump(stmt); - } - } else { - bbprintf0(stmt, - " stat = nc_def_var_filter(%s, %s, %u, %lu, ", + int k; + for(k=0;knfilters;k++) { + NC4_Filterspec* nfs = (NC4_Filterspec*)special->_Filters[k]; + bbprintf0(stmt, + " stat = nc_def_var_filter(%s, %s, %u, %u, ", groupncid(vsym->container), varncid(vsym), - special->_FilterID, - special->nparams + nfs->filterid, + nfs->nparams ); + codedump(stmt); + if(nfs->nparams == 0 || nfs->params == NULL) + codepartial("NULL"); + else { + bbprintf0(stmt,"%s_%d_filterparams",cname(vsym),k); codedump(stmt); - if(special->nparams == 0 || special->_FilterParams == NULL) - codepartial("NULL"); - else { - bbprintf0(stmt,"%s_filterparams",cname(vsym)); - codedump(stmt); - } - codeline(");"); + } + codeline(");"); + codelined(1,"check_err(stat,__LINE__,__FILE__);"); } - codelined(1,"check_err(stat,__LINE__,__FILE__);"); } - - } #endif /*USE_NETCDF4*/ diff --git a/ncgen/includes.h b/ncgen/includes.h index 49a64a1af9..289051cd0b 100644 --- a/ncgen/includes.h +++ b/ncgen/includes.h @@ -40,15 +40,14 @@ #define ENABLE_JAVA #include "netcdf.h" +#include "netcdf_filter.h" +#include "nc4internal.h" #include "data.h" #include "ncgen.h" #include "genlib.h" #include "util.h" #include "debug.h" #include "nc.h" -#ifdef USE_NETCDF4 -#include "nc4internal.h" -#endif extern int specialconstants; diff --git a/ncgen/ncgen.h b/ncgen/ncgen.h index d7e9aefba1..4683139b1c 100644 --- a/ncgen/ncgen.h +++ b/ncgen/ncgen.h @@ -6,7 +6,7 @@ * $Header: /upc/share/CVS/netcdf-3/ncgen/ncgen.h,v 1.18 2010/06/01 15:34:53 ed Exp $ *********************************************************************/ -#ifdef _MSC_VER +#ifdef _WIN32 #include #include "../ncdump/isnan.h" #define strcasecmp _stricmp @@ -117,11 +117,6 @@ int k_flag; extern struct Kvalues legalkinds[]; -struct FilterID { -char* name; -unsigned int id; -}; - #define ZIP_ID 0xFFFFFFFF #define SZIP_ID 0xFFFFFFFE #define BZIP2_ID 307U @@ -140,9 +135,8 @@ typedef struct Specialdata { int _Shuffle; /* 0 => false, 1 => true*/ int _Endianness; /* 1 =>little, 2 => big*/ int _Fill ; /* 0 => false, 1 => true WATCHOUT: this is inverse of NOFILL*/ - unsigned int _FilterID; - size_t nparams; /* |_FilterParms| ; 0 => not specified*/ - unsigned int* _FilterParams; /* NULL => defaults*/ + NC4_Filterspec** _Filters; + size_t nfilters; /* |filters| */ } Specialdata; typedef struct GlobalSpecialdata { diff --git a/ncgen/ncgen.y b/ncgen/ncgen.y index 9d5652e8b2..d6fe826538 100644 --- a/ncgen/ncgen.y +++ b/ncgen/ncgen.y @@ -129,6 +129,9 @@ static void vercheck(int ncid); static long long extractint(NCConstant* con); #ifdef USE_NETCDF4 static int parsefilterflag(const char* sdata0, Specialdata* special); +#ifdef GENDEBUG1 +static void printfilters(size_t nfilters, NC_ParsedFilterSpec** filters); +#endif #endif int yylex(void); @@ -1363,7 +1366,6 @@ makespecial(int tag, Symbol* vsym, Symbol* tsym, void* data, int isconst) if(parsefilterflag(sdata,special) == NC_NOERR) special->flags |= _FILTER_FLAG; else { - efree(special->_FilterParams); derror("_Filter: unparsable filter spec: %s",sdata); } #else @@ -1485,12 +1487,18 @@ static int parsefilterflag(const char* sdata, Specialdata* special) { int stat = NC_NOERR; + int format; if(sdata == NULL || strlen(sdata) == 0) return NC_EINVAL; - stat = NC_parsefilterspec(sdata, &special->_FilterID, &special->nparams, &special->_FilterParams); + stat = NC_parsefilterlist(sdata, &format, &special->nfilters, (NC_Filterspec***)&special->_Filters); if(stat) derror("Malformed filter spec: %s",sdata); + if(format != NC_FILTER_FORMAT_HDF5) + derror("Non HDF5 filter format encountered"); +#ifdef GENDEBUG1 +printfilters(special->nfilters,special->_Filters); +#endif return stat; } #endif @@ -1566,3 +1574,31 @@ evaluate(Symbol* fcn, Datalist* arglist) done: return result; } + +#ifdef GENDEBUG1 +static void +printfilters(size_t nfilters, NC_ParsedFilterSpec** filters) +{ + int i; + fprintf(stderr,"xxx: nfilters=%lu: ",(unsigned long)nfilters); + for(i=0;ifilterid,sp->format,(unsigned long)sp->nparams,sp->params); + if(sp->nparams == 0 || sp->params != NULL) { + fprintf(stderr," params={"); + for(k=0;knparams;k++) { + if(i==0) fprintf(stderr,","); + fprintf(stderr,"%u",sp->params[i]); + } + fprintf(stderr,"}"); + } else + fprintf(stderr,"params=NULL"); + fprintf(stderr,"}"); + } + fprintf(stderr,"\n"); + fflush(stderr); +} +#endif diff --git a/ncgen/ncgenl.c b/ncgen/ncgenl.c index bcea5c9b0e..cfbc2c020a 100644 --- a/ncgen/ncgenl.c +++ b/ncgen/ncgenl.c @@ -1,5 +1,5 @@ -#line 2 "ncgenl.c" +#line 3 "ncgenl.c" #define YY_INT_ALIGNED short int @@ -7,17 +7,11 @@ #define yy_create_buffer ncg_create_buffer #define yy_delete_buffer ncg_delete_buffer -#define yy_scan_buffer ncg_scan_buffer -#define yy_scan_string ncg_scan_string -#define yy_scan_bytes ncg_scan_bytes +#define yy_flex_debug ncg_flex_debug #define yy_init_buffer ncg_init_buffer #define yy_flush_buffer ncg_flush_buffer #define yy_load_buffer_state ncg_load_buffer_state #define yy_switch_to_buffer ncg_switch_to_buffer -#define yypush_buffer_state ncgpush_buffer_state -#define yypop_buffer_state ncgpop_buffer_state -#define yyensure_buffer_stack ncgensure_buffer_stack -#define yy_flex_debug ncg_flex_debug #define yyin ncgin #define yyleng ncgleng #define yylex ncglex @@ -33,245 +27,11 @@ #define FLEX_SCANNER #define YY_FLEX_MAJOR_VERSION 2 #define YY_FLEX_MINOR_VERSION 6 -#define YY_FLEX_SUBMINOR_VERSION 4 +#define YY_FLEX_SUBMINOR_VERSION 0 #if YY_FLEX_SUBMINOR_VERSION > 0 #define FLEX_BETA #endif -#ifdef yy_create_buffer -#define ncg_create_buffer_ALREADY_DEFINED -#else -#define yy_create_buffer ncg_create_buffer -#endif - -#ifdef yy_delete_buffer -#define ncg_delete_buffer_ALREADY_DEFINED -#else -#define yy_delete_buffer ncg_delete_buffer -#endif - -#ifdef yy_scan_buffer -#define ncg_scan_buffer_ALREADY_DEFINED -#else -#define yy_scan_buffer ncg_scan_buffer -#endif - -#ifdef yy_scan_string -#define ncg_scan_string_ALREADY_DEFINED -#else -#define yy_scan_string ncg_scan_string -#endif - -#ifdef yy_scan_bytes -#define ncg_scan_bytes_ALREADY_DEFINED -#else -#define yy_scan_bytes ncg_scan_bytes -#endif - -#ifdef yy_init_buffer -#define ncg_init_buffer_ALREADY_DEFINED -#else -#define yy_init_buffer ncg_init_buffer -#endif - -#ifdef yy_flush_buffer -#define ncg_flush_buffer_ALREADY_DEFINED -#else -#define yy_flush_buffer ncg_flush_buffer -#endif - -#ifdef yy_load_buffer_state -#define ncg_load_buffer_state_ALREADY_DEFINED -#else -#define yy_load_buffer_state ncg_load_buffer_state -#endif - -#ifdef yy_switch_to_buffer -#define ncg_switch_to_buffer_ALREADY_DEFINED -#else -#define yy_switch_to_buffer ncg_switch_to_buffer -#endif - -#ifdef yypush_buffer_state -#define ncgpush_buffer_state_ALREADY_DEFINED -#else -#define yypush_buffer_state ncgpush_buffer_state -#endif - -#ifdef yypop_buffer_state -#define ncgpop_buffer_state_ALREADY_DEFINED -#else -#define yypop_buffer_state ncgpop_buffer_state -#endif - -#ifdef yyensure_buffer_stack -#define ncgensure_buffer_stack_ALREADY_DEFINED -#else -#define yyensure_buffer_stack ncgensure_buffer_stack -#endif - -#ifdef yylex -#define ncglex_ALREADY_DEFINED -#else -#define yylex ncglex -#endif - -#ifdef yyrestart -#define ncgrestart_ALREADY_DEFINED -#else -#define yyrestart ncgrestart -#endif - -#ifdef yylex_init -#define ncglex_init_ALREADY_DEFINED -#else -#define yylex_init ncglex_init -#endif - -#ifdef yylex_init_extra -#define ncglex_init_extra_ALREADY_DEFINED -#else -#define yylex_init_extra ncglex_init_extra -#endif - -#ifdef yylex_destroy -#define ncglex_destroy_ALREADY_DEFINED -#else -#define yylex_destroy ncglex_destroy -#endif - -#ifdef yyget_debug -#define ncgget_debug_ALREADY_DEFINED -#else -#define yyget_debug ncgget_debug -#endif - -#ifdef yyset_debug -#define ncgset_debug_ALREADY_DEFINED -#else -#define yyset_debug ncgset_debug -#endif - -#ifdef yyget_extra -#define ncgget_extra_ALREADY_DEFINED -#else -#define yyget_extra ncgget_extra -#endif - -#ifdef yyset_extra -#define ncgset_extra_ALREADY_DEFINED -#else -#define yyset_extra ncgset_extra -#endif - -#ifdef yyget_in -#define ncgget_in_ALREADY_DEFINED -#else -#define yyget_in ncgget_in -#endif - -#ifdef yyset_in -#define ncgset_in_ALREADY_DEFINED -#else -#define yyset_in ncgset_in -#endif - -#ifdef yyget_out -#define ncgget_out_ALREADY_DEFINED -#else -#define yyget_out ncgget_out -#endif - -#ifdef yyset_out -#define ncgset_out_ALREADY_DEFINED -#else -#define yyset_out ncgset_out -#endif - -#ifdef yyget_leng -#define ncgget_leng_ALREADY_DEFINED -#else -#define yyget_leng ncgget_leng -#endif - -#ifdef yyget_text -#define ncgget_text_ALREADY_DEFINED -#else -#define yyget_text ncgget_text -#endif - -#ifdef yyget_lineno -#define ncgget_lineno_ALREADY_DEFINED -#else -#define yyget_lineno ncgget_lineno -#endif - -#ifdef yyset_lineno -#define ncgset_lineno_ALREADY_DEFINED -#else -#define yyset_lineno ncgset_lineno -#endif - -#ifdef yywrap -#define ncgwrap_ALREADY_DEFINED -#else -#define yywrap ncgwrap -#endif - -#ifdef yyalloc -#define ncgalloc_ALREADY_DEFINED -#else -#define yyalloc ncgalloc -#endif - -#ifdef yyrealloc -#define ncgrealloc_ALREADY_DEFINED -#else -#define yyrealloc ncgrealloc -#endif - -#ifdef yyfree -#define ncgfree_ALREADY_DEFINED -#else -#define yyfree ncgfree -#endif - -#ifdef yytext -#define ncgtext_ALREADY_DEFINED -#else -#define yytext ncgtext -#endif - -#ifdef yyleng -#define ncgleng_ALREADY_DEFINED -#else -#define yyleng ncgleng -#endif - -#ifdef yyin -#define ncgin_ALREADY_DEFINED -#else -#define yyin ncgin -#endif - -#ifdef yyout -#define ncgout_ALREADY_DEFINED -#else -#define yyout ncgout -#endif - -#ifdef yy_flex_debug -#define ncg_flex_debug_ALREADY_DEFINED -#else -#define yy_flex_debug ncg_flex_debug -#endif - -#ifdef yylineno -#define ncglineno_ALREADY_DEFINED -#else -#define yylineno ncglineno -#endif - /* First, we deal with platform-specific or compiler-specific issues. */ /* begin standard C headers. */ @@ -342,48 +102,60 @@ typedef unsigned int flex_uint32_t; #define UINT32_MAX (4294967295U) #endif -#ifndef SIZE_MAX -#define SIZE_MAX (~(size_t)0) -#endif - #endif /* ! C99 */ #endif /* ! FLEXINT_H */ -/* begin standard C++ headers. */ +#ifdef __cplusplus -/* TODO: this is always defined, so inline it */ -#define yyconst const +/* The "const" storage-class-modifier is valid. */ +#define YY_USE_CONST -#if defined(__GNUC__) && __GNUC__ >= 3 -#define yynoreturn __attribute__((__noreturn__)) +#else /* ! __cplusplus */ + +/* C99 requires __STDC__ to be defined as 1. */ +#if defined (__STDC__) + +#define YY_USE_CONST + +#endif /* defined (__STDC__) */ +#endif /* ! __cplusplus */ + +#ifdef YY_USE_CONST +#define yyconst const #else -#define yynoreturn +#define yyconst #endif /* Returned upon end-of-file. */ #define YY_NULL 0 -/* Promotes a possibly negative, possibly signed char to an - * integer in range [0..255] for use as an array index. +/* Promotes a possibly negative, possibly signed char to an unsigned + * integer for use as an array index. If the signed char is negative, + * we want to instead treat it as an 8-bit unsigned char, hence the + * double cast. */ -#define YY_SC_TO_UI(c) ((YY_CHAR) (c)) +#define YY_SC_TO_UI(c) ((unsigned int) (unsigned char) c) /* Enter a start condition. This macro really ought to take a parameter, * but we do it the disgusting crufty way forced on us by the ()-less * definition of BEGIN. */ #define BEGIN (yy_start) = 1 + 2 * + /* Translate the current start state into a value that can be later handed * to BEGIN to return to the state. The YYSTATE alias is for lex * compatibility. */ #define YY_START (((yy_start) - 1) / 2) #define YYSTATE YY_START + /* Action number for EOF rule of a given start state. */ #define YY_STATE_EOF(state) (YY_END_OF_BUFFER + state + 1) + /* Special action meaning "start processing a new file". */ -#define YY_NEW_FILE yyrestart( yyin ) +#define YY_NEW_FILE ncgrestart(ncgin ) + #define YY_END_OF_BUFFER_CHAR 0 /* Size of default input buffer. */ @@ -413,14 +185,14 @@ typedef struct yy_buffer_state *YY_BUFFER_STATE; typedef size_t yy_size_t; #endif -extern int yyleng; +extern yy_size_t ncgleng; -extern FILE *yyin, *yyout; +extern FILE *ncgin, *ncgout; #define EOB_ACT_CONTINUE_SCAN 0 #define EOB_ACT_END_OF_FILE 1 #define EOB_ACT_LAST_MATCH 2 - + #define YY_LESS_LINENO(n) #define YY_LINENO_REWIND_TO(ptr) @@ -428,15 +200,16 @@ extern FILE *yyin, *yyout; #define yyless(n) \ do \ { \ - /* Undo effects of setting up yytext. */ \ + /* Undo effects of setting up ncgtext. */ \ int yyless_macro_arg = (n); \ YY_LESS_LINENO(yyless_macro_arg);\ *yy_cp = (yy_hold_char); \ YY_RESTORE_YY_MORE_OFFSET \ (yy_c_buf_p) = yy_cp = yy_bp + yyless_macro_arg - YY_MORE_ADJ; \ - YY_DO_BEFORE_ACTION; /* set up yytext again */ \ + YY_DO_BEFORE_ACTION; /* set up ncgtext again */ \ } \ while ( 0 ) + #define unput(c) yyunput( c, (yytext_ptr) ) #ifndef YY_STRUCT_YY_BUFFER_STATE @@ -451,7 +224,7 @@ struct yy_buffer_state /* Size of input buffer in bytes, not including room for EOB * characters. */ - int yy_buf_size; + yy_size_t yy_buf_size; /* Number of characters read into yy_ch_buf, not including EOB * characters. @@ -479,7 +252,7 @@ struct yy_buffer_state int yy_bs_lineno; /**< The line count. */ int yy_bs_column; /**< The column count. */ - + /* Whether to try to fill the input buffer when we reach the * end of it. */ @@ -496,8 +269,8 @@ struct yy_buffer_state * possible backing-up. * * When we actually see the EOF, we change the status to "new" - * (via yyrestart()), so that the user can continue scanning by - * just pointing yyin at a new input file. + * (via ncgrestart()), so that the user can continue scanning by + * just pointing ncgin at a new input file. */ #define YY_BUFFER_EOF_PENDING 2 @@ -507,7 +280,7 @@ struct yy_buffer_state /* Stack of input buffers. */ static size_t yy_buffer_stack_top = 0; /**< index of top of stack. */ static size_t yy_buffer_stack_max = 0; /**< capacity of stack. */ -static YY_BUFFER_STATE * yy_buffer_stack = NULL; /**< Stack as an array. */ +static YY_BUFFER_STATE * yy_buffer_stack = 0; /**< Stack as an array. */ /* We provide macros for accessing buffer states in case in the * future we want to put the buffer states in a more general @@ -518,98 +291,109 @@ static YY_BUFFER_STATE * yy_buffer_stack = NULL; /**< Stack as an array. */ #define YY_CURRENT_BUFFER ( (yy_buffer_stack) \ ? (yy_buffer_stack)[(yy_buffer_stack_top)] \ : NULL) + /* Same as previous macro, but useful when we know that the buffer stack is not * NULL or when we need an lvalue. For internal use only. */ #define YY_CURRENT_BUFFER_LVALUE (yy_buffer_stack)[(yy_buffer_stack_top)] -/* yy_hold_char holds the character lost when yytext is formed. */ +/* yy_hold_char holds the character lost when ncgtext is formed. */ static char yy_hold_char; static int yy_n_chars; /* number of characters read into yy_ch_buf */ -int yyleng; +yy_size_t ncgleng; /* Points to current character in buffer. */ -static char *yy_c_buf_p = NULL; +static char *yy_c_buf_p = (char *) 0; static int yy_init = 0; /* whether we need to initialize */ static int yy_start = 0; /* start state number */ -/* Flag which is used to allow yywrap()'s to do buffer switches - * instead of setting up a fresh yyin. A bit of a hack ... +/* Flag which is used to allow ncgwrap()'s to do buffer switches + * instead of setting up a fresh ncgin. A bit of a hack ... */ static int yy_did_buffer_switch_on_eof; -void yyrestart ( FILE *input_file ); -void yy_switch_to_buffer ( YY_BUFFER_STATE new_buffer ); -YY_BUFFER_STATE yy_create_buffer ( FILE *file, int size ); -void yy_delete_buffer ( YY_BUFFER_STATE b ); -void yy_flush_buffer ( YY_BUFFER_STATE b ); -void yypush_buffer_state ( YY_BUFFER_STATE new_buffer ); -void yypop_buffer_state ( void ); +void ncgrestart (FILE *input_file ); +void ncg_switch_to_buffer (YY_BUFFER_STATE new_buffer ); +YY_BUFFER_STATE ncg_create_buffer (FILE *file,int size ); +void ncg_delete_buffer (YY_BUFFER_STATE b ); +void ncg_flush_buffer (YY_BUFFER_STATE b ); +void ncgpush_buffer_state (YY_BUFFER_STATE new_buffer ); +void ncgpop_buffer_state (void ); -static void yyensure_buffer_stack ( void ); -static void yy_load_buffer_state ( void ); -static void yy_init_buffer ( YY_BUFFER_STATE b, FILE *file ); -#define YY_FLUSH_BUFFER yy_flush_buffer( YY_CURRENT_BUFFER ) +static void ncgensure_buffer_stack (void ); +static void ncg_load_buffer_state (void ); +static void ncg_init_buffer (YY_BUFFER_STATE b,FILE *file ); -YY_BUFFER_STATE yy_scan_buffer ( char *base, yy_size_t size ); -YY_BUFFER_STATE yy_scan_string ( const char *yy_str ); -YY_BUFFER_STATE yy_scan_bytes ( const char *bytes, int len ); +#define YY_FLUSH_BUFFER ncg_flush_buffer(YY_CURRENT_BUFFER ) -void *yyalloc ( yy_size_t ); -void *yyrealloc ( void *, yy_size_t ); -void yyfree ( void * ); +YY_BUFFER_STATE ncg_scan_buffer (char *base,yy_size_t size ); +YY_BUFFER_STATE ncg_scan_string (yyconst char *yy_str ); +YY_BUFFER_STATE ncg_scan_bytes (yyconst char *bytes,yy_size_t len ); + +void *ncgalloc (yy_size_t ); +void *ncgrealloc (void *,yy_size_t ); +void ncgfree (void * ); + +#define yy_new_buffer ncg_create_buffer -#define yy_new_buffer yy_create_buffer #define yy_set_interactive(is_interactive) \ { \ if ( ! YY_CURRENT_BUFFER ){ \ - yyensure_buffer_stack (); \ + ncgensure_buffer_stack (); \ YY_CURRENT_BUFFER_LVALUE = \ - yy_create_buffer( yyin, YY_BUF_SIZE ); \ + ncg_create_buffer(ncgin,YY_BUF_SIZE ); \ } \ YY_CURRENT_BUFFER_LVALUE->yy_is_interactive = is_interactive; \ } + #define yy_set_bol(at_bol) \ { \ if ( ! YY_CURRENT_BUFFER ){\ - yyensure_buffer_stack (); \ + ncgensure_buffer_stack (); \ YY_CURRENT_BUFFER_LVALUE = \ - yy_create_buffer( yyin, YY_BUF_SIZE ); \ + ncg_create_buffer(ncgin,YY_BUF_SIZE ); \ } \ YY_CURRENT_BUFFER_LVALUE->yy_at_bol = at_bol; \ } + #define YY_AT_BOL() (YY_CURRENT_BUFFER_LVALUE->yy_at_bol) /* Begin user sect3 */ -typedef flex_uint8_t YY_CHAR; -FILE *yyin = NULL, *yyout = NULL; +typedef unsigned char YY_CHAR; + +FILE *ncgin = (FILE *) 0, *ncgout = (FILE *) 0; typedef int yy_state_type; -extern int yylineno; -int yylineno = 1; +extern int ncglineno; + +int ncglineno = 1; -extern char *yytext; +extern char *ncgtext; #ifdef yytext_ptr #undef yytext_ptr #endif -#define yytext_ptr yytext +#define yytext_ptr ncgtext -static yy_state_type yy_get_previous_state ( void ); -static yy_state_type yy_try_NUL_trans ( yy_state_type current_state ); -static int yy_get_next_buffer ( void ); -static void yynoreturn yy_fatal_error ( const char* msg ); +static yy_state_type yy_get_previous_state (void ); +static yy_state_type yy_try_NUL_trans (yy_state_type current_state ); +static int yy_get_next_buffer (void ); +#if defined(__GNUC__) && __GNUC__ >= 3 +__attribute__((__noreturn__)) +#endif +static void yy_fatal_error (yyconst char msg[] ); /* Done after the current pattern has been matched and before the - * corresponding action - sets up yytext. + * corresponding action - sets up ncgtext. */ #define YY_DO_BEFORE_ACTION \ (yytext_ptr) = yy_bp; \ - yyleng = (int) (yy_cp - yy_bp); \ + ncgleng = (size_t) (yy_cp - yy_bp); \ (yy_hold_char) = *yy_cp; \ *yy_cp = '\0'; \ (yy_c_buf_p) = yy_cp; + #define YY_NUM_RULES 49 #define YY_END_OF_BUFFER 50 /* This struct is not used in this scanner, @@ -619,7 +403,7 @@ struct yy_trans_info flex_int32_t yy_verify; flex_int32_t yy_nxt; }; -static const flex_int16_t yy_accept[422] = +static yyconst flex_int16_t yy_accept[422] = { 0, 0, 0, 46, 46, 0, 0, 50, 48, 1, 44, 48, 48, 48, 48, 38, 32, 36, 36, 35, 35, @@ -670,7 +454,7 @@ static const flex_int16_t yy_accept[422] = 0 } ; -static const YY_CHAR yy_ec[256] = +static yyconst YY_CHAR yy_ec[256] = { 0, 1, 1, 1, 1, 1, 1, 1, 1, 2, 3, 1, 2, 2, 1, 1, 1, 1, 1, 1, 1, @@ -702,7 +486,7 @@ static const YY_CHAR yy_ec[256] = 1, 1, 1, 1, 1 } ; -static const YY_CHAR yy_meta[69] = +static yyconst YY_CHAR yy_meta[69] = { 0, 1, 1, 2, 1, 1, 1, 3, 4, 5, 5, 6, 7, 8, 8, 8, 8, 8, 8, 8, 1, @@ -713,7 +497,7 @@ static const YY_CHAR yy_meta[69] = 11, 11, 11, 14, 1, 11, 11, 11 } ; -static const flex_int16_t yy_base[440] = +static yyconst flex_uint16_t yy_base[440] = { 0, 0, 0, 325, 321, 264, 255, 318, 2387, 67, 2387, 64, 269, 61, 62, 95, 77, 136, 259, 51, 61, @@ -765,7 +549,7 @@ static const flex_int16_t yy_base[440] = 2315, 2329, 2339, 2345, 2353, 2355, 2361, 2367, 2373 } ; -static const flex_int16_t yy_def[440] = +static yyconst flex_int16_t yy_def[440] = { 0, 421, 1, 422, 422, 423, 423, 421, 421, 421, 421, 424, 425, 421, 426, 421, 427, 421, 17, 428, 428, @@ -817,7 +601,7 @@ static const flex_int16_t yy_def[440] = 421, 421, 421, 421, 421, 421, 421, 421, 421 } ; -static const flex_int16_t yy_nxt[2456] = +static yyconst flex_uint16_t yy_nxt[2456] = { 0, 8, 9, 10, 9, 8, 11, 12, 8, 13, 14, 15, 16, 17, 18, 18, 18, 18, 18, 18, 8, @@ -1091,7 +875,7 @@ static const flex_int16_t yy_nxt[2456] = 421, 421, 421, 421, 421 } ; -static const flex_int16_t yy_chk[2456] = +static yyconst flex_int16_t yy_chk[2456] = { 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, @@ -1368,8 +1152,8 @@ static const flex_int16_t yy_chk[2456] = static yy_state_type yy_last_accepting_state; static char *yy_last_accepting_cpos; -extern int yy_flex_debug; -int yy_flex_debug = 0; +extern int ncg_flex_debug; +int ncg_flex_debug = 0; /* The intent behind this definition is that it'll catch * any uses of REJECT which flex missed. @@ -1378,7 +1162,7 @@ int yy_flex_debug = 0; #define yymore() yymore_used_but_not_detected #define YY_MORE_ADJ 0 #define YY_RESTORE_YY_MORE_OFFSET -char *yytext; +char *ncgtext; #line 1 "ncgen.l" #line 2 "ncgen.l" /********************************************************************* @@ -1521,7 +1305,7 @@ struct Specialtoken specials[] = { {NULL,0} /* null terminate */ }; -#line 1524 "ncgenl.c" + /* The most correct (validating) version of UTF8 character set (Taken from: http://www.w3.org/2005/03/23-lex-U) @@ -1564,7 +1348,7 @@ ID ([A-Za-z_]|{UTF8})([A-Z.@#\[\]a-z_0-9+-]|{UTF8})* /* Note: this definition of string will work for utf8 as well, although it is a very relaxed definition */ -#line 1567 "ncgenl.c" +#line 1352 "ncgenl.c" #define INITIAL 0 #define ST_C_COMMENT 1 @@ -1582,36 +1366,36 @@ ID ([A-Za-z_]|{UTF8})([A-Z.@#\[\]a-z_0-9+-]|{UTF8})* #define YY_EXTRA_TYPE void * #endif -static int yy_init_globals ( void ); +static int yy_init_globals (void ); /* Accessor methods to globals. These are made visible to non-reentrant scanners for convenience. */ -int yylex_destroy ( void ); +int ncglex_destroy (void ); -int yyget_debug ( void ); +int ncgget_debug (void ); -void yyset_debug ( int debug_flag ); +void ncgset_debug (int debug_flag ); -YY_EXTRA_TYPE yyget_extra ( void ); +YY_EXTRA_TYPE ncgget_extra (void ); -void yyset_extra ( YY_EXTRA_TYPE user_defined ); +void ncgset_extra (YY_EXTRA_TYPE user_defined ); -FILE *yyget_in ( void ); +FILE *ncgget_in (void ); -void yyset_in ( FILE * _in_str ); +void ncgset_in (FILE * _in_str ); -FILE *yyget_out ( void ); +FILE *ncgget_out (void ); -void yyset_out ( FILE * _out_str ); +void ncgset_out (FILE * _out_str ); - int yyget_leng ( void ); +yy_size_t ncgget_leng (void ); -char *yyget_text ( void ); +char *ncgget_text (void ); -int yyget_lineno ( void ); +int ncgget_lineno (void ); -void yyset_lineno ( int _line_number ); +void ncgset_lineno (int _line_number ); /* Macros after this point can all be overridden by user definitions in * section 1. @@ -1619,31 +1403,32 @@ void yyset_lineno ( int _line_number ); #ifndef YY_SKIP_YYWRAP #ifdef __cplusplus -extern "C" int yywrap ( void ); +extern "C" int ncgwrap (void ); #else -extern int yywrap ( void ); +extern int ncgwrap (void ); #endif #endif #ifndef YY_NO_UNPUT - static void yyunput ( int c, char *buf_ptr ); + static void yyunput (int c,char *buf_ptr ); #endif #ifndef yytext_ptr -static void yy_flex_strncpy ( char *, const char *, int ); +static void yy_flex_strncpy (char *,yyconst char *,int ); #endif #ifdef YY_NEED_STRLEN -static int yy_flex_strlen ( const char * ); +static int yy_flex_strlen (yyconst char * ); #endif #ifndef YY_NO_INPUT + #ifdef __cplusplus -static int yyinput ( void ); +static int yyinput (void ); #else -static int input ( void ); +static int input (void ); #endif #endif @@ -1663,7 +1448,7 @@ static int input ( void ); /* This used to be an fputs(), but since the string might contain NUL's, * we now use fwrite(). */ -#define ECHO do { if (fwrite( yytext, (size_t) yyleng, 1, yyout )) {} } while (0) +#define ECHO do { if (fwrite( ncgtext, ncgleng, 1, ncgout )) {} } while (0) #endif /* Gets input and stuffs it into "buf". number of characters read, or YY_NULL, @@ -1674,20 +1459,20 @@ static int input ( void ); if ( YY_CURRENT_BUFFER_LVALUE->yy_is_interactive ) \ { \ int c = '*'; \ - int n; \ + size_t n; \ for ( n = 0; n < max_size && \ - (c = getc( yyin )) != EOF && c != '\n'; ++n ) \ + (c = getc( ncgin )) != EOF && c != '\n'; ++n ) \ buf[n] = (char) c; \ if ( c == '\n' ) \ buf[n++] = (char) c; \ - if ( c == EOF && ferror( yyin ) ) \ + if ( c == EOF && ferror( ncgin ) ) \ YY_FATAL_ERROR( "input in flex scanner failed" ); \ result = n; \ } \ else \ { \ errno=0; \ - while ( (result = (int) fread(buf, 1, (yy_size_t) max_size, yyin)) == 0 && ferror(yyin)) \ + while ( (result = fread(buf, 1, max_size, ncgin))==0 && ferror(ncgin)) \ { \ if( errno != EINTR) \ { \ @@ -1695,7 +1480,7 @@ static int input ( void ); break; \ } \ errno=0; \ - clearerr(yyin); \ + clearerr(ncgin); \ } \ }\ \ @@ -1728,12 +1513,12 @@ static int input ( void ); #ifndef YY_DECL #define YY_DECL_IS_OURS 1 -extern int yylex (void); +extern int ncglex (void); -#define YY_DECL int yylex (void) +#define YY_DECL int ncglex (void) #endif /* !YY_DECL */ -/* Code executed at the beginning of each rule, after yytext and yyleng +/* Code executed at the beginning of each rule, after ncgtext and ncgleng * have been set up. */ #ifndef YY_USER_ACTION @@ -1767,31 +1552,31 @@ YY_DECL if ( ! (yy_start) ) (yy_start) = 1; /* first start state */ - if ( ! yyin ) - yyin = stdin; + if ( ! ncgin ) + ncgin = stdin; - if ( ! yyout ) - yyout = stdout; + if ( ! ncgout ) + ncgout = stdout; if ( ! YY_CURRENT_BUFFER ) { - yyensure_buffer_stack (); + ncgensure_buffer_stack (); YY_CURRENT_BUFFER_LVALUE = - yy_create_buffer( yyin, YY_BUF_SIZE ); + ncg_create_buffer(ncgin,YY_BUF_SIZE ); } - yy_load_buffer_state( ); + ncg_load_buffer_state( ); } { #line 221 "ncgen.l" -#line 1788 "ncgenl.c" +#line 1574 "ncgenl.c" while ( /*CONSTCOND*/1 ) /* loops until end-of-file is reached */ { yy_cp = (yy_c_buf_p); - /* Support of yytext. */ + /* Support of ncgtext. */ *yy_cp = (yy_hold_char); /* yy_bp points to the position in yy_ch_buf of the start of @@ -1813,9 +1598,9 @@ YY_DECL { yy_current_state = (int) yy_def[yy_current_state]; if ( yy_current_state >= 422 ) - yy_c = yy_meta[yy_c]; + yy_c = yy_meta[(unsigned int) yy_c]; } - yy_current_state = yy_nxt[yy_base[yy_current_state] + yy_c]; + yy_current_state = yy_nxt[yy_base[yy_current_state] + (unsigned int) yy_c]; ++yy_cp; } while ( yy_base[yy_current_state] != 2387 ); @@ -1864,14 +1649,14 @@ YY_RULE_SETUP /* In netcdf4, this will be used in a variety of places, so only remove escapes */ /* -if(yyleng > MAXTRST) { +if(ncgleng > MAXTRST) { yyerror("string too long, truncated\n"); -yytext[MAXTRST-1] = '\0'; +ncgtext[MAXTRST-1] = '\0'; } */ - len = unescape((char *)yytext+1,yyleng-2,!ISIDENT,&s); + len = unescape((char *)ncgtext+1,ncgleng-2,!ISIDENT,&s); if(len < 0) { - sprintf(errstr,"Illegal character: %s",yytext); + sprintf(errstr,"Illegal character: %s",ncgtext); yyerror(errstr); } bbClear(lextext); @@ -1885,8 +1670,8 @@ case 4: YY_RULE_SETUP #line 251 "ncgen.l" { /* drop leading 0x; pad to even number of chars */ - char* p = yytext+2; - int len = yyleng - 2; + char* p = ncgtext+2; + int len = ncgleng - 2; bbClear(lextext); bbAppendn(lextext,p,len); if((len % 2) == 1) bbAppend(lextext,'0'); @@ -2011,7 +1796,7 @@ case 27: YY_RULE_SETUP #line 291 "ncgen.l" { /* missing value (pre-2.4 backward compatibility) */ - if (yytext[0] == '-') { + if (ncgtext[0] == '-') { double_val = NEGNC_INFINITE; } else { double_val = NC_INFINITE; @@ -2033,7 +1818,7 @@ case 29: YY_RULE_SETUP #line 306 "ncgen.l" {/* missing value (pre-2.4 backward compatibility)*/ - if (yytext[0] == '-') { + if (ncgtext[0] == '-') { float_val = NEGNC_INFINITEF; } else { float_val = NC_INFINITEF; @@ -2069,7 +1854,7 @@ YY_RULE_SETUP #line 331 "ncgen.l" { bbClear(lextext); - bbAppendn(lextext,(char*)yytext,yyleng+1); /* include null */ + bbAppendn(lextext,(char*)ncgtext,ncgleng+1); /* include null */ bbNull(lextext); yylval.sym = makepath(bbContents(lextext)); return lexdebug(PATH); @@ -2080,7 +1865,7 @@ YY_RULE_SETUP #line 340 "ncgen.l" {struct Specialtoken* st; bbClear(lextext); - bbAppendn(lextext,(char*)yytext,yyleng+1); /* include null */ + bbAppendn(lextext,(char*)ncgtext,ncgleng+1); /* include null */ bbNull(lextext); for(st=specials;st->name;st++) { if(strcmp(bbContents(lextext),st->name)==0) {return lexdebug(st->token);} @@ -2097,7 +1882,7 @@ YY_RULE_SETUP char* p; char* q; /* copy the trimmed name */ bbClear(lextext); - bbAppendn(lextext,(char*)yytext,yyleng+1); /* include null */ + bbAppendn(lextext,(char*)ncgtext,ncgleng+1); /* include null */ bbNull(lextext); p = bbContents(lextext); q = p; @@ -2113,8 +1898,8 @@ case 35: YY_RULE_SETUP #line 367 "ncgen.l" { char* id = NULL; int len; - len = strlen(yytext); - len = unescape(yytext,len,ISIDENT,&id); + len = strlen(ncgtext); + len = unescape(ncgtext,len,ISIDENT,&id); if(NCSTREQ(id, FILL_STRING)) { efree(id); return lexdebug(FILLMARKER); @@ -2211,21 +1996,21 @@ YY_RULE_SETUP { int c; int token = 0; - int slen = strlen(yytext); + int slen = strlen(ncgtext); char* stag = NULL; int tag = NC_NAT; - char* hex = yytext+2; /* point to first true hex digit */ + char* hex = ncgtext+2; /* point to first true hex digit */ int xlen = (slen - 3); /* true hex length */ - yytext[slen-1] = '\0'; + ncgtext[slen-1] = '\0'; /* capture the tag string */ - tag = collecttag(yytext,&stag); + tag = collecttag(ncgtext,&stag); if(tag == NC_NAT) { sprintf(errstr,"Illegal integer suffix: %s",stag); yyerror(errstr); goto done; } - yytext[slen - strlen(stag)] = '\0'; + ncgtext[slen - strlen(stag)] = '\0'; if(xlen > 16) { /* truncate hi order digits */ hex += (xlen - 16); } @@ -2247,8 +2032,8 @@ YY_RULE_SETUP token = UINT64_CONST; break; default: /* should never happen */ - if (sscanf((char*)yytext, "%i", &uint32_val) != 1) { - sprintf(errstr,"bad unsigned int constant: %s",(char*)yytext); + if (sscanf((char*)ncgtext, "%i", &uint32_val) != 1) { + sprintf(errstr,"bad unsigned int constant: %s",(char*)ncgtext); yyerror(errstr); } token = UINT_CONST; @@ -2260,8 +2045,8 @@ case 38: YY_RULE_SETUP #line 504 "ncgen.l" { - if (sscanf((char*)yytext, "%le", &double_val) != 1) { - sprintf(errstr,"bad long or double constant: %s",(char*)yytext); + if (sscanf((char*)ncgtext, "%le", &double_val) != 1) { + sprintf(errstr,"bad long or double constant: %s",(char*)ncgtext); yyerror(errstr); } return lexdebug(DOUBLE_CONST); @@ -2271,8 +2056,8 @@ case 39: YY_RULE_SETUP #line 511 "ncgen.l" { - if (sscanf((char*)yytext, "%e", &float_val) != 1) { - sprintf(errstr,"bad float constant: %s",(char*)yytext); + if (sscanf((char*)ncgtext, "%e", &float_val) != 1) { + sprintf(errstr,"bad float constant: %s",(char*)ncgtext); yyerror(errstr); } return lexdebug(FLOAT_CONST); @@ -2283,7 +2068,7 @@ case 40: YY_RULE_SETUP #line 518 "ncgen.l" { - (void) sscanf((char*)&yytext[1],"%c",&byte_val); + (void) sscanf((char*)&ncgtext[1],"%c",&byte_val); return lexdebug(BYTE_CONST); } YY_BREAK @@ -2291,9 +2076,9 @@ case 41: YY_RULE_SETUP #line 522 "ncgen.l" { - int oct = unescapeoct(&yytext[2]); + int oct = unescapeoct(&ncgtext[2]); if(oct < 0) { - sprintf(errstr,"bad octal character constant: %s",(char*)yytext); + sprintf(errstr,"bad octal character constant: %s",(char*)ncgtext); yyerror(errstr); } byte_val = (unsigned int)oct; @@ -2304,9 +2089,9 @@ case 42: YY_RULE_SETUP #line 531 "ncgen.l" { - int hex = unescapehex(&yytext[3]); + int hex = unescapehex(&ncgtext[3]); if(byte_val < 0) { - sprintf(errstr,"bad hex character constant: %s",(char*)yytext); + sprintf(errstr,"bad hex character constant: %s",(char*)ncgtext); yyerror(errstr); } byte_val = (unsigned int)hex; @@ -2317,7 +2102,7 @@ case 43: YY_RULE_SETUP #line 540 "ncgen.l" { - switch ((char)yytext[2]) { + switch ((char)ncgtext[2]) { case 'a': byte_val = '\007'; break; /* not everyone under- * stands '\a' yet */ case 'b': byte_val = '\b'; break; @@ -2329,7 +2114,7 @@ YY_RULE_SETUP case '\\': byte_val = '\\'; break; case '?': byte_val = '\177'; break; case '\'': byte_val = '\''; break; - default: byte_val = (char)yytext[2]; + default: byte_val = (char)ncgtext[2]; } return lexdebug(BYTE_CONST); } @@ -2379,7 +2164,7 @@ case 48: YY_RULE_SETUP #line 583 "ncgen.l" {/* Note: this next rule will not work for UTF8 characters */ - return lexdebug(yytext[0]) ; + return lexdebug(ncgtext[0]) ; } YY_BREAK case 49: @@ -2387,7 +2172,7 @@ YY_RULE_SETUP #line 586 "ncgen.l" ECHO; YY_BREAK -#line 2390 "ncgenl.c" +#line 2176 "ncgenl.c" case YY_STATE_EOF(INITIAL): case YY_STATE_EOF(TEXT): yyterminate(); @@ -2405,15 +2190,15 @@ case YY_STATE_EOF(TEXT): { /* We're scanning a new file or input source. It's * possible that this happened because the user - * just pointed yyin at a new source and called - * yylex(). If so, then we have to assure + * just pointed ncgin at a new source and called + * ncglex(). If so, then we have to assure * consistency between YY_CURRENT_BUFFER and our * globals. Here is the right place to do so, because * this is the first action (other than possibly a * back-up) that will match for the new input source. */ (yy_n_chars) = YY_CURRENT_BUFFER_LVALUE->yy_n_chars; - YY_CURRENT_BUFFER_LVALUE->yy_input_file = yyin; + YY_CURRENT_BUFFER_LVALUE->yy_input_file = ncgin; YY_CURRENT_BUFFER_LVALUE->yy_buffer_status = YY_BUFFER_NORMAL; } @@ -2466,11 +2251,11 @@ case YY_STATE_EOF(TEXT): { (yy_did_buffer_switch_on_eof) = 0; - if ( yywrap( ) ) + if ( ncgwrap( ) ) { /* Note: because we've taken care in * yy_get_next_buffer() to have set up - * yytext, we can now set up + * ncgtext, we can now set up * yy_c_buf_p so that if some total * hoser (like flex itself) wants to * call the scanner after we return the @@ -2520,7 +2305,7 @@ case YY_STATE_EOF(TEXT): } /* end of action switch */ } /* end of scanning one token */ } /* end of user's declarations */ -} /* end of yylex */ +} /* end of ncglex */ /* yy_get_next_buffer - try to read in a new buffer * @@ -2533,7 +2318,7 @@ static int yy_get_next_buffer (void) { char *dest = YY_CURRENT_BUFFER_LVALUE->yy_ch_buf; char *source = (yytext_ptr); - int number_to_move, i; + yy_size_t number_to_move, i; int ret_val; if ( (yy_c_buf_p) > &YY_CURRENT_BUFFER_LVALUE->yy_ch_buf[(yy_n_chars) + 1] ) @@ -2562,7 +2347,7 @@ static int yy_get_next_buffer (void) /* Try to read more data. */ /* First move last chars to start of buffer. */ - number_to_move = (int) ((yy_c_buf_p) - (yytext_ptr) - 1); + number_to_move = (yy_size_t) ((yy_c_buf_p) - (yytext_ptr)) - 1; for ( i = 0; i < number_to_move; ++i ) *(dest++) = *(source++); @@ -2575,7 +2360,7 @@ static int yy_get_next_buffer (void) else { - int num_to_read = + yy_size_t num_to_read = YY_CURRENT_BUFFER_LVALUE->yy_buf_size - number_to_move - 1; while ( num_to_read <= 0 ) @@ -2589,7 +2374,7 @@ static int yy_get_next_buffer (void) if ( b->yy_is_our_buffer ) { - int new_size = b->yy_buf_size * 2; + yy_size_t new_size = b->yy_buf_size * 2; if ( new_size <= 0 ) b->yy_buf_size += b->yy_buf_size / 8; @@ -2598,12 +2383,11 @@ static int yy_get_next_buffer (void) b->yy_ch_buf = (char *) /* Include room in for 2 EOB chars. */ - yyrealloc( (void *) b->yy_ch_buf, - (yy_size_t) (b->yy_buf_size + 2) ); + ncgrealloc((void *) b->yy_ch_buf,b->yy_buf_size + 2 ); } else /* Can't grow it, we don't own it. */ - b->yy_ch_buf = NULL; + b->yy_ch_buf = 0; if ( ! b->yy_ch_buf ) YY_FATAL_ERROR( @@ -2631,7 +2415,7 @@ static int yy_get_next_buffer (void) if ( number_to_move == YY_MORE_ADJ ) { ret_val = EOB_ACT_END_OF_FILE; - yyrestart( yyin ); + ncgrestart(ncgin ); } else @@ -2645,15 +2429,12 @@ static int yy_get_next_buffer (void) else ret_val = EOB_ACT_CONTINUE_SCAN; - if (((yy_n_chars) + number_to_move) > YY_CURRENT_BUFFER_LVALUE->yy_buf_size) { + if ((int) ((yy_n_chars) + number_to_move) > YY_CURRENT_BUFFER_LVALUE->yy_buf_size) { /* Extend the array by 50%, plus the number we really need. */ int new_size = (yy_n_chars) + number_to_move + ((yy_n_chars) >> 1); - YY_CURRENT_BUFFER_LVALUE->yy_ch_buf = (char *) yyrealloc( - (void *) YY_CURRENT_BUFFER_LVALUE->yy_ch_buf, (yy_size_t) new_size ); + YY_CURRENT_BUFFER_LVALUE->yy_ch_buf = (char *) ncgrealloc((void *) YY_CURRENT_BUFFER_LVALUE->yy_ch_buf,new_size ); if ( ! YY_CURRENT_BUFFER_LVALUE->yy_ch_buf ) YY_FATAL_ERROR( "out of dynamic memory in yy_get_next_buffer()" ); - /* "- 2" to take care of EOB's */ - YY_CURRENT_BUFFER_LVALUE->yy_buf_size = (int) (new_size - 2); } (yy_n_chars) += number_to_move; @@ -2686,9 +2467,9 @@ static int yy_get_next_buffer (void) { yy_current_state = (int) yy_def[yy_current_state]; if ( yy_current_state >= 422 ) - yy_c = yy_meta[yy_c]; + yy_c = yy_meta[(unsigned int) yy_c]; } - yy_current_state = yy_nxt[yy_base[yy_current_state] + yy_c]; + yy_current_state = yy_nxt[yy_base[yy_current_state] + (unsigned int) yy_c]; } return yy_current_state; @@ -2714,9 +2495,9 @@ static int yy_get_next_buffer (void) { yy_current_state = (int) yy_def[yy_current_state]; if ( yy_current_state >= 422 ) - yy_c = yy_meta[yy_c]; + yy_c = yy_meta[(unsigned int) yy_c]; } - yy_current_state = yy_nxt[yy_base[yy_current_state] + yy_c]; + yy_current_state = yy_nxt[yy_base[yy_current_state] + (unsigned int) yy_c]; yy_is_jam = (yy_current_state == 421); return yy_is_jam ? 0 : yy_current_state; @@ -2730,13 +2511,13 @@ static int yy_get_next_buffer (void) yy_cp = (yy_c_buf_p); - /* undo effects of setting up yytext */ + /* undo effects of setting up ncgtext */ *yy_cp = (yy_hold_char); if ( yy_cp < YY_CURRENT_BUFFER_LVALUE->yy_ch_buf + 2 ) { /* need to shift things up to make room */ /* +2 for EOB chars. */ - int number_to_move = (yy_n_chars) + 2; + yy_size_t number_to_move = (yy_n_chars) + 2; char *dest = &YY_CURRENT_BUFFER_LVALUE->yy_ch_buf[ YY_CURRENT_BUFFER_LVALUE->yy_buf_size + 2]; char *source = @@ -2748,7 +2529,7 @@ static int yy_get_next_buffer (void) yy_cp += (int) (dest - source); yy_bp += (int) (dest - source); YY_CURRENT_BUFFER_LVALUE->yy_n_chars = - (yy_n_chars) = (int) YY_CURRENT_BUFFER_LVALUE->yy_buf_size; + (yy_n_chars) = YY_CURRENT_BUFFER_LVALUE->yy_buf_size; if ( yy_cp < YY_CURRENT_BUFFER_LVALUE->yy_ch_buf + 2 ) YY_FATAL_ERROR( "flex scanner push-back overflow" ); @@ -2787,7 +2568,7 @@ static int yy_get_next_buffer (void) else { /* need more input */ - int offset = (int) ((yy_c_buf_p) - (yytext_ptr)); + yy_size_t offset = (yy_c_buf_p) - (yytext_ptr); ++(yy_c_buf_p); switch ( yy_get_next_buffer( ) ) @@ -2804,14 +2585,14 @@ static int yy_get_next_buffer (void) */ /* Reset buffer status. */ - yyrestart( yyin ); + ncgrestart(ncgin ); /*FALLTHROUGH*/ case EOB_ACT_END_OF_FILE: { - if ( yywrap( ) ) - return 0; + if ( ncgwrap( ) ) + return EOF; if ( ! (yy_did_buffer_switch_on_eof) ) YY_NEW_FILE; @@ -2830,7 +2611,7 @@ static int yy_get_next_buffer (void) } c = *(unsigned char *) (yy_c_buf_p); /* cast for 8-bit char's */ - *(yy_c_buf_p) = '\0'; /* preserve yytext */ + *(yy_c_buf_p) = '\0'; /* preserve ncgtext */ (yy_hold_char) = *++(yy_c_buf_p); return c; @@ -2842,32 +2623,32 @@ static int yy_get_next_buffer (void) * * @note This function does not reset the start condition to @c INITIAL . */ - void yyrestart (FILE * input_file ) + void ncgrestart (FILE * input_file ) { if ( ! YY_CURRENT_BUFFER ){ - yyensure_buffer_stack (); + ncgensure_buffer_stack (); YY_CURRENT_BUFFER_LVALUE = - yy_create_buffer( yyin, YY_BUF_SIZE ); + ncg_create_buffer(ncgin,YY_BUF_SIZE ); } - yy_init_buffer( YY_CURRENT_BUFFER, input_file ); - yy_load_buffer_state( ); + ncg_init_buffer(YY_CURRENT_BUFFER,input_file ); + ncg_load_buffer_state( ); } /** Switch to a different input buffer. * @param new_buffer The new input buffer. * */ - void yy_switch_to_buffer (YY_BUFFER_STATE new_buffer ) + void ncg_switch_to_buffer (YY_BUFFER_STATE new_buffer ) { /* TODO. We should be able to replace this entire function body * with - * yypop_buffer_state(); - * yypush_buffer_state(new_buffer); + * ncgpop_buffer_state(); + * ncgpush_buffer_state(new_buffer); */ - yyensure_buffer_stack (); + ncgensure_buffer_stack (); if ( YY_CURRENT_BUFFER == new_buffer ) return; @@ -2880,21 +2661,21 @@ static int yy_get_next_buffer (void) } YY_CURRENT_BUFFER_LVALUE = new_buffer; - yy_load_buffer_state( ); + ncg_load_buffer_state( ); /* We don't actually know whether we did this switch during - * EOF (yywrap()) processing, but the only time this flag - * is looked at is after yywrap() is called, so it's safe + * EOF (ncgwrap()) processing, but the only time this flag + * is looked at is after ncgwrap() is called, so it's safe * to go ahead and always set it. */ (yy_did_buffer_switch_on_eof) = 1; } -static void yy_load_buffer_state (void) +static void ncg_load_buffer_state (void) { (yy_n_chars) = YY_CURRENT_BUFFER_LVALUE->yy_n_chars; (yytext_ptr) = (yy_c_buf_p) = YY_CURRENT_BUFFER_LVALUE->yy_buf_pos; - yyin = YY_CURRENT_BUFFER_LVALUE->yy_input_file; + ncgin = YY_CURRENT_BUFFER_LVALUE->yy_input_file; (yy_hold_char) = *(yy_c_buf_p); } @@ -2904,35 +2685,35 @@ static void yy_load_buffer_state (void) * * @return the allocated buffer state. */ - YY_BUFFER_STATE yy_create_buffer (FILE * file, int size ) + YY_BUFFER_STATE ncg_create_buffer (FILE * file, int size ) { YY_BUFFER_STATE b; - b = (YY_BUFFER_STATE) yyalloc( sizeof( struct yy_buffer_state ) ); + b = (YY_BUFFER_STATE) ncgalloc(sizeof( struct yy_buffer_state ) ); if ( ! b ) - YY_FATAL_ERROR( "out of dynamic memory in yy_create_buffer()" ); + YY_FATAL_ERROR( "out of dynamic memory in ncg_create_buffer()" ); - b->yy_buf_size = size; + b->yy_buf_size = (yy_size_t)size; /* yy_ch_buf has to be 2 characters longer than the size given because * we need to put in 2 end-of-buffer characters. */ - b->yy_ch_buf = (char *) yyalloc( (yy_size_t) (b->yy_buf_size + 2) ); + b->yy_ch_buf = (char *) ncgalloc(b->yy_buf_size + 2 ); if ( ! b->yy_ch_buf ) - YY_FATAL_ERROR( "out of dynamic memory in yy_create_buffer()" ); + YY_FATAL_ERROR( "out of dynamic memory in ncg_create_buffer()" ); b->yy_is_our_buffer = 1; - yy_init_buffer( b, file ); + ncg_init_buffer(b,file ); return b; } /** Destroy the buffer. - * @param b a buffer created with yy_create_buffer() + * @param b a buffer created with ncg_create_buffer() * */ - void yy_delete_buffer (YY_BUFFER_STATE b ) + void ncg_delete_buffer (YY_BUFFER_STATE b ) { if ( ! b ) @@ -2942,27 +2723,27 @@ static void yy_load_buffer_state (void) YY_CURRENT_BUFFER_LVALUE = (YY_BUFFER_STATE) 0; if ( b->yy_is_our_buffer ) - yyfree( (void *) b->yy_ch_buf ); + ncgfree((void *) b->yy_ch_buf ); - yyfree( (void *) b ); + ncgfree((void *) b ); } /* Initializes or reinitializes a buffer. * This function is sometimes called more than once on the same buffer, - * such as during a yyrestart() or at EOF. + * such as during a ncgrestart() or at EOF. */ - static void yy_init_buffer (YY_BUFFER_STATE b, FILE * file ) + static void ncg_init_buffer (YY_BUFFER_STATE b, FILE * file ) { int oerrno = errno; - yy_flush_buffer( b ); + ncg_flush_buffer(b ); b->yy_input_file = file; b->yy_fill_buffer = 1; - /* If b is the current buffer, then yy_init_buffer was _probably_ - * called from yyrestart() or through yy_get_next_buffer. + /* If b is the current buffer, then ncg_init_buffer was _probably_ + * called from ncgrestart() or through yy_get_next_buffer. * In that case, we don't want to reset the lineno or column. */ if (b != YY_CURRENT_BUFFER){ @@ -2979,7 +2760,7 @@ static void yy_load_buffer_state (void) * @param b the buffer state to be flushed, usually @c YY_CURRENT_BUFFER. * */ - void yy_flush_buffer (YY_BUFFER_STATE b ) + void ncg_flush_buffer (YY_BUFFER_STATE b ) { if ( ! b ) return; @@ -2999,7 +2780,7 @@ static void yy_load_buffer_state (void) b->yy_buffer_status = YY_BUFFER_NEW; if ( b == YY_CURRENT_BUFFER ) - yy_load_buffer_state( ); + ncg_load_buffer_state( ); } /** Pushes the new state onto the stack. The new state becomes @@ -3008,14 +2789,14 @@ static void yy_load_buffer_state (void) * @param new_buffer The new state. * */ -void yypush_buffer_state (YY_BUFFER_STATE new_buffer ) +void ncgpush_buffer_state (YY_BUFFER_STATE new_buffer ) { if (new_buffer == NULL) return; - yyensure_buffer_stack(); + ncgensure_buffer_stack(); - /* This block is copied from yy_switch_to_buffer. */ + /* This block is copied from ncg_switch_to_buffer. */ if ( YY_CURRENT_BUFFER ) { /* Flush out information for old buffer. */ @@ -3029,8 +2810,8 @@ void yypush_buffer_state (YY_BUFFER_STATE new_buffer ) (yy_buffer_stack_top)++; YY_CURRENT_BUFFER_LVALUE = new_buffer; - /* copied from yy_switch_to_buffer. */ - yy_load_buffer_state( ); + /* copied from ncg_switch_to_buffer. */ + ncg_load_buffer_state( ); (yy_did_buffer_switch_on_eof) = 1; } @@ -3038,18 +2819,18 @@ void yypush_buffer_state (YY_BUFFER_STATE new_buffer ) * The next element becomes the new top. * */ -void yypop_buffer_state (void) +void ncgpop_buffer_state (void) { if (!YY_CURRENT_BUFFER) return; - yy_delete_buffer(YY_CURRENT_BUFFER ); + ncg_delete_buffer(YY_CURRENT_BUFFER ); YY_CURRENT_BUFFER_LVALUE = NULL; if ((yy_buffer_stack_top) > 0) --(yy_buffer_stack_top); if (YY_CURRENT_BUFFER) { - yy_load_buffer_state( ); + ncg_load_buffer_state( ); (yy_did_buffer_switch_on_eof) = 1; } } @@ -3057,7 +2838,7 @@ void yypop_buffer_state (void) /* Allocates the stack if it does not exist. * Guarantees space for at least one push. */ -static void yyensure_buffer_stack (void) +static void ncgensure_buffer_stack (void) { yy_size_t num_to_alloc; @@ -3067,15 +2848,15 @@ static void yyensure_buffer_stack (void) * scanner will even need a stack. We use 2 instead of 1 to avoid an * immediate realloc on the next call. */ - num_to_alloc = 1; /* After all that talk, this was set to 1 anyways... */ - (yy_buffer_stack) = (struct yy_buffer_state**)yyalloc + num_to_alloc = 1; /* After all that talk, this was set to 1 anyways... */ + (yy_buffer_stack) = (struct yy_buffer_state**)ncgalloc (num_to_alloc * sizeof(struct yy_buffer_state*) ); if ( ! (yy_buffer_stack) ) - YY_FATAL_ERROR( "out of dynamic memory in yyensure_buffer_stack()" ); - + YY_FATAL_ERROR( "out of dynamic memory in ncgensure_buffer_stack()" ); + memset((yy_buffer_stack), 0, num_to_alloc * sizeof(struct yy_buffer_state*)); - + (yy_buffer_stack_max) = num_to_alloc; (yy_buffer_stack_top) = 0; return; @@ -3087,12 +2868,12 @@ static void yyensure_buffer_stack (void) yy_size_t grow_size = 8 /* arbitrary grow size */; num_to_alloc = (yy_buffer_stack_max) + grow_size; - (yy_buffer_stack) = (struct yy_buffer_state**)yyrealloc + (yy_buffer_stack) = (struct yy_buffer_state**)ncgrealloc ((yy_buffer_stack), num_to_alloc * sizeof(struct yy_buffer_state*) ); if ( ! (yy_buffer_stack) ) - YY_FATAL_ERROR( "out of dynamic memory in yyensure_buffer_stack()" ); + YY_FATAL_ERROR( "out of dynamic memory in ncgensure_buffer_stack()" ); /* zero only the new slots.*/ memset((yy_buffer_stack) + (yy_buffer_stack_max), 0, grow_size * sizeof(struct yy_buffer_state*)); @@ -3104,9 +2885,9 @@ static void yyensure_buffer_stack (void) * @param base the character buffer * @param size the size in bytes of the character buffer * - * @return the newly allocated buffer state object. + * @return the newly allocated buffer state object. */ -YY_BUFFER_STATE yy_scan_buffer (char * base, yy_size_t size ) +YY_BUFFER_STATE ncg_scan_buffer (char * base, yy_size_t size ) { YY_BUFFER_STATE b; @@ -3114,69 +2895,69 @@ YY_BUFFER_STATE yy_scan_buffer (char * base, yy_size_t size ) base[size-2] != YY_END_OF_BUFFER_CHAR || base[size-1] != YY_END_OF_BUFFER_CHAR ) /* They forgot to leave room for the EOB's. */ - return NULL; + return 0; - b = (YY_BUFFER_STATE) yyalloc( sizeof( struct yy_buffer_state ) ); + b = (YY_BUFFER_STATE) ncgalloc(sizeof( struct yy_buffer_state ) ); if ( ! b ) - YY_FATAL_ERROR( "out of dynamic memory in yy_scan_buffer()" ); + YY_FATAL_ERROR( "out of dynamic memory in ncg_scan_buffer()" ); - b->yy_buf_size = (int) (size - 2); /* "- 2" to take care of EOB's */ + b->yy_buf_size = size - 2; /* "- 2" to take care of EOB's */ b->yy_buf_pos = b->yy_ch_buf = base; b->yy_is_our_buffer = 0; - b->yy_input_file = NULL; + b->yy_input_file = 0; b->yy_n_chars = b->yy_buf_size; b->yy_is_interactive = 0; b->yy_at_bol = 1; b->yy_fill_buffer = 0; b->yy_buffer_status = YY_BUFFER_NEW; - yy_switch_to_buffer( b ); + ncg_switch_to_buffer(b ); return b; } -/** Setup the input buffer state to scan a string. The next call to yylex() will +/** Setup the input buffer state to scan a string. The next call to ncglex() will * scan from a @e copy of @a str. * @param yystr a NUL-terminated string to scan * * @return the newly allocated buffer state object. * @note If you want to scan bytes that may contain NUL values, then use - * yy_scan_bytes() instead. + * ncg_scan_bytes() instead. */ -YY_BUFFER_STATE yy_scan_string (const char * yystr ) +YY_BUFFER_STATE ncg_scan_string (yyconst char * yystr ) { - return yy_scan_bytes( yystr, (int) strlen(yystr) ); + return ncg_scan_bytes(yystr,strlen(yystr) ); } -/** Setup the input buffer state to scan the given bytes. The next call to yylex() will +/** Setup the input buffer state to scan the given bytes. The next call to ncglex() will * scan from a @e copy of @a bytes. * @param yybytes the byte buffer to scan * @param _yybytes_len the number of bytes in the buffer pointed to by @a bytes. * * @return the newly allocated buffer state object. */ -YY_BUFFER_STATE yy_scan_bytes (const char * yybytes, int _yybytes_len ) +YY_BUFFER_STATE ncg_scan_bytes (yyconst char * yybytes, yy_size_t _yybytes_len ) { YY_BUFFER_STATE b; char *buf; yy_size_t n; - int i; + yy_size_t i; /* Get memory for full buffer, including space for trailing EOB's. */ - n = (yy_size_t) (_yybytes_len + 2); - buf = (char *) yyalloc( n ); + n = _yybytes_len + 2; + buf = (char *) ncgalloc(n ); if ( ! buf ) - YY_FATAL_ERROR( "out of dynamic memory in yy_scan_bytes()" ); + YY_FATAL_ERROR( "out of dynamic memory in ncg_scan_bytes()" ); for ( i = 0; i < _yybytes_len; ++i ) buf[i] = yybytes[i]; buf[_yybytes_len] = buf[_yybytes_len+1] = YY_END_OF_BUFFER_CHAR; - b = yy_scan_buffer( buf, n ); + b = ncg_scan_buffer(buf,n ); if ( ! b ) - YY_FATAL_ERROR( "bad buffer in yy_scan_bytes()" ); + YY_FATAL_ERROR( "bad buffer in ncg_scan_bytes()" ); /* It's okay to grow etc. this buffer, and we should throw it * away when we're done. @@ -3190,9 +2971,9 @@ YY_BUFFER_STATE yy_scan_bytes (const char * yybytes, int _yybytes_len ) #define YY_EXIT_FAILURE 2 #endif -static void yynoreturn yy_fatal_error (const char* msg ) +static void yy_fatal_error (yyconst char* msg ) { - fprintf( stderr, "%s\n", msg ); + (void) fprintf( stderr, "%s\n", msg ); exit( YY_EXIT_FAILURE ); } @@ -3202,14 +2983,14 @@ static void yynoreturn yy_fatal_error (const char* msg ) #define yyless(n) \ do \ { \ - /* Undo effects of setting up yytext. */ \ + /* Undo effects of setting up ncgtext. */ \ int yyless_macro_arg = (n); \ YY_LESS_LINENO(yyless_macro_arg);\ - yytext[yyleng] = (yy_hold_char); \ - (yy_c_buf_p) = yytext + yyless_macro_arg; \ + ncgtext[ncgleng] = (yy_hold_char); \ + (yy_c_buf_p) = ncgtext + yyless_macro_arg; \ (yy_hold_char) = *(yy_c_buf_p); \ *(yy_c_buf_p) = '\0'; \ - yyleng = yyless_macro_arg; \ + ncgleng = yyless_macro_arg; \ } \ while ( 0 ) @@ -3218,126 +2999,126 @@ static void yynoreturn yy_fatal_error (const char* msg ) /** Get the current line number. * */ -int yyget_lineno (void) +int ncgget_lineno (void) { - - return yylineno; + + return ncglineno; } /** Get the input stream. * */ -FILE *yyget_in (void) +FILE *ncgget_in (void) { - return yyin; + return ncgin; } /** Get the output stream. * */ -FILE *yyget_out (void) +FILE *ncgget_out (void) { - return yyout; + return ncgout; } /** Get the length of the current token. * */ -int yyget_leng (void) +yy_size_t ncgget_leng (void) { - return yyleng; + return ncgleng; } /** Get the current token. * */ -char *yyget_text (void) +char *ncgget_text (void) { - return yytext; + return ncgtext; } /** Set the current line number. * @param _line_number line number * */ -void yyset_lineno (int _line_number ) +void ncgset_lineno (int _line_number ) { - yylineno = _line_number; + ncglineno = _line_number; } /** Set the input stream. This does not discard the current * input buffer. * @param _in_str A readable stream. * - * @see yy_switch_to_buffer + * @see ncg_switch_to_buffer */ -void yyset_in (FILE * _in_str ) +void ncgset_in (FILE * _in_str ) { - yyin = _in_str ; + ncgin = _in_str ; } -void yyset_out (FILE * _out_str ) +void ncgset_out (FILE * _out_str ) { - yyout = _out_str ; + ncgout = _out_str ; } -int yyget_debug (void) +int ncgget_debug (void) { - return yy_flex_debug; + return ncg_flex_debug; } -void yyset_debug (int _bdebug ) +void ncgset_debug (int _bdebug ) { - yy_flex_debug = _bdebug ; + ncg_flex_debug = _bdebug ; } static int yy_init_globals (void) { /* Initialization is the same as for the non-reentrant scanner. - * This function is called from yylex_destroy(), so don't allocate here. + * This function is called from ncglex_destroy(), so don't allocate here. */ - (yy_buffer_stack) = NULL; + (yy_buffer_stack) = 0; (yy_buffer_stack_top) = 0; (yy_buffer_stack_max) = 0; - (yy_c_buf_p) = NULL; + (yy_c_buf_p) = (char *) 0; (yy_init) = 0; (yy_start) = 0; /* Defined in main.c */ #ifdef YY_STDINIT - yyin = stdin; - yyout = stdout; + ncgin = stdin; + ncgout = stdout; #else - yyin = NULL; - yyout = NULL; + ncgin = (FILE *) 0; + ncgout = (FILE *) 0; #endif /* For future reference: Set errno on error, since we are called by - * yylex_init() + * ncglex_init() */ return 0; } -/* yylex_destroy is for both reentrant and non-reentrant scanners. */ -int yylex_destroy (void) +/* ncglex_destroy is for both reentrant and non-reentrant scanners. */ +int ncglex_destroy (void) { /* Pop the buffer stack, destroying each element. */ while(YY_CURRENT_BUFFER){ - yy_delete_buffer( YY_CURRENT_BUFFER ); + ncg_delete_buffer(YY_CURRENT_BUFFER ); YY_CURRENT_BUFFER_LVALUE = NULL; - yypop_buffer_state(); + ncgpop_buffer_state(); } /* Destroy the stack itself. */ - yyfree((yy_buffer_stack) ); + ncgfree((yy_buffer_stack) ); (yy_buffer_stack) = NULL; /* Reset the globals. This is important in a non-reentrant scanner so the next time - * yylex() is called, initialization will occur. */ + * ncglex() is called, initialization will occur. */ yy_init_globals( ); return 0; @@ -3348,7 +3129,7 @@ int yylex_destroy (void) */ #ifndef yytext_ptr -static void yy_flex_strncpy (char* s1, const char * s2, int n ) +static void yy_flex_strncpy (char* s1, yyconst char * s2, int n ) { int i; @@ -3358,7 +3139,7 @@ static void yy_flex_strncpy (char* s1, const char * s2, int n ) #endif #ifdef YY_NEED_STRLEN -static int yy_flex_strlen (const char * s ) +static int yy_flex_strlen (yyconst char * s ) { int n; for ( n = 0; s[n]; ++n ) @@ -3368,12 +3149,12 @@ static int yy_flex_strlen (const char * s ) } #endif -void *yyalloc (yy_size_t size ) +void *ncgalloc (yy_size_t size ) { - return malloc(size); + return (void *) malloc( size ); } -void *yyrealloc (void * ptr, yy_size_t size ) +void *ncgrealloc (void * ptr, yy_size_t size ) { /* The cast to (char *) in the following accommodates both @@ -3383,25 +3164,26 @@ void *yyrealloc (void * ptr, yy_size_t size ) * any pointer type to void*, and deal with argument conversions * as though doing an assignment. */ - return realloc(ptr, size); + return (void *) realloc( (char *) ptr, size ); } -void yyfree (void * ptr ) +void ncgfree (void * ptr ) { - free( (char *) ptr ); /* see yyrealloc() for (char *) cast */ + free( (char *) ptr ); /* see ncgrealloc() for (char *) cast */ } #define YYTABLES_NAME "yytables" #line 586 "ncgen.l" + static int lexdebug(int token) { if(debug >= 2) { - char* text = yytext; - text[yyleng] = 0; + char* text = ncgtext; + text[ncgleng] = 0; fprintf(stderr,"Token=%d |%s| line=%d\n",token,text,lineno); } return token; diff --git a/ncgen/ncgeny.c b/ncgen/ncgeny.c index f8630b1946..e427db7f92 100644 --- a/ncgen/ncgeny.c +++ b/ncgen/ncgeny.c @@ -192,6 +192,9 @@ static void vercheck(int ncid); static long long extractint(NCConstant* con); #ifdef USE_NETCDF4 static int parsefilterflag(const char* sdata0, Specialdata* special); +#ifdef GENDEBUG1 +static void printfilters(size_t nfilters, NC_ParsedFilterSpec** filters); +#endif #endif int yylex(void); @@ -206,7 +209,7 @@ static void yyerror(fmt,va_alist) const char* fmt; va_dcl; extern int lex_init(void); -#line 210 "ncgeny.c" /* yacc.c:339 */ +#line 213 "ncgeny.c" /* yacc.c:339 */ # ifndef YY_NULLPTR # if defined __cplusplus && 201103L <= __cplusplus @@ -302,7 +305,7 @@ extern int ncgdebug; union YYSTYPE { -#line 149 "ncgen.y" /* yacc.c:355 */ +#line 152 "ncgen.y" /* yacc.c:355 */ Symbol* sym; unsigned long size; /* allow for zero size to indicate e.g. UNLIMITED*/ @@ -311,7 +314,7 @@ int nctype; /* for tracking attribute list type*/ Datalist* datalist; NCConstant* constant; -#line 315 "ncgeny.c" /* yacc.c:355 */ +#line 318 "ncgeny.c" /* yacc.c:355 */ }; typedef union YYSTYPE YYSTYPE; @@ -328,7 +331,7 @@ int ncgparse (void); /* Copy the second part of user declarations. */ -#line 332 "ncgeny.c" /* yacc.c:358 */ +#line 335 "ncgeny.c" /* yacc.c:358 */ #ifdef short # undef short @@ -631,22 +634,22 @@ static const yytype_uint8 yytranslate[] = /* YYRLINE[YYN] -- Source line where rule number YYN was defined. */ static const yytype_uint16 yyrline[] = { - 0, 231, 231, 237, 239, 246, 253, 253, 256, 265, - 255, 270, 271, 272, 276, 276, 278, 288, 288, 291, - 292, 293, 294, 297, 297, 300, 330, 332, 349, 358, - 370, 384, 417, 418, 421, 435, 436, 437, 438, 439, - 440, 441, 442, 443, 444, 445, 446, 449, 450, 451, - 454, 455, 458, 458, 460, 461, 465, 473, 483, 495, - 496, 497, 500, 501, 504, 504, 506, 531, 535, 539, - 568, 569, 572, 573, 577, 591, 595, 600, 629, 630, - 634, 635, 640, 650, 670, 681, 692, 711, 718, 718, - 721, 723, 725, 727, 729, 738, 749, 751, 753, 755, - 757, 759, 761, 763, 765, 767, 769, 774, 781, 790, - 791, 792, 795, 796, 799, 803, 804, 808, 812, 813, - 818, 819, 823, 824, 825, 826, 827, 828, 832, 836, - 840, 842, 847, 848, 849, 850, 851, 852, 853, 854, - 855, 856, 857, 858, 862, 863, 867, 869, 871, 873, - 878, 882, 883, 889 + 0, 234, 234, 240, 242, 249, 256, 256, 259, 268, + 258, 273, 274, 275, 279, 279, 281, 291, 291, 294, + 295, 296, 297, 300, 300, 303, 333, 335, 352, 361, + 373, 387, 420, 421, 424, 438, 439, 440, 441, 442, + 443, 444, 445, 446, 447, 448, 449, 452, 453, 454, + 457, 458, 461, 461, 463, 464, 468, 476, 486, 498, + 499, 500, 503, 504, 507, 507, 509, 534, 538, 542, + 571, 572, 575, 576, 580, 594, 598, 603, 632, 633, + 637, 638, 643, 653, 673, 684, 695, 714, 721, 721, + 724, 726, 728, 730, 732, 741, 752, 754, 756, 758, + 760, 762, 764, 766, 768, 770, 772, 777, 784, 793, + 794, 795, 798, 799, 802, 806, 807, 811, 815, 816, + 821, 822, 826, 827, 828, 829, 830, 831, 835, 839, + 843, 845, 850, 851, 852, 853, 854, 855, 856, 857, + 858, 859, 860, 861, 865, 866, 870, 872, 874, 876, + 881, 885, 886, 892 }; #endif @@ -1638,19 +1641,19 @@ yyparse (void) switch (yyn) { case 2: -#line 234 "ncgen.y" /* yacc.c:1646 */ +#line 237 "ncgen.y" /* yacc.c:1646 */ {if (error_count > 0) YYABORT;} -#line 1644 "ncgeny.c" /* yacc.c:1646 */ +#line 1647 "ncgeny.c" /* yacc.c:1646 */ break; case 3: -#line 237 "ncgen.y" /* yacc.c:1646 */ +#line 240 "ncgen.y" /* yacc.c:1646 */ {createrootgroup(datasetname);} -#line 1650 "ncgeny.c" /* yacc.c:1646 */ +#line 1653 "ncgeny.c" /* yacc.c:1646 */ break; case 8: -#line 256 "ncgen.y" /* yacc.c:1646 */ +#line 259 "ncgen.y" /* yacc.c:1646 */ { Symbol* id = (yyvsp[-1].sym); markcdf4("Group specification"); @@ -1658,29 +1661,29 @@ yyparse (void) yyerror("duplicate group declaration within parent group for %s", id->name); } -#line 1662 "ncgeny.c" /* yacc.c:1646 */ +#line 1665 "ncgeny.c" /* yacc.c:1646 */ break; case 9: -#line 265 "ncgen.y" /* yacc.c:1646 */ +#line 268 "ncgen.y" /* yacc.c:1646 */ {listpop(groupstack);} -#line 1668 "ncgeny.c" /* yacc.c:1646 */ +#line 1671 "ncgeny.c" /* yacc.c:1646 */ break; case 12: -#line 271 "ncgen.y" /* yacc.c:1646 */ +#line 274 "ncgen.y" /* yacc.c:1646 */ {} -#line 1674 "ncgeny.c" /* yacc.c:1646 */ +#line 1677 "ncgeny.c" /* yacc.c:1646 */ break; case 13: -#line 273 "ncgen.y" /* yacc.c:1646 */ +#line 276 "ncgen.y" /* yacc.c:1646 */ {markcdf4("Type specification");} -#line 1680 "ncgeny.c" /* yacc.c:1646 */ +#line 1683 "ncgeny.c" /* yacc.c:1646 */ break; case 16: -#line 279 "ncgen.y" /* yacc.c:1646 */ +#line 282 "ncgen.y" /* yacc.c:1646 */ { /* Use when defining a type */ (yyvsp[0].sym)->objectclass = NC_TYPE; if(dupobjectcheck(NC_TYPE,(yyvsp[0].sym))) @@ -1688,23 +1691,23 @@ yyparse (void) (yyvsp[0].sym)->name); listpush(typdefs,(void*)(yyvsp[0].sym)); } -#line 1692 "ncgeny.c" /* yacc.c:1646 */ +#line 1695 "ncgeny.c" /* yacc.c:1646 */ break; case 17: -#line 288 "ncgen.y" /* yacc.c:1646 */ +#line 291 "ncgen.y" /* yacc.c:1646 */ {} -#line 1698 "ncgeny.c" /* yacc.c:1646 */ +#line 1701 "ncgeny.c" /* yacc.c:1646 */ break; case 18: -#line 288 "ncgen.y" /* yacc.c:1646 */ +#line 291 "ncgen.y" /* yacc.c:1646 */ {} -#line 1704 "ncgeny.c" /* yacc.c:1646 */ +#line 1707 "ncgeny.c" /* yacc.c:1646 */ break; case 25: -#line 302 "ncgen.y" /* yacc.c:1646 */ +#line 305 "ncgen.y" /* yacc.c:1646 */ { int i; addtogroup((yyvsp[-3].sym)); /* sets prefix*/ @@ -1731,17 +1734,17 @@ yyparse (void) } listsetlength(stack,stackbase);/* remove stack nodes*/ } -#line 1735 "ncgeny.c" /* yacc.c:1646 */ +#line 1738 "ncgeny.c" /* yacc.c:1646 */ break; case 26: -#line 331 "ncgen.y" /* yacc.c:1646 */ +#line 334 "ncgen.y" /* yacc.c:1646 */ {(yyval.mark)=listlength(stack); listpush(stack,(void*)(yyvsp[0].sym));} -#line 1741 "ncgeny.c" /* yacc.c:1646 */ +#line 1744 "ncgeny.c" /* yacc.c:1646 */ break; case 27: -#line 333 "ncgen.y" /* yacc.c:1646 */ +#line 336 "ncgen.y" /* yacc.c:1646 */ { int i; (yyval.mark)=(yyvsp[-2].mark); @@ -1756,22 +1759,22 @@ yyparse (void) } listpush(stack,(void*)(yyvsp[0].sym)); } -#line 1760 "ncgeny.c" /* yacc.c:1646 */ +#line 1763 "ncgeny.c" /* yacc.c:1646 */ break; case 28: -#line 350 "ncgen.y" /* yacc.c:1646 */ +#line 353 "ncgen.y" /* yacc.c:1646 */ { (yyvsp[-2].sym)->objectclass=NC_TYPE; (yyvsp[-2].sym)->subclass=NC_ECONST; (yyvsp[-2].sym)->typ.econst=(yyvsp[0].constant); (yyval.sym)=(yyvsp[-2].sym); } -#line 1771 "ncgeny.c" /* yacc.c:1646 */ +#line 1774 "ncgeny.c" /* yacc.c:1646 */ break; case 29: -#line 359 "ncgen.y" /* yacc.c:1646 */ +#line 362 "ncgen.y" /* yacc.c:1646 */ { vercheck(NC_OPAQUE); addtogroup((yyvsp[0].sym)); /*sets prefix*/ @@ -1781,11 +1784,11 @@ yyparse (void) (yyvsp[0].sym)->typ.size=int32_val; (yyvsp[0].sym)->typ.alignment=ncaux_class_alignment(NC_OPAQUE); } -#line 1785 "ncgeny.c" /* yacc.c:1646 */ +#line 1788 "ncgeny.c" /* yacc.c:1646 */ break; case 30: -#line 371 "ncgen.y" /* yacc.c:1646 */ +#line 374 "ncgen.y" /* yacc.c:1646 */ { Symbol* basetype = (yyvsp[-4].sym); vercheck(NC_VLEN); @@ -1797,11 +1800,11 @@ yyparse (void) (yyvsp[0].sym)->typ.size=VLENSIZE; (yyvsp[0].sym)->typ.alignment=ncaux_class_alignment(NC_VLEN); } -#line 1801 "ncgeny.c" /* yacc.c:1646 */ +#line 1804 "ncgeny.c" /* yacc.c:1646 */ break; case 31: -#line 385 "ncgen.y" /* yacc.c:1646 */ +#line 388 "ncgen.y" /* yacc.c:1646 */ { int i,j; vercheck(NC_COMPOUND); @@ -1831,23 +1834,23 @@ yyparse (void) } listsetlength(stack,stackbase);/* remove stack nodes*/ } -#line 1835 "ncgeny.c" /* yacc.c:1646 */ +#line 1838 "ncgeny.c" /* yacc.c:1646 */ break; case 32: -#line 417 "ncgen.y" /* yacc.c:1646 */ +#line 420 "ncgen.y" /* yacc.c:1646 */ {(yyval.mark)=(yyvsp[-1].mark);} -#line 1841 "ncgeny.c" /* yacc.c:1646 */ +#line 1844 "ncgeny.c" /* yacc.c:1646 */ break; case 33: -#line 418 "ncgen.y" /* yacc.c:1646 */ +#line 421 "ncgen.y" /* yacc.c:1646 */ {(yyval.mark)=(yyvsp[-2].mark);} -#line 1847 "ncgeny.c" /* yacc.c:1646 */ +#line 1850 "ncgeny.c" /* yacc.c:1646 */ break; case 34: -#line 422 "ncgen.y" /* yacc.c:1646 */ +#line 425 "ncgen.y" /* yacc.c:1646 */ { int i; (yyval.mark)=(yyvsp[0].mark); @@ -1859,107 +1862,107 @@ yyparse (void) f->typ.basetype = (yyvsp[-1].sym); } } -#line 1863 "ncgeny.c" /* yacc.c:1646 */ +#line 1866 "ncgeny.c" /* yacc.c:1646 */ break; case 35: -#line 435 "ncgen.y" /* yacc.c:1646 */ +#line 438 "ncgen.y" /* yacc.c:1646 */ { (yyval.sym) = primsymbols[NC_CHAR]; } -#line 1869 "ncgeny.c" /* yacc.c:1646 */ +#line 1872 "ncgeny.c" /* yacc.c:1646 */ break; case 36: -#line 436 "ncgen.y" /* yacc.c:1646 */ +#line 439 "ncgen.y" /* yacc.c:1646 */ { (yyval.sym) = primsymbols[NC_BYTE]; } -#line 1875 "ncgeny.c" /* yacc.c:1646 */ +#line 1878 "ncgeny.c" /* yacc.c:1646 */ break; case 37: -#line 437 "ncgen.y" /* yacc.c:1646 */ +#line 440 "ncgen.y" /* yacc.c:1646 */ { (yyval.sym) = primsymbols[NC_SHORT]; } -#line 1881 "ncgeny.c" /* yacc.c:1646 */ +#line 1884 "ncgeny.c" /* yacc.c:1646 */ break; case 38: -#line 438 "ncgen.y" /* yacc.c:1646 */ +#line 441 "ncgen.y" /* yacc.c:1646 */ { (yyval.sym) = primsymbols[NC_INT]; } -#line 1887 "ncgeny.c" /* yacc.c:1646 */ +#line 1890 "ncgeny.c" /* yacc.c:1646 */ break; case 39: -#line 439 "ncgen.y" /* yacc.c:1646 */ +#line 442 "ncgen.y" /* yacc.c:1646 */ { (yyval.sym) = primsymbols[NC_FLOAT]; } -#line 1893 "ncgeny.c" /* yacc.c:1646 */ +#line 1896 "ncgeny.c" /* yacc.c:1646 */ break; case 40: -#line 440 "ncgen.y" /* yacc.c:1646 */ +#line 443 "ncgen.y" /* yacc.c:1646 */ { (yyval.sym) = primsymbols[NC_DOUBLE]; } -#line 1899 "ncgeny.c" /* yacc.c:1646 */ +#line 1902 "ncgeny.c" /* yacc.c:1646 */ break; case 41: -#line 441 "ncgen.y" /* yacc.c:1646 */ +#line 444 "ncgen.y" /* yacc.c:1646 */ { vercheck(NC_UBYTE); (yyval.sym) = primsymbols[NC_UBYTE]; } -#line 1905 "ncgeny.c" /* yacc.c:1646 */ +#line 1908 "ncgeny.c" /* yacc.c:1646 */ break; case 42: -#line 442 "ncgen.y" /* yacc.c:1646 */ +#line 445 "ncgen.y" /* yacc.c:1646 */ { vercheck(NC_USHORT); (yyval.sym) = primsymbols[NC_USHORT]; } -#line 1911 "ncgeny.c" /* yacc.c:1646 */ +#line 1914 "ncgeny.c" /* yacc.c:1646 */ break; case 43: -#line 443 "ncgen.y" /* yacc.c:1646 */ +#line 446 "ncgen.y" /* yacc.c:1646 */ { vercheck(NC_UINT); (yyval.sym) = primsymbols[NC_UINT]; } -#line 1917 "ncgeny.c" /* yacc.c:1646 */ +#line 1920 "ncgeny.c" /* yacc.c:1646 */ break; case 44: -#line 444 "ncgen.y" /* yacc.c:1646 */ +#line 447 "ncgen.y" /* yacc.c:1646 */ { vercheck(NC_INT64); (yyval.sym) = primsymbols[NC_INT64]; } -#line 1923 "ncgeny.c" /* yacc.c:1646 */ +#line 1926 "ncgeny.c" /* yacc.c:1646 */ break; case 45: -#line 445 "ncgen.y" /* yacc.c:1646 */ +#line 448 "ncgen.y" /* yacc.c:1646 */ { vercheck(NC_UINT64); (yyval.sym) = primsymbols[NC_UINT64]; } -#line 1929 "ncgeny.c" /* yacc.c:1646 */ +#line 1932 "ncgeny.c" /* yacc.c:1646 */ break; case 46: -#line 446 "ncgen.y" /* yacc.c:1646 */ +#line 449 "ncgen.y" /* yacc.c:1646 */ { vercheck(NC_STRING); (yyval.sym) = primsymbols[NC_STRING]; } -#line 1935 "ncgeny.c" /* yacc.c:1646 */ +#line 1938 "ncgeny.c" /* yacc.c:1646 */ break; case 48: -#line 450 "ncgen.y" /* yacc.c:1646 */ +#line 453 "ncgen.y" /* yacc.c:1646 */ {} -#line 1941 "ncgeny.c" /* yacc.c:1646 */ +#line 1944 "ncgeny.c" /* yacc.c:1646 */ break; case 49: -#line 451 "ncgen.y" /* yacc.c:1646 */ +#line 454 "ncgen.y" /* yacc.c:1646 */ {} -#line 1947 "ncgeny.c" /* yacc.c:1646 */ +#line 1950 "ncgeny.c" /* yacc.c:1646 */ break; case 52: -#line 458 "ncgen.y" /* yacc.c:1646 */ +#line 461 "ncgen.y" /* yacc.c:1646 */ {} -#line 1953 "ncgeny.c" /* yacc.c:1646 */ +#line 1956 "ncgeny.c" /* yacc.c:1646 */ break; case 53: -#line 458 "ncgen.y" /* yacc.c:1646 */ +#line 461 "ncgen.y" /* yacc.c:1646 */ {} -#line 1959 "ncgeny.c" /* yacc.c:1646 */ +#line 1962 "ncgeny.c" /* yacc.c:1646 */ break; case 56: -#line 466 "ncgen.y" /* yacc.c:1646 */ +#line 469 "ncgen.y" /* yacc.c:1646 */ { (yyvsp[-2].sym)->dim.declsize = (size_t)extractint((yyvsp[0].constant)); #ifdef GENDEBUG1 @@ -1967,11 +1970,11 @@ fprintf(stderr,"dimension: %s = %llu\n",(yyvsp[-2].sym)->name,(unsigned long lon #endif reclaimconstant((yyvsp[0].constant)); } -#line 1971 "ncgeny.c" /* yacc.c:1646 */ +#line 1974 "ncgeny.c" /* yacc.c:1646 */ break; case 57: -#line 474 "ncgen.y" /* yacc.c:1646 */ +#line 477 "ncgen.y" /* yacc.c:1646 */ { (yyvsp[-2].sym)->dim.declsize = NC_UNLIMITED; (yyvsp[-2].sym)->dim.isunlimited = 1; @@ -1979,11 +1982,11 @@ fprintf(stderr,"dimension: %s = %llu\n",(yyvsp[-2].sym)->name,(unsigned long lon fprintf(stderr,"dimension: %s = UNLIMITED\n",(yyvsp[-2].sym)->name); #endif } -#line 1983 "ncgeny.c" /* yacc.c:1646 */ +#line 1986 "ncgeny.c" /* yacc.c:1646 */ break; case 58: -#line 484 "ncgen.y" /* yacc.c:1646 */ +#line 487 "ncgen.y" /* yacc.c:1646 */ { (yyvsp[0].sym)->objectclass=NC_DIM; if(dupobjectcheck(NC_DIM,(yyvsp[0].sym))) @@ -1993,35 +1996,35 @@ fprintf(stderr,"dimension: %s = UNLIMITED\n",(yyvsp[-2].sym)->name); (yyval.sym)=(yyvsp[0].sym); listpush(dimdefs,(void*)(yyvsp[0].sym)); } -#line 1997 "ncgeny.c" /* yacc.c:1646 */ +#line 2000 "ncgeny.c" /* yacc.c:1646 */ break; case 60: -#line 496 "ncgen.y" /* yacc.c:1646 */ +#line 499 "ncgen.y" /* yacc.c:1646 */ {} -#line 2003 "ncgeny.c" /* yacc.c:1646 */ +#line 2006 "ncgeny.c" /* yacc.c:1646 */ break; case 61: -#line 497 "ncgen.y" /* yacc.c:1646 */ +#line 500 "ncgen.y" /* yacc.c:1646 */ {} -#line 2009 "ncgeny.c" /* yacc.c:1646 */ +#line 2012 "ncgeny.c" /* yacc.c:1646 */ break; case 64: -#line 504 "ncgen.y" /* yacc.c:1646 */ +#line 507 "ncgen.y" /* yacc.c:1646 */ {} -#line 2015 "ncgeny.c" /* yacc.c:1646 */ +#line 2018 "ncgeny.c" /* yacc.c:1646 */ break; case 65: -#line 504 "ncgen.y" /* yacc.c:1646 */ +#line 507 "ncgen.y" /* yacc.c:1646 */ {} -#line 2021 "ncgeny.c" /* yacc.c:1646 */ +#line 2024 "ncgeny.c" /* yacc.c:1646 */ break; case 66: -#line 507 "ncgen.y" /* yacc.c:1646 */ +#line 510 "ncgen.y" /* yacc.c:1646 */ { int i; stackbase=(yyvsp[0].mark); @@ -2044,25 +2047,25 @@ fprintf(stderr,"dimension: %s = UNLIMITED\n",(yyvsp[-2].sym)->name); } listsetlength(stack,stackbase);/* remove stack nodes*/ } -#line 2048 "ncgeny.c" /* yacc.c:1646 */ +#line 2051 "ncgeny.c" /* yacc.c:1646 */ break; case 67: -#line 532 "ncgen.y" /* yacc.c:1646 */ +#line 535 "ncgen.y" /* yacc.c:1646 */ {(yyval.mark)=listlength(stack); listpush(stack,(void*)(yyvsp[0].sym)); } -#line 2056 "ncgeny.c" /* yacc.c:1646 */ +#line 2059 "ncgeny.c" /* yacc.c:1646 */ break; case 68: -#line 536 "ncgen.y" /* yacc.c:1646 */ +#line 539 "ncgen.y" /* yacc.c:1646 */ {(yyval.mark)=(yyvsp[-2].mark); listpush(stack,(void*)(yyvsp[0].sym));} -#line 2062 "ncgeny.c" /* yacc.c:1646 */ +#line 2065 "ncgeny.c" /* yacc.c:1646 */ break; case 69: -#line 540 "ncgen.y" /* yacc.c:1646 */ +#line 543 "ncgen.y" /* yacc.c:1646 */ { int i; Dimset dimset; @@ -2089,35 +2092,35 @@ fprintf(stderr,"dimension: %s = UNLIMITED\n",(yyvsp[-2].sym)->name); listsetlength(stack,stackbase);/* remove stack nodes*/ (yyval.sym) = var; } -#line 2093 "ncgeny.c" /* yacc.c:1646 */ +#line 2096 "ncgeny.c" /* yacc.c:1646 */ break; case 70: -#line 568 "ncgen.y" /* yacc.c:1646 */ +#line 571 "ncgen.y" /* yacc.c:1646 */ {(yyval.mark)=listlength(stack);} -#line 2099 "ncgeny.c" /* yacc.c:1646 */ +#line 2102 "ncgeny.c" /* yacc.c:1646 */ break; case 71: -#line 569 "ncgen.y" /* yacc.c:1646 */ +#line 572 "ncgen.y" /* yacc.c:1646 */ {(yyval.mark)=(yyvsp[-1].mark);} -#line 2105 "ncgeny.c" /* yacc.c:1646 */ +#line 2108 "ncgeny.c" /* yacc.c:1646 */ break; case 72: -#line 572 "ncgen.y" /* yacc.c:1646 */ +#line 575 "ncgen.y" /* yacc.c:1646 */ {(yyval.mark)=listlength(stack); listpush(stack,(void*)(yyvsp[0].sym));} -#line 2111 "ncgeny.c" /* yacc.c:1646 */ +#line 2114 "ncgeny.c" /* yacc.c:1646 */ break; case 73: -#line 574 "ncgen.y" /* yacc.c:1646 */ +#line 577 "ncgen.y" /* yacc.c:1646 */ {(yyval.mark)=(yyvsp[-2].mark); listpush(stack,(void*)(yyvsp[0].sym));} -#line 2117 "ncgeny.c" /* yacc.c:1646 */ +#line 2120 "ncgeny.c" /* yacc.c:1646 */ break; case 74: -#line 578 "ncgen.y" /* yacc.c:1646 */ +#line 581 "ncgen.y" /* yacc.c:1646 */ {Symbol* dimsym = (yyvsp[0].sym); dimsym->objectclass = NC_DIM; /* Find the actual dimension*/ @@ -2128,25 +2131,25 @@ fprintf(stderr,"dimension: %s = UNLIMITED\n",(yyvsp[-2].sym)->name); } (yyval.sym)=dimsym; } -#line 2132 "ncgeny.c" /* yacc.c:1646 */ +#line 2135 "ncgeny.c" /* yacc.c:1646 */ break; case 75: -#line 592 "ncgen.y" /* yacc.c:1646 */ +#line 595 "ncgen.y" /* yacc.c:1646 */ {(yyval.mark)=listlength(stack); listpush(stack,(void*)(yyvsp[0].sym)); } -#line 2140 "ncgeny.c" /* yacc.c:1646 */ +#line 2143 "ncgeny.c" /* yacc.c:1646 */ break; case 76: -#line 596 "ncgen.y" /* yacc.c:1646 */ +#line 599 "ncgen.y" /* yacc.c:1646 */ {(yyval.mark)=(yyvsp[-2].mark); listpush(stack,(void*)(yyvsp[0].sym));} -#line 2146 "ncgeny.c" /* yacc.c:1646 */ +#line 2149 "ncgeny.c" /* yacc.c:1646 */ break; case 77: -#line 601 "ncgen.y" /* yacc.c:1646 */ +#line 604 "ncgen.y" /* yacc.c:1646 */ { int i; Dimset dimset; @@ -2173,35 +2176,35 @@ fprintf(stderr,"dimension: %s = UNLIMITED\n",(yyvsp[-2].sym)->name); listsetlength(stack,stackbase);/* remove stack nodes*/ (yyval.sym) = (yyvsp[-1].sym); } -#line 2177 "ncgeny.c" /* yacc.c:1646 */ +#line 2180 "ncgeny.c" /* yacc.c:1646 */ break; case 78: -#line 629 "ncgen.y" /* yacc.c:1646 */ +#line 632 "ncgen.y" /* yacc.c:1646 */ {(yyval.mark)=listlength(stack);} -#line 2183 "ncgeny.c" /* yacc.c:1646 */ +#line 2186 "ncgeny.c" /* yacc.c:1646 */ break; case 79: -#line 630 "ncgen.y" /* yacc.c:1646 */ +#line 633 "ncgen.y" /* yacc.c:1646 */ {(yyval.mark)=(yyvsp[-1].mark);} -#line 2189 "ncgeny.c" /* yacc.c:1646 */ +#line 2192 "ncgeny.c" /* yacc.c:1646 */ break; case 80: -#line 634 "ncgen.y" /* yacc.c:1646 */ +#line 637 "ncgen.y" /* yacc.c:1646 */ {(yyval.mark)=listlength(stack); listpush(stack,(void*)(yyvsp[0].sym));} -#line 2195 "ncgeny.c" /* yacc.c:1646 */ +#line 2198 "ncgeny.c" /* yacc.c:1646 */ break; case 81: -#line 636 "ncgen.y" /* yacc.c:1646 */ +#line 639 "ncgen.y" /* yacc.c:1646 */ {(yyval.mark)=(yyvsp[-2].mark); listpush(stack,(void*)(yyvsp[0].sym));} -#line 2201 "ncgeny.c" /* yacc.c:1646 */ +#line 2204 "ncgeny.c" /* yacc.c:1646 */ break; case 82: -#line 641 "ncgen.y" /* yacc.c:1646 */ +#line 644 "ncgen.y" /* yacc.c:1646 */ { /* Anonymous integer dimension. Can only occur in type definitions*/ char anon[32]; @@ -2211,11 +2214,11 @@ fprintf(stderr,"dimension: %s = UNLIMITED\n",(yyvsp[-2].sym)->name); (yyval.sym)->dim.isconstant = 1; (yyval.sym)->dim.declsize = uint32_val; } -#line 2215 "ncgeny.c" /* yacc.c:1646 */ +#line 2218 "ncgeny.c" /* yacc.c:1646 */ break; case 83: -#line 651 "ncgen.y" /* yacc.c:1646 */ +#line 654 "ncgen.y" /* yacc.c:1646 */ { /* Anonymous integer dimension. Can only occur in type definitions*/ char anon[32]; @@ -2229,11 +2232,11 @@ fprintf(stderr,"dimension: %s = UNLIMITED\n",(yyvsp[-2].sym)->name); (yyval.sym)->dim.isconstant = 1; (yyval.sym)->dim.declsize = int32_val; } -#line 2233 "ncgeny.c" /* yacc.c:1646 */ +#line 2236 "ncgeny.c" /* yacc.c:1646 */ break; case 84: -#line 671 "ncgen.y" /* yacc.c:1646 */ +#line 674 "ncgen.y" /* yacc.c:1646 */ {Symbol* vsym = (yyvsp[0].sym); if(vsym->objectclass != NC_VAR) { derror("Undefined or forward referenced variable: %s",vsym->name); @@ -2241,11 +2244,11 @@ fprintf(stderr,"dimension: %s = UNLIMITED\n",(yyvsp[-2].sym)->name); } (yyval.sym)=vsym; } -#line 2245 "ncgeny.c" /* yacc.c:1646 */ +#line 2248 "ncgeny.c" /* yacc.c:1646 */ break; case 85: -#line 682 "ncgen.y" /* yacc.c:1646 */ +#line 685 "ncgen.y" /* yacc.c:1646 */ {Symbol* tsym = (yyvsp[0].sym); if(tsym->objectclass != NC_TYPE) { derror("Undefined or forward referenced type: %s",tsym->name); @@ -2253,11 +2256,11 @@ fprintf(stderr,"dimension: %s = UNLIMITED\n",(yyvsp[-2].sym)->name); } (yyval.sym)=tsym; } -#line 2257 "ncgeny.c" /* yacc.c:1646 */ +#line 2260 "ncgeny.c" /* yacc.c:1646 */ break; case 86: -#line 693 "ncgen.y" /* yacc.c:1646 */ +#line 696 "ncgen.y" /* yacc.c:1646 */ {Symbol* tvsym = (yyvsp[0].sym); Symbol* sym; /* disambiguate*/ tvsym->objectclass = NC_VAR; @@ -2276,53 +2279,53 @@ fprintf(stderr,"dimension: %s = UNLIMITED\n",(yyvsp[-2].sym)->name); } (yyval.sym)=tvsym; } -#line 2280 "ncgeny.c" /* yacc.c:1646 */ +#line 2283 "ncgeny.c" /* yacc.c:1646 */ break; case 87: -#line 711 "ncgen.y" /* yacc.c:1646 */ +#line 714 "ncgen.y" /* yacc.c:1646 */ {(yyval.sym)=(yyvsp[0].sym);} -#line 2286 "ncgeny.c" /* yacc.c:1646 */ +#line 2289 "ncgeny.c" /* yacc.c:1646 */ break; case 88: -#line 718 "ncgen.y" /* yacc.c:1646 */ +#line 721 "ncgen.y" /* yacc.c:1646 */ {} -#line 2292 "ncgeny.c" /* yacc.c:1646 */ +#line 2295 "ncgeny.c" /* yacc.c:1646 */ break; case 89: -#line 718 "ncgen.y" /* yacc.c:1646 */ +#line 721 "ncgen.y" /* yacc.c:1646 */ {} -#line 2298 "ncgeny.c" /* yacc.c:1646 */ +#line 2301 "ncgeny.c" /* yacc.c:1646 */ break; case 90: -#line 722 "ncgen.y" /* yacc.c:1646 */ +#line 725 "ncgen.y" /* yacc.c:1646 */ {(yyval.sym) = makespecial(_NCPROPS_FLAG,NULL,NULL,(void*)(yyvsp[0].constant),ISCONST);} -#line 2304 "ncgeny.c" /* yacc.c:1646 */ +#line 2307 "ncgeny.c" /* yacc.c:1646 */ break; case 91: -#line 724 "ncgen.y" /* yacc.c:1646 */ +#line 727 "ncgen.y" /* yacc.c:1646 */ {(yyval.sym) = makespecial(_ISNETCDF4_FLAG,NULL,NULL,(void*)(yyvsp[0].constant),ISCONST);} -#line 2310 "ncgeny.c" /* yacc.c:1646 */ +#line 2313 "ncgeny.c" /* yacc.c:1646 */ break; case 92: -#line 726 "ncgen.y" /* yacc.c:1646 */ +#line 729 "ncgen.y" /* yacc.c:1646 */ {(yyval.sym) = makespecial(_SUPERBLOCK_FLAG,NULL,NULL,(void*)(yyvsp[0].constant),ISCONST);} -#line 2316 "ncgeny.c" /* yacc.c:1646 */ +#line 2319 "ncgeny.c" /* yacc.c:1646 */ break; case 93: -#line 728 "ncgen.y" /* yacc.c:1646 */ +#line 731 "ncgen.y" /* yacc.c:1646 */ { (yyval.sym)=makeattribute((yyvsp[-2].sym),NULL,NULL,(yyvsp[0].datalist),ATTRGLOBAL);} -#line 2322 "ncgeny.c" /* yacc.c:1646 */ +#line 2325 "ncgeny.c" /* yacc.c:1646 */ break; case 94: -#line 730 "ncgen.y" /* yacc.c:1646 */ +#line 733 "ncgen.y" /* yacc.c:1646 */ {Symbol* tsym = (yyvsp[-5].sym); Symbol* vsym = (yyvsp[-4].sym); Symbol* asym = (yyvsp[-2].sym); if(vsym->objectclass == NC_VAR) { (yyval.sym)=makeattribute(asym,vsym,tsym,(yyvsp[0].datalist),ATTRVAR); @@ -2331,11 +2334,11 @@ fprintf(stderr,"dimension: %s = UNLIMITED\n",(yyvsp[-2].sym)->name); YYABORT; } } -#line 2335 "ncgeny.c" /* yacc.c:1646 */ +#line 2338 "ncgeny.c" /* yacc.c:1646 */ break; case 95: -#line 739 "ncgen.y" /* yacc.c:1646 */ +#line 742 "ncgen.y" /* yacc.c:1646 */ {Symbol* sym = (yyvsp[-4].sym); Symbol* asym = (yyvsp[-2].sym); if(sym->objectclass == NC_VAR) { (yyval.sym)=makeattribute(asym,sym,NULL,(yyvsp[0].datalist),ATTRVAR); @@ -2346,345 +2349,345 @@ fprintf(stderr,"dimension: %s = UNLIMITED\n",(yyvsp[-2].sym)->name); YYABORT; } } -#line 2350 "ncgeny.c" /* yacc.c:1646 */ +#line 2353 "ncgeny.c" /* yacc.c:1646 */ break; case 96: -#line 750 "ncgen.y" /* yacc.c:1646 */ +#line 753 "ncgen.y" /* yacc.c:1646 */ {(yyval.sym) = makespecial(_FILLVALUE_FLAG,(yyvsp[-4].sym),NULL,(void*)(yyvsp[0].datalist),ISLIST);} -#line 2356 "ncgeny.c" /* yacc.c:1646 */ +#line 2359 "ncgeny.c" /* yacc.c:1646 */ break; case 97: -#line 752 "ncgen.y" /* yacc.c:1646 */ +#line 755 "ncgen.y" /* yacc.c:1646 */ {(yyval.sym) = makespecial(_FILLVALUE_FLAG,(yyvsp[-4].sym),(yyvsp[-5].sym),(void*)(yyvsp[0].datalist),ISLIST);} -#line 2362 "ncgeny.c" /* yacc.c:1646 */ +#line 2365 "ncgeny.c" /* yacc.c:1646 */ break; case 98: -#line 754 "ncgen.y" /* yacc.c:1646 */ +#line 757 "ncgen.y" /* yacc.c:1646 */ {(yyval.sym) = makespecial(_STORAGE_FLAG,(yyvsp[-4].sym),NULL,(void*)(yyvsp[0].constant),ISCONST);} -#line 2368 "ncgeny.c" /* yacc.c:1646 */ +#line 2371 "ncgeny.c" /* yacc.c:1646 */ break; case 99: -#line 756 "ncgen.y" /* yacc.c:1646 */ +#line 759 "ncgen.y" /* yacc.c:1646 */ {(yyval.sym) = makespecial(_CHUNKSIZES_FLAG,(yyvsp[-4].sym),NULL,(void*)(yyvsp[0].datalist),ISLIST);} -#line 2374 "ncgeny.c" /* yacc.c:1646 */ +#line 2377 "ncgeny.c" /* yacc.c:1646 */ break; case 100: -#line 758 "ncgen.y" /* yacc.c:1646 */ +#line 761 "ncgen.y" /* yacc.c:1646 */ {(yyval.sym) = makespecial(_FLETCHER32_FLAG,(yyvsp[-4].sym),NULL,(void*)(yyvsp[0].constant),ISCONST);} -#line 2380 "ncgeny.c" /* yacc.c:1646 */ +#line 2383 "ncgeny.c" /* yacc.c:1646 */ break; case 101: -#line 760 "ncgen.y" /* yacc.c:1646 */ +#line 763 "ncgen.y" /* yacc.c:1646 */ {(yyval.sym) = makespecial(_DEFLATE_FLAG,(yyvsp[-4].sym),NULL,(void*)(yyvsp[0].constant),ISCONST);} -#line 2386 "ncgeny.c" /* yacc.c:1646 */ +#line 2389 "ncgeny.c" /* yacc.c:1646 */ break; case 102: -#line 762 "ncgen.y" /* yacc.c:1646 */ +#line 765 "ncgen.y" /* yacc.c:1646 */ {(yyval.sym) = makespecial(_SHUFFLE_FLAG,(yyvsp[-4].sym),NULL,(void*)(yyvsp[0].constant),ISCONST);} -#line 2392 "ncgeny.c" /* yacc.c:1646 */ +#line 2395 "ncgeny.c" /* yacc.c:1646 */ break; case 103: -#line 764 "ncgen.y" /* yacc.c:1646 */ +#line 767 "ncgen.y" /* yacc.c:1646 */ {(yyval.sym) = makespecial(_ENDIAN_FLAG,(yyvsp[-4].sym),NULL,(void*)(yyvsp[0].constant),ISCONST);} -#line 2398 "ncgeny.c" /* yacc.c:1646 */ +#line 2401 "ncgeny.c" /* yacc.c:1646 */ break; case 104: -#line 766 "ncgen.y" /* yacc.c:1646 */ +#line 769 "ncgen.y" /* yacc.c:1646 */ {(yyval.sym) = makespecial(_FILTER_FLAG,(yyvsp[-4].sym),NULL,(void*)(yyvsp[0].constant),ISCONST);} -#line 2404 "ncgeny.c" /* yacc.c:1646 */ +#line 2407 "ncgeny.c" /* yacc.c:1646 */ break; case 105: -#line 768 "ncgen.y" /* yacc.c:1646 */ +#line 771 "ncgen.y" /* yacc.c:1646 */ {(yyval.sym) = makespecial(_NOFILL_FLAG,(yyvsp[-4].sym),NULL,(void*)(yyvsp[0].constant),ISCONST);} -#line 2410 "ncgeny.c" /* yacc.c:1646 */ +#line 2413 "ncgeny.c" /* yacc.c:1646 */ break; case 106: -#line 770 "ncgen.y" /* yacc.c:1646 */ +#line 773 "ncgen.y" /* yacc.c:1646 */ {(yyval.sym) = makespecial(_FORMAT_FLAG,NULL,NULL,(void*)(yyvsp[0].constant),ISCONST);} -#line 2416 "ncgeny.c" /* yacc.c:1646 */ +#line 2419 "ncgeny.c" /* yacc.c:1646 */ break; case 107: -#line 775 "ncgen.y" /* yacc.c:1646 */ +#line 778 "ncgen.y" /* yacc.c:1646 */ { (yyval.sym)=(yyvsp[0].sym); (yyvsp[0].sym)->ref.is_ref=1; (yyvsp[0].sym)->is_prefixed=0; setpathcurrent((yyvsp[0].sym)); } -#line 2427 "ncgeny.c" /* yacc.c:1646 */ +#line 2430 "ncgeny.c" /* yacc.c:1646 */ break; case 108: -#line 782 "ncgen.y" /* yacc.c:1646 */ +#line 785 "ncgen.y" /* yacc.c:1646 */ { (yyval.sym)=(yyvsp[0].sym); (yyvsp[0].sym)->ref.is_ref=1; (yyvsp[0].sym)->is_prefixed=1; /* path is set in ncgen.l*/ } -#line 2438 "ncgeny.c" /* yacc.c:1646 */ +#line 2441 "ncgeny.c" /* yacc.c:1646 */ break; case 110: -#line 791 "ncgen.y" /* yacc.c:1646 */ +#line 794 "ncgen.y" /* yacc.c:1646 */ {} -#line 2444 "ncgeny.c" /* yacc.c:1646 */ +#line 2447 "ncgeny.c" /* yacc.c:1646 */ break; case 111: -#line 792 "ncgen.y" /* yacc.c:1646 */ +#line 795 "ncgen.y" /* yacc.c:1646 */ {} -#line 2450 "ncgeny.c" /* yacc.c:1646 */ +#line 2453 "ncgeny.c" /* yacc.c:1646 */ break; case 114: -#line 800 "ncgen.y" /* yacc.c:1646 */ +#line 803 "ncgen.y" /* yacc.c:1646 */ {(yyvsp[-2].sym)->data = (yyvsp[0].datalist);} -#line 2456 "ncgeny.c" /* yacc.c:1646 */ +#line 2459 "ncgeny.c" /* yacc.c:1646 */ break; case 115: -#line 803 "ncgen.y" /* yacc.c:1646 */ +#line 806 "ncgen.y" /* yacc.c:1646 */ {(yyval.datalist) = (yyvsp[0].datalist);} -#line 2462 "ncgeny.c" /* yacc.c:1646 */ +#line 2465 "ncgeny.c" /* yacc.c:1646 */ break; case 116: -#line 804 "ncgen.y" /* yacc.c:1646 */ +#line 807 "ncgen.y" /* yacc.c:1646 */ {(yyval.datalist) = (yyvsp[0].datalist);} -#line 2468 "ncgeny.c" /* yacc.c:1646 */ +#line 2471 "ncgeny.c" /* yacc.c:1646 */ break; case 117: -#line 808 "ncgen.y" /* yacc.c:1646 */ +#line 811 "ncgen.y" /* yacc.c:1646 */ {(yyval.datalist) = builddatalist(0);} -#line 2474 "ncgeny.c" /* yacc.c:1646 */ +#line 2477 "ncgeny.c" /* yacc.c:1646 */ break; case 118: -#line 812 "ncgen.y" /* yacc.c:1646 */ +#line 815 "ncgen.y" /* yacc.c:1646 */ {(yyval.datalist) = const2list((yyvsp[0].constant));} -#line 2480 "ncgeny.c" /* yacc.c:1646 */ +#line 2483 "ncgeny.c" /* yacc.c:1646 */ break; case 119: -#line 814 "ncgen.y" /* yacc.c:1646 */ +#line 817 "ncgen.y" /* yacc.c:1646 */ {dlappend((yyvsp[-2].datalist),((yyvsp[0].constant))); (yyval.datalist)=(yyvsp[-2].datalist); } -#line 2486 "ncgeny.c" /* yacc.c:1646 */ +#line 2489 "ncgeny.c" /* yacc.c:1646 */ break; case 120: -#line 818 "ncgen.y" /* yacc.c:1646 */ +#line 821 "ncgen.y" /* yacc.c:1646 */ {(yyval.constant)=(yyvsp[0].constant);} -#line 2492 "ncgeny.c" /* yacc.c:1646 */ +#line 2495 "ncgeny.c" /* yacc.c:1646 */ break; case 121: -#line 819 "ncgen.y" /* yacc.c:1646 */ +#line 822 "ncgen.y" /* yacc.c:1646 */ {(yyval.constant)=builddatasublist((yyvsp[-1].datalist));} -#line 2498 "ncgeny.c" /* yacc.c:1646 */ +#line 2501 "ncgeny.c" /* yacc.c:1646 */ break; case 122: -#line 823 "ncgen.y" /* yacc.c:1646 */ +#line 826 "ncgen.y" /* yacc.c:1646 */ {(yyval.constant)=(yyvsp[0].constant);} -#line 2504 "ncgeny.c" /* yacc.c:1646 */ +#line 2507 "ncgeny.c" /* yacc.c:1646 */ break; case 123: -#line 824 "ncgen.y" /* yacc.c:1646 */ +#line 827 "ncgen.y" /* yacc.c:1646 */ {(yyval.constant)=makeconstdata(NC_OPAQUE);} -#line 2510 "ncgeny.c" /* yacc.c:1646 */ +#line 2513 "ncgeny.c" /* yacc.c:1646 */ break; case 124: -#line 825 "ncgen.y" /* yacc.c:1646 */ +#line 828 "ncgen.y" /* yacc.c:1646 */ {(yyval.constant)=makeconstdata(NC_FILLVALUE);} -#line 2516 "ncgeny.c" /* yacc.c:1646 */ +#line 2519 "ncgeny.c" /* yacc.c:1646 */ break; case 125: -#line 826 "ncgen.y" /* yacc.c:1646 */ +#line 829 "ncgen.y" /* yacc.c:1646 */ {(yyval.constant)=makeconstdata(NC_NIL);} -#line 2522 "ncgeny.c" /* yacc.c:1646 */ +#line 2525 "ncgeny.c" /* yacc.c:1646 */ break; case 126: -#line 827 "ncgen.y" /* yacc.c:1646 */ +#line 830 "ncgen.y" /* yacc.c:1646 */ {(yyval.constant)=(yyvsp[0].constant);} -#line 2528 "ncgeny.c" /* yacc.c:1646 */ +#line 2531 "ncgeny.c" /* yacc.c:1646 */ break; case 128: -#line 832 "ncgen.y" /* yacc.c:1646 */ +#line 835 "ncgen.y" /* yacc.c:1646 */ {(yyval.constant) = makeenumconstref((yyvsp[0].sym));} -#line 2534 "ncgeny.c" /* yacc.c:1646 */ +#line 2537 "ncgeny.c" /* yacc.c:1646 */ break; case 129: -#line 836 "ncgen.y" /* yacc.c:1646 */ +#line 839 "ncgen.y" /* yacc.c:1646 */ {(yyval.constant)=evaluate((yyvsp[-3].sym),(yyvsp[-1].datalist));} -#line 2540 "ncgeny.c" /* yacc.c:1646 */ +#line 2543 "ncgeny.c" /* yacc.c:1646 */ break; case 130: -#line 841 "ncgen.y" /* yacc.c:1646 */ +#line 844 "ncgen.y" /* yacc.c:1646 */ {(yyval.datalist) = const2list((yyvsp[0].constant));} -#line 2546 "ncgeny.c" /* yacc.c:1646 */ +#line 2549 "ncgeny.c" /* yacc.c:1646 */ break; case 131: -#line 843 "ncgen.y" /* yacc.c:1646 */ +#line 846 "ncgen.y" /* yacc.c:1646 */ {dlappend((yyvsp[-2].datalist),((yyvsp[0].constant))); (yyval.datalist)=(yyvsp[-2].datalist);} -#line 2552 "ncgeny.c" /* yacc.c:1646 */ +#line 2555 "ncgeny.c" /* yacc.c:1646 */ break; case 132: -#line 847 "ncgen.y" /* yacc.c:1646 */ +#line 850 "ncgen.y" /* yacc.c:1646 */ {(yyval.constant)=makeconstdata(NC_CHAR);} -#line 2558 "ncgeny.c" /* yacc.c:1646 */ +#line 2561 "ncgeny.c" /* yacc.c:1646 */ break; case 133: -#line 848 "ncgen.y" /* yacc.c:1646 */ +#line 851 "ncgen.y" /* yacc.c:1646 */ {(yyval.constant)=makeconstdata(NC_BYTE);} -#line 2564 "ncgeny.c" /* yacc.c:1646 */ +#line 2567 "ncgeny.c" /* yacc.c:1646 */ break; case 134: -#line 849 "ncgen.y" /* yacc.c:1646 */ +#line 852 "ncgen.y" /* yacc.c:1646 */ {(yyval.constant)=makeconstdata(NC_SHORT);} -#line 2570 "ncgeny.c" /* yacc.c:1646 */ +#line 2573 "ncgeny.c" /* yacc.c:1646 */ break; case 135: -#line 850 "ncgen.y" /* yacc.c:1646 */ +#line 853 "ncgen.y" /* yacc.c:1646 */ {(yyval.constant)=makeconstdata(NC_INT);} -#line 2576 "ncgeny.c" /* yacc.c:1646 */ +#line 2579 "ncgeny.c" /* yacc.c:1646 */ break; case 136: -#line 851 "ncgen.y" /* yacc.c:1646 */ +#line 854 "ncgen.y" /* yacc.c:1646 */ {(yyval.constant)=makeconstdata(NC_INT64);} -#line 2582 "ncgeny.c" /* yacc.c:1646 */ +#line 2585 "ncgeny.c" /* yacc.c:1646 */ break; case 137: -#line 852 "ncgen.y" /* yacc.c:1646 */ +#line 855 "ncgen.y" /* yacc.c:1646 */ {(yyval.constant)=makeconstdata(NC_UBYTE);} -#line 2588 "ncgeny.c" /* yacc.c:1646 */ +#line 2591 "ncgeny.c" /* yacc.c:1646 */ break; case 138: -#line 853 "ncgen.y" /* yacc.c:1646 */ +#line 856 "ncgen.y" /* yacc.c:1646 */ {(yyval.constant)=makeconstdata(NC_USHORT);} -#line 2594 "ncgeny.c" /* yacc.c:1646 */ +#line 2597 "ncgeny.c" /* yacc.c:1646 */ break; case 139: -#line 854 "ncgen.y" /* yacc.c:1646 */ +#line 857 "ncgen.y" /* yacc.c:1646 */ {(yyval.constant)=makeconstdata(NC_UINT);} -#line 2600 "ncgeny.c" /* yacc.c:1646 */ +#line 2603 "ncgeny.c" /* yacc.c:1646 */ break; case 140: -#line 855 "ncgen.y" /* yacc.c:1646 */ +#line 858 "ncgen.y" /* yacc.c:1646 */ {(yyval.constant)=makeconstdata(NC_UINT64);} -#line 2606 "ncgeny.c" /* yacc.c:1646 */ +#line 2609 "ncgeny.c" /* yacc.c:1646 */ break; case 141: -#line 856 "ncgen.y" /* yacc.c:1646 */ +#line 859 "ncgen.y" /* yacc.c:1646 */ {(yyval.constant)=makeconstdata(NC_FLOAT);} -#line 2612 "ncgeny.c" /* yacc.c:1646 */ +#line 2615 "ncgeny.c" /* yacc.c:1646 */ break; case 142: -#line 857 "ncgen.y" /* yacc.c:1646 */ +#line 860 "ncgen.y" /* yacc.c:1646 */ {(yyval.constant)=makeconstdata(NC_DOUBLE);} -#line 2618 "ncgeny.c" /* yacc.c:1646 */ +#line 2621 "ncgeny.c" /* yacc.c:1646 */ break; case 143: -#line 858 "ncgen.y" /* yacc.c:1646 */ +#line 861 "ncgen.y" /* yacc.c:1646 */ {(yyval.constant)=makeconstdata(NC_STRING);} -#line 2624 "ncgeny.c" /* yacc.c:1646 */ +#line 2627 "ncgeny.c" /* yacc.c:1646 */ break; case 144: -#line 862 "ncgen.y" /* yacc.c:1646 */ +#line 865 "ncgen.y" /* yacc.c:1646 */ {(yyval.datalist) = const2list((yyvsp[0].constant));} -#line 2630 "ncgeny.c" /* yacc.c:1646 */ +#line 2633 "ncgeny.c" /* yacc.c:1646 */ break; case 145: -#line 863 "ncgen.y" /* yacc.c:1646 */ +#line 866 "ncgen.y" /* yacc.c:1646 */ {(yyval.datalist)=(yyvsp[-2].datalist); dlappend((yyvsp[-2].datalist),((yyvsp[0].constant)));} -#line 2636 "ncgeny.c" /* yacc.c:1646 */ +#line 2639 "ncgeny.c" /* yacc.c:1646 */ break; case 146: -#line 868 "ncgen.y" /* yacc.c:1646 */ +#line 871 "ncgen.y" /* yacc.c:1646 */ {(yyval.constant)=makeconstdata(NC_INT);} -#line 2642 "ncgeny.c" /* yacc.c:1646 */ +#line 2645 "ncgeny.c" /* yacc.c:1646 */ break; case 147: -#line 870 "ncgen.y" /* yacc.c:1646 */ +#line 873 "ncgen.y" /* yacc.c:1646 */ {(yyval.constant)=makeconstdata(NC_UINT);} -#line 2648 "ncgeny.c" /* yacc.c:1646 */ +#line 2651 "ncgeny.c" /* yacc.c:1646 */ break; case 148: -#line 872 "ncgen.y" /* yacc.c:1646 */ +#line 875 "ncgen.y" /* yacc.c:1646 */ {(yyval.constant)=makeconstdata(NC_INT64);} -#line 2654 "ncgeny.c" /* yacc.c:1646 */ +#line 2657 "ncgeny.c" /* yacc.c:1646 */ break; case 149: -#line 874 "ncgen.y" /* yacc.c:1646 */ +#line 877 "ncgen.y" /* yacc.c:1646 */ {(yyval.constant)=makeconstdata(NC_UINT64);} -#line 2660 "ncgeny.c" /* yacc.c:1646 */ +#line 2663 "ncgeny.c" /* yacc.c:1646 */ break; case 150: -#line 878 "ncgen.y" /* yacc.c:1646 */ +#line 881 "ncgen.y" /* yacc.c:1646 */ {(yyval.constant)=makeconstdata(NC_STRING);} -#line 2666 "ncgeny.c" /* yacc.c:1646 */ +#line 2669 "ncgeny.c" /* yacc.c:1646 */ break; case 151: -#line 882 "ncgen.y" /* yacc.c:1646 */ +#line 885 "ncgen.y" /* yacc.c:1646 */ {(yyval.constant)=(yyvsp[0].constant);} -#line 2672 "ncgeny.c" /* yacc.c:1646 */ +#line 2675 "ncgeny.c" /* yacc.c:1646 */ break; case 152: -#line 883 "ncgen.y" /* yacc.c:1646 */ +#line 886 "ncgen.y" /* yacc.c:1646 */ {(yyval.constant)=(yyvsp[0].constant);} -#line 2678 "ncgeny.c" /* yacc.c:1646 */ +#line 2681 "ncgeny.c" /* yacc.c:1646 */ break; case 153: -#line 889 "ncgen.y" /* yacc.c:1646 */ +#line 892 "ncgen.y" /* yacc.c:1646 */ {(yyval.sym)=(yyvsp[0].sym);} -#line 2684 "ncgeny.c" /* yacc.c:1646 */ +#line 2687 "ncgeny.c" /* yacc.c:1646 */ break; -#line 2688 "ncgeny.c" /* yacc.c:1646 */ +#line 2691 "ncgeny.c" /* yacc.c:1646 */ default: break; } /* User semantic actions sometimes alter yychar, and that requires @@ -2912,7 +2915,7 @@ fprintf(stderr,"dimension: %s = UNLIMITED\n",(yyvsp[-2].sym)->name); #endif return yyresult; } -#line 892 "ncgen.y" /* yacc.c:1906 */ +#line 895 "ncgen.y" /* yacc.c:1906 */ #ifndef NO_STDARG @@ -3387,7 +3390,6 @@ makespecial(int tag, Symbol* vsym, Symbol* tsym, void* data, int isconst) if(parsefilterflag(sdata,special) == NC_NOERR) special->flags |= _FILTER_FLAG; else { - efree(special->_FilterParams); derror("_Filter: unparsable filter spec: %s",sdata); } #else @@ -3509,12 +3511,18 @@ static int parsefilterflag(const char* sdata, Specialdata* special) { int stat = NC_NOERR; + int format; if(sdata == NULL || strlen(sdata) == 0) return NC_EINVAL; - stat = NC_parsefilterspec(sdata, &special->_FilterID, &special->nparams, &special->_FilterParams); + stat = NC_parsefilterlist(sdata, &format, &special->nfilters, (NC_Filterspec***)&special->_Filters); if(stat) derror("Malformed filter spec: %s",sdata); + if(format != NC_FILTER_FORMAT_HDF5) + derror("Non HDF5 filter format encountered"); +#ifdef GENDEBUG1 +printfilters(special->nfilters,special->_Filters); +#endif return stat; } #endif @@ -3590,3 +3598,31 @@ evaluate(Symbol* fcn, Datalist* arglist) done: return result; } + +#ifdef GENDEBUG1 +static void +printfilters(size_t nfilters, NC_ParsedFilterSpec** filters) +{ + int i; + fprintf(stderr,"xxx: nfilters=%lu: ",(unsigned long)nfilters); + for(i=0;ifilterid,sp->format,(unsigned long)sp->nparams,sp->params); + if(sp->nparams == 0 || sp->params != NULL) { + fprintf(stderr," params={"); + for(k=0;knparams;k++) { + if(i==0) fprintf(stderr,","); + fprintf(stderr,"%u",sp->params[i]); + } + fprintf(stderr,"}"); + } else + fprintf(stderr,"params=NULL"); + fprintf(stderr,"}"); + } + fprintf(stderr,"\n"); + fflush(stderr); +} +#endif diff --git a/ncgen/ncgeny.h b/ncgen/ncgeny.h index daee848901..92e376b062 100644 --- a/ncgen/ncgeny.h +++ b/ncgen/ncgeny.h @@ -106,7 +106,7 @@ extern int ncgdebug; union YYSTYPE { -#line 149 "ncgen.y" /* yacc.c:1909 */ +#line 152 "ncgen.y" /* yacc.c:1909 */ Symbol* sym; unsigned long size; /* allow for zero size to indicate e.g. UNLIMITED*/ diff --git a/ncgen/util.c b/ncgen/util.c index dfd14a1978..0b5e3886aa 100644 --- a/ncgen/util.c +++ b/ncgen/util.c @@ -117,8 +117,15 @@ freeSpecialdata(Specialdata* data) reclaimdatalist(data->_Fillvalue); if(data->_ChunkSizes) efree(data->_ChunkSizes); - if(data->_FilterParams) - efree(data->_FilterParams); + if(data->_Filters) { + int i; + for(i=0;infilters;i++) { + NC4_Filterspec* f = (NC4_Filterspec*)data->_Filters[i]; + efree(f->params); + efree(f); + } + efree(data->_Filters); + } efree(data); } diff --git a/plugins/CMakeLists.txt b/plugins/CMakeLists.txt index 2ae7e5e447..e2940fa97e 100644 --- a/plugins/CMakeLists.txt +++ b/plugins/CMakeLists.txt @@ -10,6 +10,8 @@ SET(libh5bzip2_SOURCES blocksort.c huffman.c crctable.c randtable.c compress.c d SET(libmisc_SOURCES H5Zmisc.c H5Zutil.c h5misc.h) +SET(libnoop_SOURCES H5Znoop.c H5Zutil.c h5noop.h) + IF(ENABLE_FILTER_TESTING) IF(BUILD_UTILITIES) @@ -36,6 +38,12 @@ SET_TARGET_PROPERTIES(misc PROPERTIES ARCHIVE_OUTPUT_NAME "misc") SET_TARGET_PROPERTIES(misc PROPERTIES RUNTIME_OUTPUT_NAME "misc") TARGET_LINK_LIBRARIES(misc ${ALL_TLL_LIBS}) +ADD_LIBRARY(noop MODULE ${libnoop_SOURCES}) +SET_TARGET_PROPERTIES(noop PROPERTIES LIBRARY_OUTPUT_NAME "noop") +SET_TARGET_PROPERTIES(noop PROPERTIES ARCHIVE_OUTPUT_NAME "noop") +SET_TARGET_PROPERTIES(noop PROPERTIES RUNTIME_OUTPUT_NAME "noop") +TARGET_LINK_LIBRARIES(noop ${ALL_TLL_LIBS}) + ENDIF(BUILD_UTILITIES) ENDIF(ENABLE_FILTER_TESTING) diff --git a/plugins/H5Znoop.c b/plugins/H5Znoop.c new file mode 100644 index 0000000000..3b357e8360 --- /dev/null +++ b/plugins/H5Znoop.c @@ -0,0 +1,117 @@ +#include "config.h" +#include +#include +#include +#include +#include +#include +/* Older versions of the hdf library may define H5PL_type_t here */ +#include +#include "h5noop.h" + +#ifndef DLL_EXPORT +#define DLL_EXPORT +#endif + +/* WARNING: +Starting with HDF5 version 1.10.x, the plugin code MUST be +careful when using the standard *malloc()*, *realloc()*, and +*free()* function. + +In the event that the code is allocating, reallocating, for +free'ing memory that either came from or will be exported to the +calling HDF5 library, then one MUST use the corresponding HDF5 +functions *H5allocate_memory()*, *H5resize_memory()*, +*H5free_memory()* [5] to avoid memory failures. + +Additionally, if your filter code leaks memory, then the HDF5 library +will generate an error. + +*/ + +#undef DEBUG + +extern void NC_filterfix8(void* mem, int decode); + +const H5Z_class2_t H5Z_NOOP[1] = {{ + H5Z_CLASS_T_VERS, /* H5Z_class_t version */ + (H5Z_filter_t)(H5Z_FILTER_NOOP), /* Filter id number */ + 1, /* encoder_present flag (set to true) */ + 1, /* decoder_present flag (set to true) */ + "noop", /* Filter name for debugging */ + (H5Z_can_apply_func_t)H5Z_noop_can_apply, /* The "can apply" callback */ + NULL, /* The "set local" callback */ + (H5Z_func_t)H5Z_filter_noop, /* The actual filter function */ +}}; + +/* External Discovery Functions */ +H5PL_type_t +H5PLget_plugin_type(void) +{ + return H5PL_TYPE_FILTER; +} + +const void* +H5PLget_plugin_info(void) +{ + return H5Z_NOOP; +} + +/* Make this explicit */ +/* + * The "can_apply" callback returns positive a valid combination, zero for an + * invalid combination and negative for an error. + */ +htri_t +H5Z_noop_can_apply(hid_t dcpl_id, hid_t type_id, hid_t space_id) +{ + return 1; /* Assume it can always apply */ +} + +size_t +H5Z_filter_noop(unsigned int flags, size_t cd_nelmts, + const unsigned int cd_values[], size_t nbytes, + size_t *buf_size, void **buf) +{ + void* newbuf; + + if (flags & H5Z_FLAG_REVERSE) { + + /* Replace buffer */ +#ifdef HAVE_H5ALLOCATE_MEMORY + newbuf = H5allocate_memory(*buf_size,0); +#else + newbuf = malloc(*buf_size); +#endif + if(newbuf == NULL) abort(); + memcpy(newbuf,*buf,*buf_size); + /* reclaim old buffer */ +#ifdef HAVE_H5FREE_MEMORY + H5free_memory(*buf); +#else + free(*buf); +#endif + *buf = newbuf; + + } else { + + /* Replace buffer */ +#ifdef HAVE_H5ALLOCATE_MEMORY + newbuf = H5allocate_memory(*buf_size,0); +#else + newbuf = malloc(*buf_size); +#endif + if(newbuf == NULL) abort(); + memcpy(newbuf,*buf,*buf_size); + /* reclaim old buffer */ +#ifdef HAVE_H5FREE_MEMORY + H5free_memory(*buf); +#else + free(*buf); +#endif + *buf = newbuf; + + } + + return *buf_size; +} diff --git a/plugins/Makefile.am b/plugins/Makefile.am index ddd2ddfcf9..c0c6207256 100644 --- a/plugins/Makefile.am +++ b/plugins/Makefile.am @@ -11,7 +11,7 @@ PLUGINSRC=H5Zbzip2.c PLUGINHDRS=h5bzip2.h EXTRA_DIST=${PLUGINSRC} ${BZIP2SRC} ${PLUGINHDRS} ${BZIP2HDRS} \ - H5Ztemplate.c H5Zmisc.c H5Zutil.c CMakeLists.txt + H5Ztemplate.c H5Zmisc.c H5Zutil.c H5Znoop.c CMakeLists.txt # WARNING: This list must be kept consistent with the corresponding # AC_CONFIG_LINK commands near the end of configure.ac. @@ -19,7 +19,7 @@ HDF5PLUGINSRC=${PLUGINSRC} ${BZIP2SRC} ${PLUGINHDRS} ${BZIP2HDRS} if ENABLE_FILTER_TESTING -noinst_LTLIBRARIES = libmisc.la +noinst_LTLIBRARIES = libmisc.la libnoop.la lib_LTLIBRARIES = libh5bzip2.la libh5bzip2_la_SOURCES = ${HDF5PLUGINSRC} @@ -28,4 +28,8 @@ libh5bzip2_la_LDFLAGS = -module -avoid-version -shared -export-dynamic -no-undef libmisc_la_SOURCES = H5Zmisc.c H5Zutil.c h5misc.h libmisc_la_LDFLAGS = -module -avoid-version -shared -export-dynamic -no-undefined -rpath ${abs_builddir} +# The noop filter is to allow testing of multifilters +libnoop_la_SOURCES = H5Znoop.c H5Zutil.c h5noop.h +libnoop_la_LDFLAGS = -module -avoid-version -shared -export-dynamic -no-undefined -rpath ${abs_builddir} + endif #ENABLE_FILTER_TESTING diff --git a/plugins/h5noop.h b/plugins/h5noop.h new file mode 100644 index 0000000000..627692380c --- /dev/null +++ b/plugins/h5noop.h @@ -0,0 +1,27 @@ +#ifndef H5MISC_H +#define H5MISC_H + +#ifdef _WIN32 + #ifdef DLL_EXPORT /* define when building the library */ + #define DECLSPEC __declspec(dllexport) + #else + #define DECLSPEC __declspec(dllimport) + #endif +#else + #define DECLSPEC extern +#endif + +/* use an integer greater than 256 to be id of the registered filter. */ +#define H5Z_FILTER_NOOP 40000 + +/* declare the hdf5 interface */ +DECLSPEC H5PL_type_t H5PLget_plugin_type(void); +DECLSPEC const void* H5PLget_plugin_info(void); +DECLSPEC const H5Z_class2_t H5Z_NOOP[1]; + +/* Declare filter specific functions */ +DECLSPEC htri_t H5Z_noop_can_apply(hid_t dcpl_id, hid_t type_id, hid_t space_id); +DECLSPEC size_t H5Z_filter_noop(unsigned flags,size_t cd_nelmts,const unsigned cd_values[], + size_t nbytes,size_t *buf_size,void**buf); + +#endif /*H5MISC_H*/