Adding nc_def_var_quantize()/nc_inq_var_quantize()

CMakeLists.txt

@@ -37,7 +37,7 @@ SET(PACKAGE_VERSION ${VERSION})
 
 # Version of the dispatch table. This must match the value in
 # configure.ac.
-SET(NC_DISPATCH_VERSION 3)
+SET(NC_DISPATCH_VERSION 4)
 
 # Get system configuration, Use it to determine osname, os release, cpu. These
 # will be used when committing to CDash.
@@ -1404,6 +1404,9 @@ ENDIF()
 # Always enable DISKLESS
 OPTION(ENABLE_DISKLESS "Enable in-memory files" ON)
 
+# Always enable quantization.
+OPTION(ENABLE_QUANTIZE "Enable variable quantization" ON)
+
 # By default, MSVC has a stack size of 1000000.
 # Allow a user to override this.
 IF(MSVC)
@@ -2180,6 +2183,7 @@ is_enabled(ENABLE_NCZARR HAS_NCZARR)
 is_enabled(ENABLE_NCZARR_S3_TESTS DO_NCZARR_S3_TESTS)
 is_enabled(ENABLE_MULTIFILTERS HAS_MULTIFILTERS)
 is_enabled(ENABLE_NCZARR_ZIP DO_NCZARR_ZIP_TESTS)
+is_enabled(ENABLE_QUANTIZE HAS_QUANTIZE)
 
 # Generate file from template.
 CONFIGURE_FILE("${CMAKE_CURRENT_SOURCE_DIR}/libnetcdf.settings.in"

configure.ac

@@ -1649,6 +1649,7 @@ AC_SUBST(HAS_NCZARR,[$enable_nczarr])
 AC_SUBST(DO_NCZARR_S3_TESTS,[$enable_nczarr_s3_tests])
 AC_SUBST(HAS_MULTIFILTERS,[$has_multifilters])
 AC_SUBST(DO_NCZARR_ZIP_TESTS,[$enable_nczarr_zip])
+AC_SUBST([HAS_QUANTIZE],[yes])
 
 # Include some specifics for netcdf on windows.
 #AH_VERBATIM([_WIN32_STRICMP],
@@ -1728,7 +1729,7 @@ AX_SET_META([NC_HAS_MULTIFILTERS],[$has_multifilters],[yes])
 # dispatch table to submit. If this is changed, make sure the value in
 # CMakeLists.txt also changes to match.
 
-AC_SUBST([NC_DISPATCH_VERSION], [3])
+AC_SUBST([NC_DISPATCH_VERSION], [4])
 AC_DEFINE_UNQUOTED([NC_DISPATCH_VERSION], [${NC_DISPATCH_VERSION}], [Dispatch table version.])
 
 #####

include/hdf5dispatch.h

@@ -47,7 +47,8 @@ extern "C" {
                          int *shufflep, int *deflatep, int *deflate_levelp,
                          int *fletcher32p, int *contiguousp, size_t *chunksizesp,
                          int *no_fill, void *fill_valuep, int *endiannessp,
-                         unsigned int *idp, size_t *nparamsp, unsigned int *params);
+                         unsigned int *idp, size_t *nparamsp, unsigned int *params,
+			 int *quantize_modep, int *nsdp);
 
     EXTERNL int
     NC4_HDF5_set_var_chunk_cache(int ncid, int varid, size_t size, size_t nelems,

include/hdf5internal.h

@@ -200,6 +200,10 @@ int NC4_hdf5_addfilter(NC_VAR_INFO_T* var, unsigned int id, size_t nparams, cons
 int NC4_hdf5_filter_freelist(NC_VAR_INFO_T* var);
 int NC4_hdf5_find_missing_filter(NC_VAR_INFO_T* var, unsigned int* idp);
 
+/* Add an attribute to the attribute list. */
+int nc4_put_att(NC_GRP_INFO_T* grp, int varid, const char *name, nc_type file_type,
+		size_t len, const void *data, nc_type mem_type, int force);
+
 /* Support functions for provenance info (defined in nc4hdf.c) */
 extern int NC4_hdf5get_libversion(unsigned*,unsigned*,unsigned*);/*libsrc4/nc4hdf.c*/
 extern int NC4_hdf5get_superblock(struct NC_FILE_INFO*, int*);/*libsrc4/nc4hdf.c*/

include/nc4dispatch.h

@@ -101,7 +101,8 @@ extern "C" {
                     int *shufflep, int *deflatep, int *deflate_levelp,
                     int *fletcher32p, int *contiguousp, size_t *chunksizesp,
                     int *no_fill, void *fill_valuep, int *endiannessp,
-                    unsigned int* idp, size_t* nparamsp, unsigned int* params);
+                    unsigned int* idp, size_t* nparamsp, unsigned int* params,
+		    int *quantize_modep, int *nsdp);
 
     EXTERNL int
     NC4_inq_varid(int ncid, const char *name, int *varidp);
@@ -251,6 +252,13 @@ extern "C" {
     EXTERNL int
     NC4_inq_var_filter_info(int ncid, int varid, unsigned int id, size_t* nparams, unsigned int* params);
 
+    EXTERNL int
+    NC4_def_var_quantize(int ncid, int varid, int quantize_mode, int nsd);
+
+    EXTERNL int
+    NC4_inq_var_quantize(int ncid, int varid, int *quantize_modep, int *nsdp);
+
+
 #if defined(__cplusplus)
 }
 #endif

include/nc4internal.h

@@ -204,6 +204,8 @@ typedef struct NC_VAR_INFO
     size_t chunk_cache_size;     /**< Size in bytes of the var chunk chache. */
     size_t chunk_cache_nelems;   /**< Number of slots in var chunk cache. */
     float chunk_cache_preemption; /**< Chunk cache preemtion policy. */
+    int quantize_mode;           /**< Quantize mode. NC_NOQUANTIZE is 0, and means no quantization. */
+    int nsd;                     /**< Number of significant digits if quantization is used, 0 if not. */
     void *format_var_info;       /**< Pointer to any binary format info. */
     void* filters;             /**< Record of the list of filters to be applied to var data; format dependent */
 } NC_VAR_INFO_T;

include/ncdispatch.h

@@ -249,7 +249,8 @@ NCDISPATCH_inq_var_all(int ncid, int varid, char *name, nc_type *xtypep,
                int *shufflep, int *deflatep, int *deflate_levelp,
                int *fletcher32p, int *contiguousp, size_t *chunksizesp,
                int *no_fill, void *fill_valuep, int *endiannessp,
-	       unsigned int* idp, size_t* nparamsp, unsigned int* paramsp
+	       unsigned int* idp, size_t* nparamsp, unsigned int* paramsp,
+	       int *quantize_modep, int *nsdp
                );
 EXTERNL int
 NCDISPATCH_get_att(int ncid, int varid, const char* name, void* value, nc_type t);

include/netcdf.h

@@ -324,6 +324,13 @@ there. */
 #define NC_MIN_DEFLATE_LEVEL 0 /**< Minimum deflate level. */
 #define NC_MAX_DEFLATE_LEVEL 9 /**< Maximum deflate level. */
 
+#define NC_NOQUANTIZE 0 /**< No quantization in use. */    
+#define NC_QUANTIZE_BITGROOM 1 /**< Use bitgroom quantization. */
+
+/** When quantization is used for a variable, an attribute of this
+ * name is added. */
+#define NC_QUANTIZE_ATT_NAME "number_of_significant_digits" 
+
 /** The netcdf version 3 functions all return integer error status.
  * These are the possible values, in addition to certain values from
  * the system errno.h.
@@ -854,6 +861,16 @@ nc_get_varm(int ncid, int varid,  const size_t *startp,
 
 /* Extra netcdf-4 stuff. */
 
+/* Set quantization settings for a variable. Quantizing data improves
+ * later compression. Must be called after nc_def_var and before
+ * nc_enddef. */
+EXTERNL int
+nc_def_var_quantize(int ncid, int varid, int quantize_mode, int nsd);
+
+/* Find out quantization settings of a var. */
+EXTERNL int
+nc_inq_var_quantize(int ncid, int varid, int *quantize_modep, int *nsdp);
+
 /* Set compression settings for a variable. Lower is faster, higher is
  * better. Must be called after nc_def_var and before nc_enddef. */
 EXTERNL int

include/netcdf_dispatch.h.in

@@ -93,7 +93,7 @@ struct NC_Dispatch
                        int *fletcher32p, int *contiguousp, size_t *chunksizesp,
                        int *no_fill, void *fill_valuep, int *endiannessp,
                        unsigned int *idp, size_t *nparamsp,
-                       unsigned int *params);
+                       unsigned int *params, int *quantize_modep, int *nsdp);
 
     int (*var_par_access)(int, int, int);
     int (*def_var_fill)(int, int, int, const void *);
@@ -147,6 +147,7 @@ struct NC_Dispatch
     /* Version 3 Replace filteractions with more specific functions */
     int (*inq_var_filter_ids)(int ncid, int varid, size_t* nfilters, unsigned int* filterids);
     int (*inq_var_filter_info)(int ncid, int varid, unsigned int id, size_t* nparams, unsigned int* params);
+    int (*def_var_quantize)(int ncid, int varid, int quantize_mode, int nsd);
 };
 
 #if defined(__cplusplus)
@@ -223,6 +224,7 @@ extern "C" {
     EXTERNL int NC_NOTNC4_inq_typeids(int, int *, int *);
     EXTERNL int NC_NOTNC4_inq_user_type(int, nc_type, char *, size_t *,
                                         nc_type *, size_t *, int *);
+    EXTERNL int NC_NOTNC4_def_var_quantize(int, int,  int, int);
 
     /* These functions are for dispatch layers that don't implement
      * the enhanced model, but want to succeed anyway.

include/netcdf_meta.h.in

@@ -62,5 +62,6 @@
 #define NC_HAS_PAR_FILTERS @NC_HAS_PAR_FILTERS@ /* Parallel I/O with filter support. */
 #define NC_HAS_NCZARR      @NC_HAS_NCZARR@
 #define NC_HAS_MULTIFILTERS   @NC_HAS_MULTIFILTERS@
+#define NC_HAS_QUANTIZE    @NC_HAS_QUANTIZE@
 
 #endif

libdap2/ncd2dispatch.c

@@ -178,6 +178,7 @@ NCD2_get_var_chunk_cache,
 NC_NOOP_inq_var_filter_ids,
 NC_NOOP_inq_var_filter_info,
 
+NC_NOTNC4_def_var_quantize,
 };
 
 const NC_Dispatch* NCD2_dispatch_table = NULL; /* moved here from ddispatch.c */
@@ -2441,8 +2442,9 @@ NCD2_inq_var_all(int ncid, int varid, char *name, nc_type* xtypep,
                int* shufflep, int* deflatep, int* deflate_levelp,
                int* fletcher32p, int* contiguousp, size_t* chunksizesp,
                int* no_fill, void* fill_valuep, int* endiannessp,
-	       unsigned int* idp, size_t* nparamsp, unsigned int* params
-               )
+		 unsigned int* idp, size_t* nparamsp, unsigned int* params,
+		 int *quantize_modep, int *nsdp
+    )
 {
     NC* drno;
     int ret;
@@ -2452,7 +2454,7 @@ NCD2_inq_var_all(int ncid, int varid, char *name, nc_type* xtypep,
                shufflep, deflatep, deflate_levelp,
                fletcher32p, contiguousp, chunksizesp,
                no_fill, fill_valuep, endiannessp,
-	       idp,nparamsp,params
+               idp,nparamsp,params,quantize_modep, nsdp
 	       );
     return THROW(ret);
 }

libdap2/ncd2dispatch.h

@@ -124,21 +124,23 @@ NCD2_def_var(int ncid, const char *name,
 
 extern int
 NCD2_inq_var_all(int ncid, int varid, char *name, nc_type *xtypep,
-             int *ndimsp, int *dimidsp, int *nattsp,
-             int *shufflep, int *deflatep, int *deflate_levelp,
-             int *fletcher32p, int *contiguousp, size_t *chunksizesp,
-             int *no_fill, void *fill_valuep, int *endiannessp,
-	       unsigned int* idp, size_t* nparamsp, unsigned int* params
-             );
+		 int *ndimsp, int *dimidsp, int *nattsp,
+		 int *shufflep, int *deflatep, int *deflate_levelp,
+		 int *fletcher32p, int *contiguousp, size_t *chunksizesp,
+		 int *no_fill, void *fill_valuep, int *endiannessp,
+		 unsigned int* idp, size_t* nparamsp, unsigned int* params,
+		 int *quantize_modep, int *nsdp
+    );
 
 extern int
 NC3_inq_var_all(int ncid, int varid, char *name, nc_type *xtypep,
-             int *ndimsp, int *dimidsp, int *nattsp,
-             int *shufflep, int *deflatep, int *deflate_levelp,
-             int *fletcher32p, int *contiguousp, size_t *chunksizesp,
-             int *no_fill, void *fill_valuep, int *endiannessp,
-	       unsigned int* idp, size_t* nparamsp, unsigned int* params
-             );
+		int *ndimsp, int *dimidsp, int *nattsp,
+		int *shufflep, int *deflatep, int *deflate_levelp,
+		int *fletcher32p, int *contiguousp, size_t *chunksizesp,
+		int *no_fill, void *fill_valuep, int *endiannessp,
+		unsigned int* idp, size_t* nparamsp, unsigned int* params,
+		int *quantize_modep, int *nsdp		
+    );
 
 extern int
 NCD2_inq_varid(int ncid, const char *name, int *varidp);

libdap4/ncd4dispatch.c

@@ -456,8 +456,9 @@ NCD4_inq_var_all(int ncid, int varid, char *name, nc_type* xtypep,
                int* shufflep, int* deflatep, int* deflate_levelp,
                int* fletcher32p, int* contiguousp, size_t* chunksizesp,
                int* no_fill, void* fill_valuep, int* endiannessp,
-	       unsigned int* idp, size_t* nparamsp, unsigned int* params
-               )
+		 unsigned int* idp, size_t* nparamsp, unsigned int* params,
+		 int *quantize_modep, int *nsdp
+    )
 {
     NC* ncp;
     int ret;
@@ -469,7 +470,7 @@ NCD4_inq_var_all(int ncid, int varid, char *name, nc_type* xtypep,
                shufflep, deflatep, deflate_levelp,
                fletcher32p, contiguousp, chunksizesp,
                no_fill, fill_valuep, endiannessp,
-               idp, nparamsp, params);
+		 idp, nparamsp, params, quantize_modep, nsdp);
     return (ret);
 }
 
@@ -974,4 +975,5 @@ NCD4_get_var_chunk_cache,
 
 NC_NOTNC4_inq_var_filter_ids,
 NC_NOTNC4_inq_var_filter_info,
+NC_NOTNC4_def_var_quantize,
 };

libdispatch/dinternal.c

@@ -16,12 +16,12 @@ in libdap2.
 
 int
 NCDISPATCH_inq_var_all(int ncid, int varid, char *name, nc_type *xtypep,
-               int *ndimsp, int *dimidsp, int *nattsp,
-               int *shufflep, int *deflatep, int *deflate_levelp,
-               int *fletcher32p, int *contiguousp, size_t *chunksizesp,
-               int *no_fill, void *fill_valuep, int *endiannessp,
-	       unsigned int* idp, size_t* nparamsp, unsigned int* params
-               )
+		       int *ndimsp, int *dimidsp, int *nattsp,
+		       int *shufflep, int *deflatep, int *deflate_levelp,
+		       int *fletcher32p, int *contiguousp, size_t *chunksizesp,
+		       int *no_fill, void *fill_valuep, int *endiannessp,
+		       unsigned int* idp, size_t* nparamsp, unsigned int* params,
+		       int *quantize_modep, int *nsdp)
 {
    NC* ncp;
    int stat = NC_check_id(ncid,&ncp);
@@ -33,7 +33,7 @@ NCDISPATCH_inq_var_all(int ncid, int varid, char *name, nc_type *xtypep,
       contiguousp, chunksizesp,
       no_fill, fill_valuep,
       endiannessp,
-      idp, nparamsp, params);
+      idp, nparamsp, params, quantize_modep, nsdp);
 }
 
 int

libdispatch/dnotnc4.c

@@ -19,6 +19,23 @@
 #include "ncdispatch.h"
 #include "nc4internal.h"
 
+/**
+ * @internal Not implemented in some dispatch tables
+ *
+ * @param ncid Ignored.
+ * @param varid Ignored.
+ * @param quantize_mode Ignored.
+ * @param nsd Ignored.
+ *
+ * @return ::NC_ENOTNC4 Not implemented for a dispatch table
+ * @author Ed Hartnett
+ */
+int
+NC_NOTNC4_def_var_quantize(int ncid, int varid,  int quantize_mode, int nsd)
+{
+    return NC_ENOTNC4;
+}
+
 /**
  * @internal Not implemented in some dispatch tables
  *

libdispatch/dvar.c

@@ -461,6 +461,69 @@ nc_def_var_deflate(int ncid, int varid, int shuffle, int deflate, int deflate_le
     return ncp->dispatch->def_var_deflate(ncid,varid,shuffle,deflate,deflate_level);
 }
 
+/**
+   Turn on quantization for a variable.
+
+   The data data are quantized by setting unneeded bits alternately to
+   1/0, so that they may compress well. Quantization is lossy (data
+   are irretrievably altered), and it improves the compression ratio
+   provided by a subsequent lossless compression filter. Quantization
+   alone will not reduce the size of the data - lossless compression
+   like zlib must also be used (see nc_def_var_deflate()).
+
+   This data quantization used the bitgroom algorithm. A notable
+   feature of BitGroom is that the data it processes remain in IEEE754
+   format after quantization. Therefore the BitGroom algorithm does
+   nothing when data are read.
+
+   Quantization is only available for variables of type NC_FLOAT or
+   NC_DOUBLE. Attempts to set quantization for other variable
+   types return an error (NC_EINVAL). 
+
+   Quantization is not applied to values equal to the value of the
+   _FillValue attribute, if any. 
+
+   For more information about quantization and the bitgroom filter, see 
+
+   Zender, C. S. (2016), Bit Grooming: Statistically accurate
+   precision-preserving quantization with compression, evaluated in
+   the netCDF Operators (NCO, v4.4.8+), Geosci. Model Dev., 9,
+   3199-3211, doi:10.5194/gmd-9-3199-2016 Retrieved on Sep 21, 2020
+   from
+   https://www.researchgate.net/publication/301575383_Bit_Grooming_Statistically_accurate_precision-preserving_quantization_with_compression_evaluated_in_the_netCDF_Operators_NCO_v448.
+
+   @param ncid File ID.
+   @param varid Variable ID. NC_GLOBAL is not a valid varid, and may
+   not be used.
+   @param quantize_mode A integer flag specifying the quantization
+   used. Current NC_QUANTIZE_BITGROOM is the only available setting.
+   @param nsd Number of significant digits to retain. Allowed single- and
+   double-precision NSDs are 1-7 and 1-15, respectively.
+
+   @return ::NC_NOERR No error.
+   @return ::NC_EGLOBAL Can't use ::NC_GLOBAL with this function.
+   @return ::NC_EBADID Bad ncid.
+   @return ::NC_ENOTVAR Invalid variable ID.
+   @return ::NC_ENOTNC4 Attempting netcdf-4 operation on file that is
+   not netCDF-4/HDF5.
+   @return ::NC_ESTRICTNC3 Attempting netcdf-4 operation on strict nc3
+   netcdf-4 file.
+   @return ::NC_ELATEDEF Too late to change settings for this variable.
+   @return ::NC_EINVAL Invalid input
+   @author Charlie Zender, Ed Hartnett
+ */
+int
+nc_def_var_quantize(int ncid, int varid, int quantize_mode, int nsd)
+{
+    NC* ncp;
+    int stat = NC_check_id(ncid,&ncp);
+    if(stat != NC_NOERR) return stat;
+
+    /* Using NC_GLOBAL is illegal. */
+    if (varid == NC_GLOBAL) return NC_EGLOBAL;
+    return ncp->dispatch->def_var_quantize(ncid,varid,quantize_mode,nsd);
+}
+
 /**
    Set checksum for a var.