Merge pull request #306 from Unidata/revert-305-revert-304-ejh_quantize

Add quantize feature to F77 and F90 APIs, with tests and documentation
Unidata · Jan 4, 2022 · 7ac1a3c · 7ac1a3c
2 parents cde90be + 24d42db
commit 7ac1a3c
Show file tree

Hide file tree

Showing 22 changed files with 689 additions and 24 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -0,0 +1,70 @@
+name: build
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+
+    env:
+      CPPFLAGS: "-I/home/runner/hdf5/include -I/home/runner/netcdf-c/include"
+      LDFLAGS: "-L/home/runner/hdf5/lib -L/home/runner/netcdf-c/lib"
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Installs
+      run: |
+        sudo apt-get install doxygen graphviz wget gfortran libjpeg-dev libz-dev
+    - name: cache-hdf5
+      id: cache-hdf5
+      uses: actions/cache@v2
+      with:
+        path: ~/hdf5
+        key: hdf5-${{ runner.os }}-1.10.7
+
+    - name: build-hdf5
+      if: steps.cache-hdf5.outputs.cache-hit != 'true'
+      run: |
+        set -x
+        wget https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-1.10/hdf5-1.10.7/src/hdf5-1.10.7.tar.gz &> /dev/null
+        tar -xzf hdf5-1.10.7.tar.gz
+        pushd hdf5-1.10.7
+        ./configure --prefix=/home/runner/hdf5 --disable-tools --disable-fortran --disable-cxx 
+        make
+        sudo make install
+        popd
+    - name: build-netcdf-c
+      run: |
+        set -x
+        git clone -b ejh_quantize_2 https://github.com/edwardhartnett/netcdf-c.git
+        pushd netcdf-c
+        autoreconf -i
+        ./configure --prefix=/home/runner/netcdf-c --disable-dap --disable-utilities
+        make -j
+        sudo make install
+        popd
+    # - name: cmake build
+    #   run: |
+    #     set -x
+    #     gcc --version
+    #     export LD_LIBRARY_PATH="/home/runner/netcdf-c/lib:/home/runner/hdf5/lib:$LD_LIBRARY_PATH"
+    #     mkdir build
+    #     cd build
+    #     cmake -Wno-dev -DNetCDF_C_LIBRARY=/home/runner/netcdf-c/lib/libnetcdf.so -DNetCDF_C_INCLUDE_DIR=/home/runner/netcdf-c/include .. || (cat CMakeFiles/CMakeOutput.log && cat CMakeFiles/CMakeError.log)
+    #     make VERBOSE=1
+    #     make tests VERBOSE=1
+#        ctest -VV
+    - name: autotools build
+      run: |
+        set -x
+        gcc --version
+        autoreconf -i
+        ./configure
+        make -j check
+        
+        
diff --git a/.gitignore b/.gitignore
@@ -1,5 +1,4 @@
 CTestConfig.cmake
-build*
 \#.\#
 *.*~
 html

diff --git a/docs/netcdf-f90-sec6-variables.md b/docs/netcdf-f90-sec6-variables.md
@@ -127,7 +127,8 @@ in a netCDF-4/HDF5 file.
    integer, optional, intent(in) :: deflate_level
    logical, optional, intent(in) :: shuffle, fletcher32
    integer, optional, intent(in) :: endianness
-    integer, optional, intent(in) :: cache_size, cache_nelems, cache_preemption
+   integer, optional, intent(in) :: cache_size, cache_nelems, cache_preemption
+   integer, optional, intent(in) :: quantize_mode, nsd
    integer                                      :: nf90_def_var
 
 
@@ -250,7 +251,19 @@ in a netCDF-4/HDF5 file.
     while a value of 100 means fully read chunks are always preempted
     before other chunks.
 
+`quantize_mode`
 
+:   Set to nf90_noquantize (0) for no quantization (the default), or
+    nf90_quantize_bitgroom (1) for bitgroom quantization. Quantization
+    allows users to specify a number of significant digits (NSD) in
+    nf90_float and nf90_double variables. Bits beyond those needed to
+    maintain the NSD are set to zero or one (alternating). This causes
+    the data to compress better when compression is applied.
+
+`nsd`
+
+:   If quantize_mode is set to nf90_quantize_bitgroom, then this is the
+    number of significant digits to be maintained in the data.
 
 ## Return Codes
 
@@ -723,6 +736,7 @@ netCDF-4/HDF5 file.
     integer, optional, intent(out) :: deflate_level
     logical, optional, intent(out) :: shuffle, fletcher32
     integer, optional, intent(out) :: endianness
+    integer, optional, intent(in) :: quantize_mode, nsd
     integer :: nf90_inquire_variable
 
 
@@ -798,6 +812,22 @@ netCDF-4/HDF5 file.
     big-endian format, and NF90\_ENDIAN\_NATIVE if the endianness is not
     set, and the variable is not created yet.
 
+`quantize_mode`
+
+:   Will be set to nf90_noquantize (0) for no quantization, or
+    nf90_quantize_bitgroom (1) for bitgroom quantization. Quantization
+    allows users to specify a number of significant digits (NSD) in
+    nf90_float and nf90_double variables. Bits beyond those needed to
+    maintain the NSD are set to zero or one (alternating). This causes
+    the data to compress better when compression is applied.
+
+`nsd`
+
+:   If quantize_mode is set to nf90_quantize_bitgroom, then this will be
+    set to the number of significant digits to be maintained in the
+    data.
+
+
 These functions return the value NF90\_NOERR if no errors occurred.
 Otherwise, the returned status indicates an error. Possible causes of
 errors include:

diff --git a/fortran/module_netcdf4_nc_interfaces.f90 b/fortran/module_netcdf4_nc_interfaces.f90
@@ -724,6 +724,19 @@ Function nc_inq_var_szip(ncid, varid, options_mask, pixels_per_block) BIND(C)
 
  End Function nc_inq_var_szip
 End Interface
+!------------------------------- nc_inq_var_quantize ------------------------------
+Interface
+ Function nc_inq_var_quantize(ncid, varid, quantize_mode, nsd) BIND(C)
+
+ USE ISO_C_BINDING, ONLY: C_INT
+
+ Integer(C_INT), VALUE         :: ncid, varid
+ Integer(C_INT), Intent(INOUT) :: quantize_mode, nsd 
+
+ Integer(C_INT)                :: nc_inq_var_quantize
+
+ End Function nc_inq_var_quantize
+End Interface
 !------------------------------- nc_def_var_fletcher32 ------------------------
 Interface
  Function nc_def_var_fletcher32(ncid, varid, fletcher32) BIND(C)

diff --git a/fortran/module_netcdf4_nf_interfaces.F90 b/fortran/module_netcdf4_nf_interfaces.F90
@@ -544,6 +544,28 @@ Function nf_inq_var_szip(ncid, varid, options_mask, pixels_per_block) RESULT(sta
 
  End Function nf_inq_var_szip
 End Interface
+!-------------------------------- nf_def_var_quantize --------------------------
+Interface
+ Function nf_def_var_quantize( ncid, varid, quantize_mode, nsd) &
+                               RESULT (status)
+
+ Integer, Intent(IN) :: ncid, varid, quantize_mode, nsd
+ Integer             :: status
+
+ End Function nf_def_var_quantize
+End Interface
+!-------------------------------- nf_inq_var_quantize -----------------------------
+Interface
+ Function nf_inq_var_quantize(ncid, varid, quantize_mode, nsd) RESULT(status)
+
+ Implicit NONE
+
+ Integer, Intent(IN)    :: ncid, varid
+ Integer, Intent(INOUT) :: quantize_mode, nsd
+ Integer                :: status
+
+ End Function nf_inq_var_quantize
+End Interface
 !-------------------------------- nf_def_var_fletcher32 ------------------------
 Interface
  Function nf_def_var_fletcher32( ncid, varid, fletcher32) RESULT(status)

diff --git a/fortran/module_netcdf_nc_data.F90 b/fortran/module_netcdf_nc_data.F90
@@ -405,6 +405,10 @@ Module netcdf_nc_data
  Integer(C_INT), Parameter :: NC_ECANTEXTEND    = -130
  Integer(C_INT), Parameter :: NC_EMPI           = -131
 
+! Quantize feature 
+ Integer(C_INT), Parameter :: NC_NOQUANTIZE           = 0
+ Integer(C_INT), Parameter :: NC_QUANTIZE_BITGROOM    = 1
+
 #endif
 
 !------------------------------------------------------------------------------

diff --git a/fortran/module_netcdf_nc_interfaces.f90 b/fortran/module_netcdf_nc_interfaces.f90
@@ -2540,6 +2540,19 @@ Function nc_def_var_szip(ncid, varid, options_mask, pixels_per_block) BIND(C)
  End Function nc_def_var_szip
 End Interface
 
+!---------------------------------- nc_def_var_quantize ---------------------------------
+Interface
+ Function nc_def_var_quantize(ncid, varid, quantize_mode, nsd) BIND(C)
+
+ USE ISO_C_BINDING, ONLY: C_CHAR, C_INT
+
+ Integer(C_INT), VALUE :: ncid, varid, quantize_mode, nsd
+
+ Integer(C_INT) :: nc_def_var_quantize
+
+ End Function nc_def_var_quantize
+End Interface
+
 !---------------------------- Start of module procedures ---------------------
 CONTAINS
 

diff --git a/fortran/module_netcdf_nf_data.F90 b/fortran/module_netcdf_nf_data.F90
@@ -161,7 +161,11 @@ Module netcdf_nf_data
 ! Error handling codes
 
  Integer, Parameter :: NF_FATAL   = NC_FATAL 
- Integer, Parameter :: NF_VERBOSE = NC_VERBOSE 
+ Integer, Parameter :: NF_VERBOSE = NC_VERBOSE
+
+ ! Quantize feature
+ Integer, Parameter :: NF_NOQUANTIZE   = NC_NOQUANTIZE 
+ Integer, Parameter :: NF_QUANTIZE_BITGROOM  = NC_QUANTIZE_BITGROOM
 
 #ifdef ENABLE_CDF5
  ! new format types

diff --git a/fortran/netcdf4.inc b/fortran/netcdf4.inc
@@ -94,6 +94,12 @@
       integer nf_collective
       parameter (nf_collective = 1)
 
+!     For NF_DEF_VAR_QUANTIZE.
+      integer nf_noquantize
+      parameter (nf_noquantize = 0)
+      integer nf_quantize_bitgroom
+      parameter (nf_quantize_bitgroom = 1)
+
 !     New error codes.
       integer nf_ehdferr        ! Error at HDF5 layer. 
       parameter (nf_ehdferr = -101)
@@ -211,6 +217,12 @@
       integer nf_inq_var_szip
       external nf_inq_var_szip
 
+      integer nf_def_var_quantize
+      external nf_def_var_quantize
+
+      integer nf_inq_var_quantize
+      external nf_inq_var_quantize
+
       integer nf_def_var_fletcher32
       external nf_def_var_fletcher32
 

diff --git a/fortran/netcdf4_externals.f90 b/fortran/netcdf4_externals.f90
@@ -28,4 +28,4 @@
        nf_get_var_int64, nf_get_chunk_cache, nf_set_chunk_cache, &
        nf_inq_var_szip, nf_def_var_szip, nf_free_vlens, nf_free_string, &
        nf_set_var_chunk_cache, nf_get_var_chunk_cache, nf_rename_grp, &
-       nf_def_var_filter, nf_inq_var_filter
+       nf_def_var_filter, nf_inq_var_filter, nf_inq_var_quantize, nf_def_var_quantize
diff --git a/fortran/netcdf4_func.f90 b/fortran/netcdf4_func.f90
@@ -531,6 +531,16 @@ function nf90_def_var_szip(ncid, varid, options_mask, pixels_per_block)
     nf90_def_var_szip = nf_def_var_szip(ncid, varid, options_mask, pixels_per_block)
   end function nf90_def_var_szip
   ! -----------
+  function nf90_def_var_quantize(ncid, varid, quantize_mode, nsd)
+    integer, intent(in) :: ncid
+    integer, intent(in) :: varid
+    integer, intent(in) :: quantize_mode
+    integer, intent(in) :: nsd
+    integer :: nf90_def_var_quantize
+
+    nf90_def_var_quantize = nf_def_var_quantize(ncid, varid, quantize_mode, nsd)
+  end function nf90_def_var_quantize
+  ! -----------
   function nf90_def_var_fletcher32(ncid, varid, fletcher32)
     integer, intent(in) :: ncid
     integer, intent(in) :: varid
@@ -571,6 +581,16 @@ function nf90_inq_var_szip(ncid, varid, options_mask, pixels_per_block)
     nf90_inq_var_szip = nf_inq_var_szip(ncid, varid, options_mask, pixels_per_block)
   end function nf90_inq_var_szip
   ! -----------
+  function nf90_inq_var_quantize(ncid, varid, quantize_mode, nsd)
+    integer, intent(in) :: ncid
+    integer, intent(in) :: varid
+    integer, intent(out) :: quantize_mode
+    integer, intent(out) :: nsd
+    integer :: nf90_inq_var_quantize
+
+    nf90_inq_var_quantize = nf_inq_var_quantize(ncid, varid, quantize_mode, nsd)
+  end function nf90_inq_var_quantize
+  ! -----------
   function nf90_inq_var_fletcher32(ncid, varid, fletcher32)
     integer, intent(in) :: ncid
     integer, intent(in) :: varid

diff --git a/fortran/netcdf4_variables.f90 b/fortran/netcdf4_variables.f90
@@ -19,7 +19,7 @@ end function nf90_def_var_Scalar
   ! ----- 
   function nf90_def_var_oneDim(ncid, name, xtype, dimids, varid, contiguous, &
        chunksizes, deflate_level, shuffle, fletcher32, endianness, &
-       cache_size, cache_nelems, cache_preemption)
+       cache_size, cache_nelems, cache_preemption, quantize_mode, nsd)
     integer, intent( in) :: ncid
     character (len = *), intent( in) :: name
     integer, intent(in) :: xtype
@@ -31,6 +31,7 @@ function nf90_def_var_oneDim(ncid, name, xtype, dimids, varid, contiguous, &
     logical, optional, intent(in) :: shuffle, fletcher32
     integer, optional, intent(in) :: endianness
     integer, optional, intent(in) :: cache_size, cache_nelems, cache_preemption
+    integer, optional, intent(in) :: quantize_mode, nsd
     integer :: nf90_def_var_oneDim
 
     integer, dimension(1) :: dimidsA, chunksizes1
@@ -123,11 +124,23 @@ function nf90_def_var_oneDim(ncid, name, xtype, dimids, varid, contiguous, &
        if (nf90_def_var_oneDim .ne. nf90_noerr) return
     endif
 
+    ! Set quantize if the user wants to.
+    if (present(quantize_mode)) then
+       if (.not. present(nsd)) then
+          nf90_def_var_oneDim = nf90_einval
+          return
+       endif
+       nf90_def_var_oneDim = nf_def_var_quantize(ncid, varid, quantize_mode, nsd)
+       if (nf90_def_var_oneDim .ne. nf90_noerr) return
+
+    endif
+
+
   end function nf90_def_var_oneDim
   ! ----- 
   function nf90_def_var_ManyDims(ncid, name, xtype, dimids, varid, contiguous, &
        chunksizes, deflate_level, shuffle, fletcher32, endianness, cache_size, &
-       cache_nelems, cache_preemption)
+       cache_nelems, cache_preemption, quantize_mode, nsd)
     integer, intent(in) :: ncid
     character (len = *), intent(in) :: name
     integer, intent( in) :: xtype
@@ -139,6 +152,7 @@ function nf90_def_var_ManyDims(ncid, name, xtype, dimids, varid, contiguous, &
     logical, optional, intent(in) :: shuffle, fletcher32
     integer, optional, intent(in) :: endianness
     integer, optional, intent(in) :: cache_size, cache_nelems, cache_preemption
+    integer, optional, intent(in) :: quantize_mode, nsd
     integer :: nf90_def_var_ManyDims
 
     ! Local variables.
@@ -239,6 +253,16 @@ function nf90_def_var_ManyDims(ncid, name, xtype, dimids, varid, contiguous, &
        if (nf90_def_var_ManyDims .ne. nf90_noerr) return
     endif
 
+    ! Set quantize if the user wants to.
+    if (present(quantize_mode)) then
+       if (.not. present(nsd)) then
+          nf90_def_var_ManyDims = nf90_einval
+          return
+       endif
+       nf90_def_var_ManyDims = nf_def_var_quantize(ncid, varid, quantize_mode, nsd)
+       if (nf90_def_var_ManyDims .ne. nf90_noerr) return
+    endif
+
   end function nf90_def_var_ManyDims
   ! ----- 
   function nf90_inq_varid(ncid, name, varid)
@@ -264,7 +288,7 @@ end function nf90_set_var_chunk_cache
   ! ----- 
   function nf90_inquire_variable(ncid, varid, name, xtype, ndims, dimids, nAtts, &
        contiguous, chunksizes, deflate_level, shuffle, fletcher32, endianness, &
-       cache_size, cache_nelems, cache_preemption)
+       cache_size, cache_nelems, cache_preemption, quantize_mode, nsd)
     integer, intent(in) :: ncid, varid
     character (len = *), optional, intent(out) :: name
     integer, optional, intent(out) :: xtype, ndims 
@@ -276,6 +300,7 @@ function nf90_inquire_variable(ncid, varid, name, xtype, ndims, dimids, nAtts, &
     logical, optional, intent(out) :: shuffle, fletcher32
     integer, optional, intent(out) :: endianness
     integer, optional, intent(out) :: cache_size, cache_nelems, cache_preemption
+    integer, optional, intent(out) :: quantize_mode, nsd
     integer :: nf90_inquire_variable
 
     ! Local variables
@@ -348,6 +373,13 @@ function nf90_inquire_variable(ncid, varid, name, xtype, ndims, dimids, nAtts, &
        if (present(cache_nelems)) cache_nelems = nelems1
        if (present(cache_preemption)) cache_preemption = preemption1
     endif
+
+    ! And the quantization...
+    if (present(quantize_mode)) then
+       nf90_inquire_variable = nf_inq_var_quantize(ncid, varid, quantize_mode, nsd)
+       if (nf90_inquire_variable .ne. nf90_noerr) return
+    endif
+
   end function nf90_inquire_variable
   ! ----- 
   function nf90_rename_var(ncid, varid, newname)
-Original file line number
+Diff line change
@@ -1,5 +1,4 @@
     CTestConfig.cmake
-    build*
     \#.\#
     *.*~
     html
@@ Expand Down @@