From 12818833daa2991a07e6be5c77d8068809a50483 Mon Sep 17 00:00:00 2001 From: Edward Hartnett Date: Fri, 3 Sep 2021 12:55:50 -0600 Subject: [PATCH 01/16] adding quantize to F77 and F90 apis --- fortran/module_netcdf4_nc_interfaces.f90 | 13 +++++++ fortran/module_netcdf4_nf_interfaces.F90 | 22 +++++++++++ fortran/module_netcdf_nc_interfaces.f90 | 13 +++++++ fortran/netcdf4.inc | 12 ++++++ fortran/netcdf4_externals.f90 | 2 +- fortran/netcdf4_func.f90 | 20 ++++++++++ fortran/netcdf4_visibility.f90 | 3 +- fortran/netcdf_constants.f90 | 4 +- fortran/nf_nc4.f90 | 49 ++++++++++++++++++++++++ nf03_test4/run_f90_par_test.sh | 12 +++--- nf_test4/Makefile.am | 17 +++++--- 11 files changed, 153 insertions(+), 14 deletions(-) diff --git a/fortran/module_netcdf4_nc_interfaces.f90 b/fortran/module_netcdf4_nc_interfaces.f90 index 122b0e79..c80c0f69 100755 --- a/fortran/module_netcdf4_nc_interfaces.f90 +++ b/fortran/module_netcdf4_nc_interfaces.f90 @@ -724,6 +724,19 @@ Function nc_inq_var_szip(ncid, varid, options_mask, pixels_per_block) BIND(C) End Function nc_inq_var_szip End Interface +!------------------------------- nc_inq_var_quantize ------------------------------ +Interface + Function nc_inq_var_quantize(ncid, varid, quantize_mode, nsd) BIND(C) + + USE ISO_C_BINDING, ONLY: C_INT + + Integer(C_INT), VALUE :: ncid, varid + Integer(C_INT), Intent(INOUT) :: quantize_mode, nsd + + Integer(C_INT) :: nc_inq_var_quantize + + End Function nc_inq_var_quantize +End Interface !------------------------------- nc_def_var_fletcher32 ------------------------ Interface Function nc_def_var_fletcher32(ncid, varid, fletcher32) BIND(C) diff --git a/fortran/module_netcdf4_nf_interfaces.F90 b/fortran/module_netcdf4_nf_interfaces.F90 index 2090ce1b..dbb92d86 100755 --- a/fortran/module_netcdf4_nf_interfaces.F90 +++ b/fortran/module_netcdf4_nf_interfaces.F90 @@ -544,6 +544,28 @@ Function nf_inq_var_szip(ncid, varid, options_mask, pixels_per_block) RESULT(sta End Function nf_inq_var_szip End Interface +!-------------------------------- nf_def_var_quantize -------------------------- +Interface + Function nf_def_var_quantize( ncid, varid, quantize_mode, nsd) & + RESULT (status) + + Integer, Intent(IN) :: ncid, varid, quantize_mode, nsd + Integer :: status + + End Function nf_def_var_quantize +End Interface +!-------------------------------- nf_inq_var_quantize ----------------------------- +Interface + Function nf_inq_var_quantize(ncid, varid, quantize_mode, nsd) RESULT(status) + + Implicit NONE + + Integer, Intent(IN) :: ncid, varid + Integer, Intent(INOUT) :: quantize_mode, nsd + Integer :: status + + End Function nf_inq_var_quantize +End Interface !-------------------------------- nf_def_var_fletcher32 ------------------------ Interface Function nf_def_var_fletcher32( ncid, varid, fletcher32) RESULT(status) diff --git a/fortran/module_netcdf_nc_interfaces.f90 b/fortran/module_netcdf_nc_interfaces.f90 index 9d78fb65..eec92b5d 100755 --- a/fortran/module_netcdf_nc_interfaces.f90 +++ b/fortran/module_netcdf_nc_interfaces.f90 @@ -2540,6 +2540,19 @@ Function nc_def_var_szip(ncid, varid, options_mask, pixels_per_block) BIND(C) End Function nc_def_var_szip End Interface +!---------------------------------- nc_def_var_quantize --------------------------------- +Interface + Function nc_def_var_quantize(ncid, varid, quantize_mode, nsd) BIND(C) + + USE ISO_C_BINDING, ONLY: C_CHAR, C_INT + + Integer(C_INT), VALUE :: ncid, varid, quantize_mode, nsd + + Integer(C_INT) :: nc_def_var_quantize + + End Function nc_def_var_quantize +End Interface + !---------------------------- Start of module procedures --------------------- CONTAINS diff --git a/fortran/netcdf4.inc b/fortran/netcdf4.inc index c2a11f70..47796e36 100755 --- a/fortran/netcdf4.inc +++ b/fortran/netcdf4.inc @@ -94,6 +94,12 @@ integer nf_collective parameter (nf_collective = 1) +! For NF_DEF_VAR_QUANTIZE. + integer nf_noquantize + parameter (nf_noquantize = 0) + integer nf_quantize_bitgroom + parameter (nf_quantize_bitgroom = 1) + ! New error codes. integer nf_ehdferr ! Error at HDF5 layer. parameter (nf_ehdferr = -101) @@ -211,6 +217,12 @@ integer nf_inq_var_szip external nf_inq_var_szip + integer nf_def_var_quantize + external nf_def_var_quantize + + integer nf_inq_var_quantize + external nf_inq_var_quantize + integer nf_def_var_fletcher32 external nf_def_var_fletcher32 diff --git a/fortran/netcdf4_externals.f90 b/fortran/netcdf4_externals.f90 index 4b0690a4..ca735f7c 100644 --- a/fortran/netcdf4_externals.f90 +++ b/fortran/netcdf4_externals.f90 @@ -28,4 +28,4 @@ nf_get_var_int64, nf_get_chunk_cache, nf_set_chunk_cache, & nf_inq_var_szip, nf_def_var_szip, nf_free_vlens, nf_free_string, & nf_set_var_chunk_cache, nf_get_var_chunk_cache, nf_rename_grp, & - nf_def_var_filter, nf_inq_var_filter + nf_def_var_filter, nf_inq_var_filter, nf_inq_var_quantize, nf_def_var_quantize diff --git a/fortran/netcdf4_func.f90 b/fortran/netcdf4_func.f90 index a8315faa..0b7da068 100644 --- a/fortran/netcdf4_func.f90 +++ b/fortran/netcdf4_func.f90 @@ -531,6 +531,16 @@ function nf90_def_var_szip(ncid, varid, options_mask, pixels_per_block) nf90_def_var_szip = nf_def_var_szip(ncid, varid, options_mask, pixels_per_block) end function nf90_def_var_szip ! ----------- + function nf90_def_var_quantize(ncid, varid, quantize_mode, nsd) + integer, intent(in) :: ncid + integer, intent(in) :: varid + integer, intent(in) :: quantize_mode + integer, intent(in) :: nsd + integer :: nf90_def_var_quantize + + nf90_def_var_quantize = nf_def_var_quantize(ncid, varid, quantize_mode, nsd) + end function nf90_def_var_quantize + ! ----------- function nf90_def_var_fletcher32(ncid, varid, fletcher32) integer, intent(in) :: ncid integer, intent(in) :: varid @@ -571,6 +581,16 @@ function nf90_inq_var_szip(ncid, varid, options_mask, pixels_per_block) nf90_inq_var_szip = nf_inq_var_szip(ncid, varid, options_mask, pixels_per_block) end function nf90_inq_var_szip ! ----------- + function nf90_inq_var_quantize(ncid, varid, quantize_mode, nsd) + integer, intent(in) :: ncid + integer, intent(in) :: varid + integer, intent(out) :: quantize_mode + integer, intent(out) :: nsd + integer :: nf90_inq_var_quantize + + nf90_inq_var_quantize = nf_inq_var_quantize(ncid, varid, quantize_mode, nsd) + end function nf90_inq_var_quantize + ! ----------- function nf90_inq_var_fletcher32(ncid, varid, fletcher32) integer, intent(in) :: ncid integer, intent(in) :: varid diff --git a/fortran/netcdf4_visibility.f90 b/fortran/netcdf4_visibility.f90 index 5fb23b11..34b6989e 100644 --- a/fortran/netcdf4_visibility.f90 +++ b/fortran/netcdf4_visibility.f90 @@ -23,4 +23,5 @@ nf90_def_var_endian, nf90_inq_var_endian, nf90_inq_user_type, & nf90_put_att_any, nf90_get_att_any, nf90_get_var_any, nf90_put_var_any, & nf90_rename_grp, nf90_def_var_filter, nf90_inq_var_filter, & - nf90_def_var_szip, nf90_inq_var_szip + nf90_def_var_szip, nf90_inq_var_szip, nf90_def_var_quantize, nf90_inq_var_quantize + diff --git a/fortran/netcdf_constants.f90 b/fortran/netcdf_constants.f90 index 786e379f..47913d90 100644 --- a/fortran/netcdf_constants.f90 +++ b/fortran/netcdf_constants.f90 @@ -241,7 +241,9 @@ nf90_noshuffle = 0, & nf90_shuffle = 1, & nf90_szip_ec_option_mask = 4, & - nf90_szip_nn_option_mask = 32 + nf90_szip_nn_option_mask = 32, & + nf90_noquantize = 0, & + nf90_quantize_bitgroom = 1 ! This is the position of NC_NETCDF4 in cmode, counting from the ! right, starting (uncharacteristically for fortran) at 0. It's needed diff --git a/fortran/nf_nc4.f90 b/fortran/nf_nc4.f90 index 08890ecf..2ba42120 100755 --- a/fortran/nf_nc4.f90 +++ b/fortran/nf_nc4.f90 @@ -1569,6 +1569,55 @@ Function nf_inq_var_szip(ncid, varid, options_mask, pixels_per_block) RESULT(sta End Function nf_inq_var_szip +!-------------------------------- nf_def_var_quantize -------------------------- + Function nf_def_var_quantize(ncid, varid, quantize_mode, nsd) & + RESULT (status) +! define variable deflation + USE netcdf4_nc_interfaces + Implicit NONE + Integer, Intent(IN) :: ncid, varid, quantize_mode, nsd + Integer :: status + Integer(C_INT) :: cncid, cvarid, cquantize_mode, cnsd, cstatus + + cncid = ncid + cvarid = varid-1 + cquantize_mode = quantize_mode + cnsd = nsd + + cstatus = nc_def_var_quantize(cncid, cvarid, cquantize_mode, cnsd) + status = cstatus + + End Function nf_def_var_quantize +!-------------------------------- nf_inq_var_quantize ----------------------------- + Function nf_inq_var_quantize(ncid, varid, quantize_mode, nsd) RESULT(status) + +! get quantize variables + + USE netcdf4_nc_interfaces + + Implicit NONE + + Integer, Intent(IN) :: ncid, varid + Integer, Intent(INOUT) :: quantize_mode, nsd + + Integer :: status + + Integer(C_INT) :: cncid, cvarid, cquantize_mode, cnsd, cstatus + + cncid = ncid + cvarid = varid-1 + + cstatus = nc_inq_var_quantize(cncid, cvarid, cquantize_mode, cnsd) + + If (cstatus == NC_NOERR) Then + quantize_mode = cquantize_mode + nsd = cnsd + EndIf + + status = cstatus + + End Function nf_inq_var_quantize + !-------------------------------- nf_def_var_fletcher32 ----------------------- Function nf_def_var_fletcher32( ncid, varid, fletcher32) RESULT(status) diff --git a/nf03_test4/run_f90_par_test.sh b/nf03_test4/run_f90_par_test.sh index b7033b9c..d8e91f03 100755 --- a/nf03_test4/run_f90_par_test.sh +++ b/nf03_test4/run_f90_par_test.sh @@ -6,12 +6,12 @@ set -e echo "Testing netCDF parallel I/O through the F90 API." -mpirun -n 4 ./f90tst_parallel -mpirun -n 4 ./f90tst_parallel2 -mpirun -n 4 ./f90tst_parallel3 -mpirun -n 8 ./f90tst_nc4_par -mpirun -n 4 ./f90tst_parallel_fill -mpirun -n 4 ./f90tst_parallel_compressed +mpiexec -n 4 ./f90tst_parallel +mpiexec -n 4 ./f90tst_parallel2 +mpiexec -n 4 ./f90tst_parallel3 +mpiexec -n 8 ./f90tst_nc4_par +mpiexec -n 4 ./f90tst_parallel_fill +mpiexec -n 4 ./f90tst_parallel_compressed echo "SUCCESS!!!" exit 0 diff --git a/nf_test4/Makefile.am b/nf_test4/Makefile.am index b6d18624..82aad914 100644 --- a/nf_test4/Makefile.am +++ b/nf_test4/Makefile.am @@ -13,8 +13,8 @@ LDADD = ${top_builddir}/fortran/libnetcdff.la # These programs test the F77 netCDF-4 API using include netcdf.inc. NF77_TESTS = ftst_groups ftst_vars ftst_vars2 ftst_vars3 ftst_vars4 \ -ftst_vars5 ftst_vars6 ftst_types ftst_types2 ftst_types3 ftst_path \ -ftst_rengrps f03tst_open_mem ftst_var_compact +ftst_vars5 ftst_vars6 ftst_vars7 ftst_types ftst_types2 ftst_types3 \ +ftst_path ftst_rengrps f03tst_open_mem ftst_var_compact # Compile and run the F77 API tests. check_PROGRAMS = $(NF77_TESTS) @@ -28,6 +28,7 @@ ftst_vars3_SOURCES = ftst_vars3.F ftst_vars4_SOURCES = ftst_vars4.F ftst_vars5_SOURCES = ftst_vars5.F ftst_vars6_SOURCES = ftst_vars6.F +ftst_vars7_SOURCES = ftst_vars7.F ftst_types_SOURCES = ftst_types.F ftst_types2_SOURCES = ftst_types2.F ftst_types3_SOURCES = ftst_types3.F @@ -73,12 +74,12 @@ if USE_SED # These test fortran codes will be created using sed. F03_TEST_CODES = f03tst_groups.F f03tst_vars.F f03tst_vars2.F \ f03tst_vars3.F f03tst_vars4.F f03tst_vars5.F f03tst_vars6.F \ -f03tst_types.F f03tst_types2.F f03tst_types3.F +f03tst_vars7.F f03tst_types.F f03tst_types2.F f03tst_types3.F # The fortran codes will compile into these tests. F03_TESTS = f03tst_groups f03tst_vars f03tst_vars2 f03tst_vars3 \ -f03tst_vars4 f03tst_vars5 f03tst_vars6 f03tst_types f03tst_types2 \ -f03tst_types3 +f03tst_vars4 f03tst_vars5 f03tst_vars6 f03tst_vars7 f03tst_types \ +f03tst_types2 f03tst_types3 # This is a netCDF-4 V2 F77 test program. if BUILD_V2 @@ -118,6 +119,7 @@ f03tst_vars3_SOURCES = f03tst_vars3.F f03tst_vars4_SOURCES = f03tst_vars4.F f03tst_vars5_SOURCES = f03tst_vars5.F f03tst_vars6_SOURCES = f03tst_vars6.F +f03tst_vars7_SOURCES = f03tst_vars7.F f03tst_types_SOURCES = f03tst_types.F f03tst_types2_SOURCES = f03tst_types2.F f03tst_types3_SOURCES = f03tst_types3.F @@ -158,6 +160,11 @@ f03tst_vars6.F: ftst_vars6.F $(SED) "s|include 'netcdf.inc'|implicit none|" $@.tmp > $@.tmp2 $(SED) "s|ftst_vars6.nc|f03tst_vars6.nc|" $@.tmp2 > $@ +f03tst_vars7.F: ftst_vars7.F + $(SED) "s|implicit none|USE netcdf4_f03|" $< > $@.tmp + $(SED) "s|include 'netcdf.inc'|implicit none|" $@.tmp > $@.tmp2 + $(SED) "s|ftst_vars7.nc|f03tst_vars7.nc|" $@.tmp2 > $@ + f03tst_types.F: ftst_types.F $(SED) "s|implicit none|USE netcdf4_f03|" $< > $@.tmp $(SED) "s|include 'netcdf.inc'|implicit none|" $@.tmp > $@.tmp2 From 67bbda86f86326e2eab2203b66130db611a9ad4d Mon Sep 17 00:00:00 2001 From: Edward Hartnett Date: Fri, 3 Sep 2021 14:16:07 -0600 Subject: [PATCH 02/16] test for quantize for F77 API working --- nf_test4/ftst_vars7.F | 213 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 213 insertions(+) create mode 100644 nf_test4/ftst_vars7.F diff --git a/nf_test4/ftst_vars7.F b/nf_test4/ftst_vars7.F new file mode 100644 index 00000000..4937a332 --- /dev/null +++ b/nf_test4/ftst_vars7.F @@ -0,0 +1,213 @@ +C This is part of the netCDF package. +C Copyright 2008 University Corporation for Atmospheric Research/Unidata. +C See COPYRIGHT file for conditions of use. + +C This program tests netCDF-4 variable functions from fortran, +C checking the quantize feature. + +C Ed Hartnett, 9/3/21 + + program ftst_vars7 + implicit none + include 'netcdf.inc' + +C This is the name of the data file we will create. + character*(*) FILE_NAME + parameter (FILE_NAME='ftst_vars7.nc') + integer NDIM1 + parameter (NDIM1 = 1) + integer DIM_LEN_5 + parameter (DIM_LEN_5 = 5) + integer NVARS + parameter (NVARS = 2) + integer check_file + integer NSD_3 + parameter (NSD_3 = 3) + + integer ncid, varid(NVARS), dimids(NDIM1) + character*(10) var_name(NVARS) + integer var_type(NVARS) + character*(4) dim_name + parameter (dim_name = 'dim1') + integer start(NDIM1), count(NDIM1) + real real_data(DIM_LEN_5) + real*8 double_data(DIM_LEN_5) + +C Loop index and error handling. + integer x, retval + + print *, '' + print *,'*** Testing quantize feature.' + +C Set up var names. + var_name(1) = 'var__float' + var_type(1) = NF_FLOAT + var_name(2) = 'var_double' + var_type(2) = NF_DOUBLE + +C Set up data. +C float real_data[DIM_LEN_5] = {1.11111111, 1.0, 9.99999999, 12345.67, .1234567}; +C double double_data[DIM_LEN_5] = {1.1111111, 1.0, 9.999999999, 1234567890.12345, 123456789012345.0}; + real_data(1) = 1.11111111 + real_data(2) = 1.0 + real_data(3) = 9.99999999 + real_data(4) = 12345.67 + real_data(5) = .1234567 + double_data(1) = 1.1111111 + double_data(2) = 1.0 + double_data(3) = 9.999999999 + double_data(4) = 1234567890.12345 + double_data(5) = 1234567890 + +C Create the netCDF file. + retval = nf_create(FILE_NAME, NF_NETCDF4, ncid) + if (retval .ne. nf_noerr) stop 1 + +C Create a dimension. + retval = nf_def_dim(ncid, dim_name, DIM_LEN_5, dimids(1)) + if (retval .ne. nf_noerr) stop 2 + +C Create some variables. + do x = 1, NVARS + retval = nf_def_var(ncid, var_name(x), var_type(x), NDIM1, + $ dimids, varid(x)) + if (retval .ne. nf_noerr) stop 3 + +C Turn on quantize. + retval = nf_def_var_quantize(ncid, varid(x), + $ NF_QUANTIZE_BITGROOM, NSD_3) + if (retval .ne. nf_noerr) stop 3 + end do + +C Write some data (which automatically calls nf_enddef). + start(1) = 1 + count(1) = DIM_LEN_5 + retval = nf_put_vara_real(ncid, varid(1), start, count, + $ real_data) + if (retval .ne. 0) stop 200 + retval = nf_put_vara_double(ncid, varid(2), start, count, + $ double_data) + if (retval .ne. 0) stop 200 + +C Check it out. + retval = check_file(ncid, var_name, var_type, dim_name) + if (retval .ne. 0) stop 2 + +C Close the file. + retval = nf_close(ncid) + if (retval .ne. nf_noerr) stop 4 + +c$$$C Reopen the file. +c$$$ retval = nf_open(FILE_NAME, NF_NOWRITE, ncid) +c$$$ if (retval .ne. nf_noerr) stop 5 +c$$$ +c$$$C Check it out. +c$$$ retval = check_file(ncid, var_name, var_type, dim_name) +c$$$ if (retval .ne. 0) stop 6 +c$$$ +c$$$C Close the file. +c$$$ retval = nf_close(ncid) +c$$$ if (retval .ne. nf_noerr) stop 7 + + print *,'*** SUCCESS!' + end + +C This function check the file to make sure everything is OK. + integer function check_file(ncid, var_name, var_type, dim_name) + implicit none + include 'netcdf.inc' + +C I need these in both here and the main program. + integer NDIM1 + parameter (NDIM1 = 1) + integer DIM_LEN_5 + parameter (DIM_LEN_5 = 5) + integer NVARS + parameter (NVARS = 2) + integer NSD_3 + parameter (NSD_3 = 3) + real EPSILON + parameter (EPSILON = .01) + +C Parameters + integer ncid + character*(10) var_name(NVARS) + integer var_type(NVARS) + character*(4) dim_name + +C Values that are read in, to check the file. + integer ndims_in, nvars_in, ngatts_in, unlimdimid_in + integer xtype_in, dimids_in(NDIM1), natts_in + integer varid_in(NVARS), dimid_in, quantize_mode_in, nsd_in + character*(10) var_name_in + real real_data_in(DIM_LEN_5) + real*8 double_data_in(DIM_LEN_5) + real real_data_expected(DIM_LEN_5) + real*8 double_data_expected(DIM_LEN_5) + real diff + real*8 diff8 + + integer x, retval + +C What we expect to get back. + real_data_expected(1) = 1.11084 + real_data_expected(2) = 1.000488 + real_data_expected(3) = 10 + real_data_expected(4) = 12348 + real_data_expected(5) = 0.1234436 + double_data_expected(1) = 1.11083984375 + double_data_expected(2) = 1.00048828125 + double_data_expected(3) = 10 + double_data_expected(4) = 1234698240 + double_data_expected(5) = 1234173952 + +C Check it out. + retval = nf_inq(ncid, ndims_in, nvars_in, ngatts_in, + $ unlimdimid_in) + if (retval .ne. nf_noerr) stop 11 + if (ndims_in .ne. 1 .or. nvars_in .ne. NVARS .or. ngatts_in .ne. 0 + $ .or. unlimdimid_in .ne. -1) stop 12 + +C Get the varids and the dimid. + do x = 1, NVARS + retval = nf_inq_varid(ncid, var_name(x), varid_in(x)) + if (retval .ne. nf_noerr) stop 13 + if (varid_in(x) .ne. x) stop 14 + end do + retval = nf_inq_dimid(ncid, dim_name, dimid_in) + if (retval .ne. nf_noerr) stop 15 + if (dimid_in .ne. 1) stop 16 + +C These things are the same for all variables, except natts_in is +C different for quantize vars after the file is written, there is an +C extra attribute. + do x = 1, NVARS + retval = nf_inq_var(ncid, varid_in(x), var_name_in, xtype_in, + $ ndims_in, dimids_in, natts_in) + if (retval .ne. nf_noerr) stop 17 + if (ndims_in .ne. 1 .or. xtype_in .ne. var_type(x) .or. + $ dimids_in(1).ne. dimid_in) stop 18 + retval = nf_inq_var_quantize(ncid, varid_in(x), + $ quantize_mode_in, nsd_in) + if (quantize_mode_in .ne. nf_quantize_bitgroom) stop 20 + if (nsd_in .ne. NSD_3) stop 20 + end do + +C Get the data. + retval = nf_get_var_real(ncid, varid_in(1), real_data_in) + if (retval .ne. nf_noerr) stop 1 + retval = nf_get_var_double(ncid, varid_in(2), double_data_in) + if (retval .ne. nf_noerr) stop 1 + +C Check the data. + do x = 1, DIM_LEN_5 + diff = abs(real_data_in(x) - real_data_expected(x)) + if (diff .gt. EPSILON) stop 23 + diff = abs(double_data_in(x) - double_data_expected(x)) +c$$$ print *, double_data_in(x), double_data_expected(x) +c$$$ print *, 'x = ', x, ' diff = ', diff + if (diff .gt. EPSILON) stop 24 + end do + + check_file = 0 + end From fb63f79a0085916bd8266f63b3e2e3dec2ede9e6 Mon Sep 17 00:00:00 2001 From: Edward Hartnett Date: Fri, 3 Sep 2021 17:17:06 -0600 Subject: [PATCH 03/16] adding f90 api test for quantize --- fortran/module_netcdf_nc_data.F90 | 4 + fortran/module_netcdf_nf_data.F90 | 6 +- nf03_test4/CMakeLists.txt | 3 +- nf03_test4/Makefile.am | 5 +- nf03_test4/f90tst_vars5.f90 | 192 ++++++++++++++++++++++++++++++ nf_test4/CMakeLists.txt | 3 +- 6 files changed, 208 insertions(+), 5 deletions(-) create mode 100644 nf03_test4/f90tst_vars5.f90 diff --git a/fortran/module_netcdf_nc_data.F90 b/fortran/module_netcdf_nc_data.F90 index 4508fe4e..b112ddd8 100755 --- a/fortran/module_netcdf_nc_data.F90 +++ b/fortran/module_netcdf_nc_data.F90 @@ -405,6 +405,10 @@ Module netcdf_nc_data Integer(C_INT), Parameter :: NC_ECANTEXTEND = -130 Integer(C_INT), Parameter :: NC_EMPI = -131 +! Quantize feature + Integer(C_INT), Parameter :: NC_NOQUANTIZE = 0 + Integer(C_INT), Parameter :: NC_QUANTIZE_BITGROOM = 1 + #endif !------------------------------------------------------------------------------ diff --git a/fortran/module_netcdf_nf_data.F90 b/fortran/module_netcdf_nf_data.F90 index c554a614..e5c070db 100755 --- a/fortran/module_netcdf_nf_data.F90 +++ b/fortran/module_netcdf_nf_data.F90 @@ -161,7 +161,11 @@ Module netcdf_nf_data ! Error handling codes Integer, Parameter :: NF_FATAL = NC_FATAL - Integer, Parameter :: NF_VERBOSE = NC_VERBOSE + Integer, Parameter :: NF_VERBOSE = NC_VERBOSE + + ! Quantize feature + Integer, Parameter :: NF_NOQUANTIZE = NC_NOQUANTIZE + Integer, Parameter :: NF_QUANTIZE_BITGROOM = NC_QUANTIZE_BITGROOM #ifdef ENABLE_CDF5 ! new format types diff --git a/nf03_test4/CMakeLists.txt b/nf03_test4/CMakeLists.txt index 9c876cbb..cbd3832c 100644 --- a/nf03_test4/CMakeLists.txt +++ b/nf03_test4/CMakeLists.txt @@ -14,7 +14,7 @@ FILE(COPY ${COPY_FILES} DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/) if (USE_NETCDF4) # Add these netCDF-4 test programs. SET(nc4_check_PROGRAMS f90tst_vars f90tst_vars_vlen f90tst_grps - f90tst_fill f90tst_fill2 f90tst_vars3 f90tst_vars4 f90tst_vars2 + f90tst_fill f90tst_fill2 f90tst_vars3 f90tst_vars4 f90tst_vars5 f90tst_vars2 f90tst_path f90tst_rengrps f90tst_nc4 f90tst_types f90tst_types2) SET(check_PROGRAMS ${check_PROGRAMS} ${nc4_check_PROGRAMS}) SET(TESTS ${TESTS} ${nc4_check_PROGRAMS}) @@ -27,6 +27,7 @@ if (USE_NETCDF4) SET(f90tst_vars2_SOURCES f90tst_vars2.f90) SET(f90tst_vars3_SOURCES f90tst_vars3.f90) SET(f90tst_vars4_SOURCES f90tst_vars4.f90) + SET(f90tst_vars5_SOURCES f90tst_vars5.f90) SET(f90tst_path_SOURCES f90tst_path.f90) SET(f90tst_rengrps_SOURCES f90tst_rengrps.f90) SET(f90tst_nc4_SOURCES f90tst_nc4.f90) diff --git a/nf03_test4/Makefile.am b/nf03_test4/Makefile.am index cb7579c0..0a47f540 100644 --- a/nf03_test4/Makefile.am +++ b/nf03_test4/Makefile.am @@ -14,8 +14,8 @@ LDADD = ${top_builddir}/fortran/libnetcdff.la # tst_f90_nc4 NC4_F90_TESTS = f90tst_vars f90tst_vars_vlen f90tst_grps f90tst_fill \ -f90tst_fill2 f90tst_vars2 f90tst_vars3 f90tst_vars4 f90tst_path \ -f90tst_rengrps f90tst_nc4 f90tst_types f90tst_types2 +f90tst_fill2 f90tst_vars2 f90tst_vars3 f90tst_vars4 f90tst_vars5 \ +f90tst_path f90tst_rengrps f90tst_nc4 f90tst_types f90tst_types2 check_PROGRAMS = $(NC4_F90_TESTS) TESTS = $(NC4_F90_TESTS) @@ -27,6 +27,7 @@ f90tst_fill2_SOURCES = f90tst_fill2.f90 f90tst_vars2_SOURCES = f90tst_vars2.f90 f90tst_vars3_SOURCES = f90tst_vars3.f90 f90tst_vars4_SOURCES = f90tst_vars4.f90 +f90tst_vars5_SOURCES = f90tst_vars5.f90 f90tst_path_SOURCES = f90tst_path.f90 f90tst_rengrps_SOURCES = f90tst_rengrps.f90 f90tst_nc4_SOURCES = f90tst_nc4.f90 diff --git a/nf03_test4/f90tst_vars5.f90 b/nf03_test4/f90tst_vars5.f90 new file mode 100644 index 00000000..1da7d8ed --- /dev/null +++ b/nf03_test4/f90tst_vars5.f90 @@ -0,0 +1,192 @@ +! This is part of the netCDF package. Copyright 2006-2019 +! University Corporation for Atmospheric Research/Unidata. See +! COPYRIGHT file for conditions of use. + +! This program tests netCDF-4 variable functions from fortran. + +! $Id: f90tst_vars2.f90,v 1.7 2010/01/25 21:01:07 ed Exp $ + +program f90tst_vars5 + use typeSizes + use netcdf + implicit none + + ! This is the name of the data file we will create. + character (len = *), parameter :: FILE_NAME = "f90tst_vars5.nc" + + ! We are writing 2D data, a 6 x 12 grid. + integer, parameter :: MAX_DIMS = 2 + integer, parameter :: NX = 6, NY = 12 + integer :: data_out(NY, NX), data_in(NY, NX) + integer :: data_out_1d(NX), data_in_1d(NX) + + ! We need these ids and other gunk for netcdf. + integer :: ncid, varid1, varid2, varid3, varid4, varid5, dimids(MAX_DIMS) + integer :: chunksizes(MAX_DIMS), chunksizes_in(MAX_DIMS) + integer :: x_dimid, y_dimid + integer :: nvars, ngatts, ndims, unlimdimid, file_format + integer :: x, y + integer, parameter :: DEFLATE_LEVEL = 4 + integer (kind = EightByteInt), parameter :: TOE_SAN_VALUE = 2147483648_EightByteInt + character (len = *), parameter :: VAR1_NAME = "Chon-Ji" + character (len = *), parameter :: VAR2_NAME = "Tan-Gun" + character (len = *), parameter :: VAR3_NAME = "Toe-San" + character (len = *), parameter :: VAR4_NAME = "Won-Hyo" + character (len = *), parameter :: VAR5_NAME = "Yul-Guk" + integer, parameter :: CACHE_SIZE = 8, CACHE_NELEMS = 571 + integer, parameter :: CACHE_PREEMPTION = 66 + + ! Information read back from the file to check correctness. + integer :: varid1_in, varid2_in, varid3_in, varid4_in, varid5_in + integer :: xtype_in, ndims_in, natts_in, dimids_in(MAX_DIMS) + character (len = nf90_max_name) :: name_in + integer :: endianness_in, deflate_level_in + logical :: shuffle_in, fletcher32_in, contiguous_in + integer (kind = EightByteInt) :: toe_san_in + integer :: cache_size_in, cache_nelems_in, cache_preemption_in + + print *, '' + print *,'*** Testing definition of netCDF-4 vars from Fortran 90.' + + ! Create some pretend data. + do x = 1, NX + do y = 1, NY + data_out(y, x) = (x - 1) * NY + (y - 1) + end do + end do + do x = 1, NX + data_out_1d(x) = x + end do + + ! Create the netCDF file. + call check(nf90_create(FILE_NAME, nf90_netcdf4, ncid, cache_nelems = CACHE_NELEMS, & + cache_size = CACHE_SIZE)) + + ! Define the dimensions. + call check(nf90_def_dim(ncid, "x", NX, x_dimid)) + call check(nf90_def_dim(ncid, "y", NY, y_dimid)) + dimids = (/ y_dimid, x_dimid /) + + ! Define some variables. + chunksizes = (/ NY, NX /) + call check(nf90_def_var(ncid, VAR1_NAME, NF90_INT, dimids, varid1, chunksizes = chunksizes, & + shuffle = .TRUE., fletcher32 = .TRUE., endianness = nf90_endian_big, deflate_level = DEFLATE_LEVEL)) + call check(nf90_def_var(ncid, VAR2_NAME, NF90_INT, dimids, varid2, contiguous = .TRUE.)) + call check(nf90_def_var(ncid, VAR3_NAME, NF90_INT64, varid3)) + call check(nf90_def_var(ncid, VAR4_NAME, NF90_INT, x_dimid, varid4, contiguous = .TRUE.)) + call check(nf90_def_var(ncid, VAR5_NAME, NF90_INT, dimids, varid5, chunksizes = chunksizes)) + + ! Write the pretend data to the file. + call check(nf90_put_var(ncid, varid1, data_out)) + call check(nf90_put_var(ncid, varid2, data_out)) + call check(nf90_put_var(ncid, varid3, TOE_SAN_VALUE)) + call check(nf90_put_var(ncid, varid4, data_out_1d)) + call check(nf90_put_var(ncid, varid5, data_out)) + + ! Close the file. + call check(nf90_close(ncid)) + + ! Reopen the file. + call check(nf90_open(FILE_NAME, nf90_nowrite, ncid)) + + ! Check some stuff out. + call check(nf90_inquire(ncid, ndims, nvars, ngatts, unlimdimid, file_format)) + if (ndims /= 2 .or. nvars /= 5 .or. ngatts /= 0 .or. unlimdimid /= -1 .or. & + file_format /= nf90_format_netcdf4) stop 2 + + ! Get varids. + call check(nf90_inq_varid(ncid, VAR1_NAME, varid1_in)) + call check(nf90_inq_varid(ncid, VAR2_NAME, varid2_in)) + call check(nf90_inq_varid(ncid, VAR3_NAME, varid3_in)) + call check(nf90_inq_varid(ncid, VAR4_NAME, varid4_in)) + call check(nf90_inq_varid(ncid, VAR5_NAME, varid5_in)) + + ! Check variable 1. + call check(nf90_inquire_variable(ncid, varid1_in, name_in, xtype_in, ndims_in, dimids_in, & + natts_in, chunksizes = chunksizes_in, endianness = endianness_in, fletcher32 = fletcher32_in, & + deflate_level = deflate_level_in, shuffle = shuffle_in, cache_size = cache_size_in, & + cache_nelems = cache_nelems_in, cache_preemption = cache_preemption_in)) + if (name_in .ne. VAR1_NAME .or. xtype_in .ne. NF90_INT .or. ndims_in .ne. MAX_DIMS .or. & + natts_in .ne. 0 .or. dimids_in(1) .ne. dimids(1) .or. dimids_in(2) .ne. dimids(2)) stop 3 + if (chunksizes_in(1) /= chunksizes(1) .or. chunksizes_in(2) /= chunksizes(2)) & + stop 4 + if (endianness_in .ne. nf90_endian_big) stop 5 + + ! This test code commented out because it fails parallel builds, + ! which don't use the cache, so don't get the same size + ! settings. Since we are not (yet) using a preprocessor on the + ! fortran code, there's no way to ifdef out these tests. + ! print *, cache_size_in, cache_nelems_in, cache_preemption + ! if (cache_size_in .ne. 16 .or. cache_nelems_in .ne. 4133 .or. & + ! cache_preemption .ne. CACHE_PREEMPTION) stop 555 + + ! Check variable 2. + call check(nf90_inquire_variable(ncid, varid2_in, name_in, xtype_in, ndims_in, dimids_in, & + natts_in, contiguous = contiguous_in, endianness = endianness_in, fletcher32 = fletcher32_in, & + deflate_level = deflate_level_in, shuffle = shuffle_in)) + if (name_in .ne. VAR2_NAME .or. xtype_in .ne. NF90_INT .or. ndims_in .ne. MAX_DIMS .or. & + natts_in .ne. 0 .or. dimids_in(1) .ne. dimids(1) .or. dimids_in(2) .ne. dimids(2)) stop 6 + if (deflate_level_in .ne. 0 .or. .not. contiguous_in .or. fletcher32_in .or. shuffle_in) stop 7 + + ! Check variable 3. + call check(nf90_inquire_variable(ncid, varid3_in, name_in, xtype_in, ndims_in, dimids_in, & + natts_in, contiguous = contiguous_in, endianness = endianness_in, fletcher32 = fletcher32_in, & + deflate_level = deflate_level_in, shuffle = shuffle_in)) + if (name_in .ne. VAR3_NAME .or. xtype_in .ne. NF90_INT64 .or. ndims_in .ne. 0 .or. & + natts_in .ne. 0) stop 8 + if (deflate_level_in .ne. 0 .or. .not. contiguous_in .or. fletcher32_in .or. shuffle_in) stop 9 + + ! Check variable 4. + call check(nf90_inquire_variable(ncid, varid4_in, name_in, xtype_in, ndims_in, dimids_in, & + natts_in, contiguous = contiguous_in, endianness = endianness_in, fletcher32 = fletcher32_in, & + deflate_level = deflate_level_in, shuffle = shuffle_in)) + if (name_in .ne. VAR4_NAME .or. xtype_in .ne. NF90_INT .or. ndims_in .ne. 1 .or. & + natts_in .ne. 0 .or. dimids_in(1) .ne. x_dimid) stop 10 + if (deflate_level_in .ne. 0 .or. .not. contiguous_in .or. fletcher32_in .or. shuffle_in) stop 11 + + ! Check the data. + call check(nf90_get_var(ncid, varid1_in, data_in)) + do x = 1, NX + do y = 1, NY + if (data_out(y, x) .ne. data_in(y, x)) stop 12 + end do + end do + call check(nf90_get_var(ncid, varid2_in, data_in)) + do x = 1, NX + do y = 1, NY + if (data_out(y, x) .ne. data_in(y, x)) stop 13 + end do + end do + call check(nf90_get_var(ncid, varid3_in, toe_san_in)) + if (toe_san_in .ne. TOE_SAN_VALUE) stop 14 + call check(nf90_get_var(ncid, varid4_in, data_in_1d)) + do x = 1, NX + if (data_out_1d(x) .ne. data_in_1d(x)) stop 15 + end do + call check(nf90_get_var(ncid, varid5_in, data_in)) + do x = 1, NX + do y = 1, NY + if (data_out(y, x) .ne. data_in(y, x)) stop 12 + end do + end do + + ! Close the file. + call check(nf90_close(ncid)) + + print *,'*** SUCCESS!' + +contains +! This subroutine handles errors by printing an error message and +! exiting with a non-zero status. + subroutine check(errcode) + use netcdf + implicit none + integer, intent(in) :: errcode + + if(errcode /= nf90_noerr) then + print *, 'Error: ', trim(nf90_strerror(errcode)) + stop 2 + endif + end subroutine check +end program f90tst_vars5 + diff --git a/nf_test4/CMakeLists.txt b/nf_test4/CMakeLists.txt index 9a404609..260c6190 100644 --- a/nf_test4/CMakeLists.txt +++ b/nf_test4/CMakeLists.txt @@ -8,7 +8,7 @@ SET(CMAKE_INCLUDE_CURRENT_DIR ON) if (USE_NETCDF4) # Add these netCDF-4 test programs. SET(nc4_check_PROGRAMS ftst_groups ftst_vars ftst_vars2 ftst_vars3 - ftst_vars4 ftst_vars5 ftst_vars6 ftst_types ftst_types2 + ftst_vars4 ftst_vars5 ftst_vars6 ftst_vars7 ftst_types ftst_types2 ftst_types3 ftst_path ftst_rengrps f03tst_open_mem ftst_var_compact) SET(check_PROGRAMS ${check_PROGRAMS} ${nc4_check_PROGRAMS}) SET(TESTS ${TESTS} ${nc4_check_PROGRAMS}) @@ -20,6 +20,7 @@ if (USE_NETCDF4) SET(ftst_vars4_SOURCES ftst_vars4.F) SET(ftst_vars5_SOURCES ftst_vars5.F) SET(ftst_vars6_SOURCES ftst_vars6.F) + SET(ftst_vars7_SOURCES ftst_vars7.F) SET(ftst_types_SOURCES ftst_types.F) SET(ftst_types2_SOURCES ftst_types2.F) SET(ftst_types3_SOURCES ftst_types3.F) From 449197c937f4a8fde3f5b200accafd8220943523 Mon Sep 17 00:00:00 2001 From: Edward Hartnett Date: Sat, 4 Sep 2021 04:38:37 -0600 Subject: [PATCH 04/16] adding quantize to f90 API --- fortran/netcdf4_variables.f90 | 28 ++++- nf03_test4/f90tst_vars5.f90 | 227 +++++++++++++++++----------------- 2 files changed, 140 insertions(+), 115 deletions(-) diff --git a/fortran/netcdf4_variables.f90 b/fortran/netcdf4_variables.f90 index 399d6275..3235f8b5 100644 --- a/fortran/netcdf4_variables.f90 +++ b/fortran/netcdf4_variables.f90 @@ -19,7 +19,7 @@ end function nf90_def_var_Scalar ! ----- function nf90_def_var_oneDim(ncid, name, xtype, dimids, varid, contiguous, & chunksizes, deflate_level, shuffle, fletcher32, endianness, & - cache_size, cache_nelems, cache_preemption) + cache_size, cache_nelems, cache_preemption, quantize_mode, nsd) integer, intent( in) :: ncid character (len = *), intent( in) :: name integer, intent(in) :: xtype @@ -31,6 +31,7 @@ function nf90_def_var_oneDim(ncid, name, xtype, dimids, varid, contiguous, & logical, optional, intent(in) :: shuffle, fletcher32 integer, optional, intent(in) :: endianness integer, optional, intent(in) :: cache_size, cache_nelems, cache_preemption + integer, optional, intent(in) :: quantize_mode, nsd integer :: nf90_def_var_oneDim integer, dimension(1) :: dimidsA, chunksizes1 @@ -123,11 +124,23 @@ function nf90_def_var_oneDim(ncid, name, xtype, dimids, varid, contiguous, & if (nf90_def_var_oneDim .ne. nf90_noerr) return endif + ! Set quantize if the user wants to. + if (present(quantize_mode)) then + if (.not. present(nsd)) then + nf90_def_var_oneDim = nf90_einval + return + endif + nf90_def_var_oneDim = nf_def_var_quantize(ncid, varid, quantize_mode, nsd) + if (nf90_def_var_oneDim .ne. nf90_noerr) return + + endif + + end function nf90_def_var_oneDim ! ----- function nf90_def_var_ManyDims(ncid, name, xtype, dimids, varid, contiguous, & chunksizes, deflate_level, shuffle, fletcher32, endianness, cache_size, & - cache_nelems, cache_preemption) + cache_nelems, cache_preemption, quantize_mode, nsd) integer, intent(in) :: ncid character (len = *), intent(in) :: name integer, intent( in) :: xtype @@ -139,6 +152,7 @@ function nf90_def_var_ManyDims(ncid, name, xtype, dimids, varid, contiguous, & logical, optional, intent(in) :: shuffle, fletcher32 integer, optional, intent(in) :: endianness integer, optional, intent(in) :: cache_size, cache_nelems, cache_preemption + integer, optional, intent(in) :: quantize_mode, nsd integer :: nf90_def_var_ManyDims ! Local variables. @@ -239,6 +253,16 @@ function nf90_def_var_ManyDims(ncid, name, xtype, dimids, varid, contiguous, & if (nf90_def_var_ManyDims .ne. nf90_noerr) return endif + ! Set quantize if the user wants to. + if (present(quantize_mode)) then + if (.not. present(nsd)) then + nf90_def_var_ManyDims = nf90_einval + return + endif + nf90_def_var_ManyDims = nf_def_var_quantize(ncid, varid, quantize_mode, nsd) + if (nf90_def_var_ManyDims .ne. nf90_noerr) return + endif + end function nf90_def_var_ManyDims ! ----- function nf90_inq_varid(ncid, name, varid) diff --git a/nf03_test4/f90tst_vars5.f90 b/nf03_test4/f90tst_vars5.f90 index 1da7d8ed..013d92a1 100644 --- a/nf03_test4/f90tst_vars5.f90 +++ b/nf03_test4/f90tst_vars5.f90 @@ -15,14 +15,15 @@ program f90tst_vars5 character (len = *), parameter :: FILE_NAME = "f90tst_vars5.nc" ! We are writing 2D data, a 6 x 12 grid. - integer, parameter :: MAX_DIMS = 2 - integer, parameter :: NX = 6, NY = 12 - integer :: data_out(NY, NX), data_in(NY, NX) - integer :: data_out_1d(NX), data_in_1d(NX) + integer, parameter :: NDIM1 = 1 + integer, parameter :: DIM_LEN_5 = 5 + integer start(NDIM1), count(NDIM1) + real real_data(DIM_LEN_5) + real*8 double_data(DIM_LEN_5) ! We need these ids and other gunk for netcdf. - integer :: ncid, varid1, varid2, varid3, varid4, varid5, dimids(MAX_DIMS) - integer :: chunksizes(MAX_DIMS), chunksizes_in(MAX_DIMS) + integer :: ncid, varid1, varid2, varid3, varid4, varid5, dimids(NDIM1) + integer :: chunksizes(NDIM1), chunksizes_in(NDIM1) integer :: x_dimid, y_dimid integer :: nvars, ngatts, ndims, unlimdimid, file_format integer :: x, y @@ -38,7 +39,7 @@ program f90tst_vars5 ! Information read back from the file to check correctness. integer :: varid1_in, varid2_in, varid3_in, varid4_in, varid5_in - integer :: xtype_in, ndims_in, natts_in, dimids_in(MAX_DIMS) + integer :: xtype_in, ndims_in, natts_in, dimids_in(NDIM1) character (len = nf90_max_name) :: name_in integer :: endianness_in, deflate_level_in logical :: shuffle_in, fletcher32_in, contiguous_in @@ -46,42 +47,42 @@ program f90tst_vars5 integer :: cache_size_in, cache_nelems_in, cache_preemption_in print *, '' - print *,'*** Testing definition of netCDF-4 vars from Fortran 90.' + print *,'*** Testing use of quantize feature on netCDF-4 vars from Fortran 90.' ! Create some pretend data. - do x = 1, NX - do y = 1, NY - data_out(y, x) = (x - 1) * NY + (y - 1) - end do - end do - do x = 1, NX - data_out_1d(x) = x - end do + real_data(1) = 1.11111111 + real_data(2) = 1.0 + real_data(3) = 9.99999999 + real_data(4) = 12345.67 + real_data(5) = .1234567 + double_data(1) = 1.1111111 + double_data(2) = 1.0 + double_data(3) = 9.999999999 + double_data(4) = 1234567890.12345 + double_data(5) = 1234567890 ! Create the netCDF file. call check(nf90_create(FILE_NAME, nf90_netcdf4, ncid, cache_nelems = CACHE_NELEMS, & cache_size = CACHE_SIZE)) - ! Define the dimensions. - call check(nf90_def_dim(ncid, "x", NX, x_dimid)) - call check(nf90_def_dim(ncid, "y", NY, y_dimid)) - dimids = (/ y_dimid, x_dimid /) + ! Define the dimension. + call check(nf90_def_dim(ncid, "x", DIM_LEN_5, x_dimid)) + dimids = (/ x_dimid /) ! Define some variables. - chunksizes = (/ NY, NX /) - call check(nf90_def_var(ncid, VAR1_NAME, NF90_INT, dimids, varid1, chunksizes = chunksizes, & - shuffle = .TRUE., fletcher32 = .TRUE., endianness = nf90_endian_big, deflate_level = DEFLATE_LEVEL)) - call check(nf90_def_var(ncid, VAR2_NAME, NF90_INT, dimids, varid2, contiguous = .TRUE.)) - call check(nf90_def_var(ncid, VAR3_NAME, NF90_INT64, varid3)) - call check(nf90_def_var(ncid, VAR4_NAME, NF90_INT, x_dimid, varid4, contiguous = .TRUE.)) - call check(nf90_def_var(ncid, VAR5_NAME, NF90_INT, dimids, varid5, chunksizes = chunksizes)) - - ! Write the pretend data to the file. - call check(nf90_put_var(ncid, varid1, data_out)) - call check(nf90_put_var(ncid, varid2, data_out)) - call check(nf90_put_var(ncid, varid3, TOE_SAN_VALUE)) - call check(nf90_put_var(ncid, varid4, data_out_1d)) - call check(nf90_put_var(ncid, varid5, data_out)) + call check(nf90_def_var(ncid, VAR1_NAME, NF90_FLOAT, dimids, varid1& + &, deflate_level = DEFLATE_LEVEL, quantize_mode =& + & nf90_quantize_bitgroom, nsd = 3)) + call check(nf90_def_var(ncid, VAR2_NAME, NF90_DOUBLE, dimids,& + & varid2, contiguous = .TRUE., quantize_mode =& + & nf90_quantize_bitgroom, nsd = 3)) + + ! ! Write the pretend data to the file. + ! call check(nf90_put_var(ncid, varid1, data_out)) + ! call check(nf90_put_var(ncid, varid2, data_out)) + ! call check(nf90_put_var(ncid, varid3, TOE_SAN_VALUE)) + ! call check(nf90_put_var(ncid, varid4, data_out_1d)) + ! call check(nf90_put_var(ncid, varid5, data_out)) ! Close the file. call check(nf90_close(ncid)) @@ -89,86 +90,86 @@ program f90tst_vars5 ! Reopen the file. call check(nf90_open(FILE_NAME, nf90_nowrite, ncid)) - ! Check some stuff out. - call check(nf90_inquire(ncid, ndims, nvars, ngatts, unlimdimid, file_format)) - if (ndims /= 2 .or. nvars /= 5 .or. ngatts /= 0 .or. unlimdimid /= -1 .or. & - file_format /= nf90_format_netcdf4) stop 2 - - ! Get varids. - call check(nf90_inq_varid(ncid, VAR1_NAME, varid1_in)) - call check(nf90_inq_varid(ncid, VAR2_NAME, varid2_in)) - call check(nf90_inq_varid(ncid, VAR3_NAME, varid3_in)) - call check(nf90_inq_varid(ncid, VAR4_NAME, varid4_in)) - call check(nf90_inq_varid(ncid, VAR5_NAME, varid5_in)) - - ! Check variable 1. - call check(nf90_inquire_variable(ncid, varid1_in, name_in, xtype_in, ndims_in, dimids_in, & - natts_in, chunksizes = chunksizes_in, endianness = endianness_in, fletcher32 = fletcher32_in, & - deflate_level = deflate_level_in, shuffle = shuffle_in, cache_size = cache_size_in, & - cache_nelems = cache_nelems_in, cache_preemption = cache_preemption_in)) - if (name_in .ne. VAR1_NAME .or. xtype_in .ne. NF90_INT .or. ndims_in .ne. MAX_DIMS .or. & - natts_in .ne. 0 .or. dimids_in(1) .ne. dimids(1) .or. dimids_in(2) .ne. dimids(2)) stop 3 - if (chunksizes_in(1) /= chunksizes(1) .or. chunksizes_in(2) /= chunksizes(2)) & - stop 4 - if (endianness_in .ne. nf90_endian_big) stop 5 - - ! This test code commented out because it fails parallel builds, - ! which don't use the cache, so don't get the same size - ! settings. Since we are not (yet) using a preprocessor on the - ! fortran code, there's no way to ifdef out these tests. - ! print *, cache_size_in, cache_nelems_in, cache_preemption - ! if (cache_size_in .ne. 16 .or. cache_nelems_in .ne. 4133 .or. & - ! cache_preemption .ne. CACHE_PREEMPTION) stop 555 - - ! Check variable 2. - call check(nf90_inquire_variable(ncid, varid2_in, name_in, xtype_in, ndims_in, dimids_in, & - natts_in, contiguous = contiguous_in, endianness = endianness_in, fletcher32 = fletcher32_in, & - deflate_level = deflate_level_in, shuffle = shuffle_in)) - if (name_in .ne. VAR2_NAME .or. xtype_in .ne. NF90_INT .or. ndims_in .ne. MAX_DIMS .or. & - natts_in .ne. 0 .or. dimids_in(1) .ne. dimids(1) .or. dimids_in(2) .ne. dimids(2)) stop 6 - if (deflate_level_in .ne. 0 .or. .not. contiguous_in .or. fletcher32_in .or. shuffle_in) stop 7 - - ! Check variable 3. - call check(nf90_inquire_variable(ncid, varid3_in, name_in, xtype_in, ndims_in, dimids_in, & - natts_in, contiguous = contiguous_in, endianness = endianness_in, fletcher32 = fletcher32_in, & - deflate_level = deflate_level_in, shuffle = shuffle_in)) - if (name_in .ne. VAR3_NAME .or. xtype_in .ne. NF90_INT64 .or. ndims_in .ne. 0 .or. & - natts_in .ne. 0) stop 8 - if (deflate_level_in .ne. 0 .or. .not. contiguous_in .or. fletcher32_in .or. shuffle_in) stop 9 + ! ! Check some stuff out. + ! call check(nf90_inquire(ncid, ndims, nvars, ngatts, unlimdimid, file_format)) + ! if (ndims /= 2 .or. nvars /= 5 .or. ngatts /= 0 .or. unlimdimid /= -1 .or. & + ! file_format /= nf90_format_netcdf4) stop 2 + + ! ! Get varids. + ! call check(nf90_inq_varid(ncid, VAR1_NAME, varid1_in)) + ! call check(nf90_inq_varid(ncid, VAR2_NAME, varid2_in)) + ! call check(nf90_inq_varid(ncid, VAR3_NAME, varid3_in)) + ! call check(nf90_inq_varid(ncid, VAR4_NAME, varid4_in)) + ! call check(nf90_inq_varid(ncid, VAR5_NAME, varid5_in)) + + ! ! Check variable 1. + ! call check(nf90_inquire_variable(ncid, varid1_in, name_in, xtype_in, ndims_in, dimids_in, & + ! natts_in, chunksizes = chunksizes_in, endianness = endianness_in, fletcher32 = fletcher32_in, & + ! deflate_level = deflate_level_in, shuffle = shuffle_in, cache_size = cache_size_in, & + ! cache_nelems = cache_nelems_in, cache_preemption = cache_preemption_in)) + ! if (name_in .ne. VAR1_NAME .or. xtype_in .ne. NF90_INT .or. ndims_in .ne. NDIM1 .or. & + ! natts_in .ne. 0 .or. dimids_in(1) .ne. dimids(1) .or. dimids_in(2) .ne. dimids(2)) stop 3 + ! if (chunksizes_in(1) /= chunksizes(1) .or. chunksizes_in(2) /= chunksizes(2)) & + ! stop 4 + ! if (endianness_in .ne. nf90_endian_big) stop 5 + + ! ! This test code commented out because it fails parallel builds, + ! ! which don't use the cache, so don't get the same size + ! ! settings. Since we are not (yet) using a preprocessor on the + ! ! fortran code, there's no way to ifdef out these tests. + ! ! print *, cache_size_in, cache_nelems_in, cache_preemption + ! ! if (cache_size_in .ne. 16 .or. cache_nelems_in .ne. 4133 .or. & + ! ! cache_preemption .ne. CACHE_PREEMPTION) stop 555 + + ! ! Check variable 2. + ! call check(nf90_inquire_variable(ncid, varid2_in, name_in, xtype_in, ndims_in, dimids_in, & + ! natts_in, contiguous = contiguous_in, endianness = endianness_in, fletcher32 = fletcher32_in, & + ! deflate_level = deflate_level_in, shuffle = shuffle_in)) + ! if (name_in .ne. VAR2_NAME .or. xtype_in .ne. NF90_INT .or. ndims_in .ne. NDIM1 .or. & + ! natts_in .ne. 0 .or. dimids_in(1) .ne. dimids(1) .or. dimids_in(2) .ne. dimids(2)) stop 6 + ! if (deflate_level_in .ne. 0 .or. .not. contiguous_in .or. fletcher32_in .or. shuffle_in) stop 7 + + ! ! Check variable 3. + ! call check(nf90_inquire_variable(ncid, varid3_in, name_in, xtype_in, ndims_in, dimids_in, & + ! natts_in, contiguous = contiguous_in, endianness = endianness_in, fletcher32 = fletcher32_in, & + ! deflate_level = deflate_level_in, shuffle = shuffle_in)) + ! if (name_in .ne. VAR3_NAME .or. xtype_in .ne. NF90_INT64 .or. ndims_in .ne. 0 .or. & + ! natts_in .ne. 0) stop 8 + ! if (deflate_level_in .ne. 0 .or. .not. contiguous_in .or. fletcher32_in .or. shuffle_in) stop 9 - ! Check variable 4. - call check(nf90_inquire_variable(ncid, varid4_in, name_in, xtype_in, ndims_in, dimids_in, & - natts_in, contiguous = contiguous_in, endianness = endianness_in, fletcher32 = fletcher32_in, & - deflate_level = deflate_level_in, shuffle = shuffle_in)) - if (name_in .ne. VAR4_NAME .or. xtype_in .ne. NF90_INT .or. ndims_in .ne. 1 .or. & - natts_in .ne. 0 .or. dimids_in(1) .ne. x_dimid) stop 10 - if (deflate_level_in .ne. 0 .or. .not. contiguous_in .or. fletcher32_in .or. shuffle_in) stop 11 - - ! Check the data. - call check(nf90_get_var(ncid, varid1_in, data_in)) - do x = 1, NX - do y = 1, NY - if (data_out(y, x) .ne. data_in(y, x)) stop 12 - end do - end do - call check(nf90_get_var(ncid, varid2_in, data_in)) - do x = 1, NX - do y = 1, NY - if (data_out(y, x) .ne. data_in(y, x)) stop 13 - end do - end do - call check(nf90_get_var(ncid, varid3_in, toe_san_in)) - if (toe_san_in .ne. TOE_SAN_VALUE) stop 14 - call check(nf90_get_var(ncid, varid4_in, data_in_1d)) - do x = 1, NX - if (data_out_1d(x) .ne. data_in_1d(x)) stop 15 - end do - call check(nf90_get_var(ncid, varid5_in, data_in)) - do x = 1, NX - do y = 1, NY - if (data_out(y, x) .ne. data_in(y, x)) stop 12 - end do - end do + ! ! Check variable 4. + ! call check(nf90_inquire_variable(ncid, varid4_in, name_in, xtype_in, ndims_in, dimids_in, & + ! natts_in, contiguous = contiguous_in, endianness = endianness_in, fletcher32 = fletcher32_in, & + ! deflate_level = deflate_level_in, shuffle = shuffle_in)) + ! if (name_in .ne. VAR4_NAME .or. xtype_in .ne. NF90_INT .or. ndims_in .ne. 1 .or. & + ! natts_in .ne. 0 .or. dimids_in(1) .ne. x_dimid) stop 10 + ! if (deflate_level_in .ne. 0 .or. .not. contiguous_in .or. fletcher32_in .or. shuffle_in) stop 11 + + ! ! Check the data. + ! call check(nf90_get_var(ncid, varid1_in, data_in)) + ! do x = 1, DIM_LEN_5 + ! do y = 1, NY + ! if (data_out(y, x) .ne. data_in(y, x)) stop 12 + ! end do + ! end do + ! call check(nf90_get_var(ncid, varid2_in, data_in)) + ! do x = 1, DIM_LEN_5 + ! do y = 1, NY + ! if (data_out(y, x) .ne. data_in(y, x)) stop 13 + ! end do + ! end do + ! call check(nf90_get_var(ncid, varid3_in, toe_san_in)) + ! if (toe_san_in .ne. TOE_SAN_VALUE) stop 14 + ! call check(nf90_get_var(ncid, varid4_in, data_in_1d)) + ! do x = 1, DIM_LEN_5 + ! if (data_out_1d(x) .ne. data_in_1d(x)) stop 15 + ! end do + ! call check(nf90_get_var(ncid, varid5_in, data_in)) + ! do x = 1, DIM_LEN_5 + ! do y = 1, NY + ! if (data_out(y, x) .ne. data_in(y, x)) stop 12 + ! end do + ! end do ! Close the file. call check(nf90_close(ncid)) From 83b3bd97267023e22a7806e31cd7ed0d4f1ff5be Mon Sep 17 00:00:00 2001 From: Edward Hartnett Date: Sat, 4 Sep 2021 04:56:35 -0600 Subject: [PATCH 05/16] more adding quantize to F90 API --- fortran/netcdf4_variables.f90 | 10 +++- nf03_test4/f90tst_vars5.f90 | 108 +++++++++++++++------------------- nf_test4/ftst_vars7.F | 22 +++---- 3 files changed, 67 insertions(+), 73 deletions(-) diff --git a/fortran/netcdf4_variables.f90 b/fortran/netcdf4_variables.f90 index 3235f8b5..b7b1e4b4 100644 --- a/fortran/netcdf4_variables.f90 +++ b/fortran/netcdf4_variables.f90 @@ -288,7 +288,7 @@ end function nf90_set_var_chunk_cache ! ----- function nf90_inquire_variable(ncid, varid, name, xtype, ndims, dimids, nAtts, & contiguous, chunksizes, deflate_level, shuffle, fletcher32, endianness, & - cache_size, cache_nelems, cache_preemption) + cache_size, cache_nelems, cache_preemption, quantize_mode, nsd) integer, intent(in) :: ncid, varid character (len = *), optional, intent(out) :: name integer, optional, intent(out) :: xtype, ndims @@ -300,6 +300,7 @@ function nf90_inquire_variable(ncid, varid, name, xtype, ndims, dimids, nAtts, & logical, optional, intent(out) :: shuffle, fletcher32 integer, optional, intent(out) :: endianness integer, optional, intent(out) :: cache_size, cache_nelems, cache_preemption + integer, optional, intent(out) :: quantize_mode, nsd integer :: nf90_inquire_variable ! Local variables @@ -372,6 +373,13 @@ function nf90_inquire_variable(ncid, varid, name, xtype, ndims, dimids, nAtts, & if (present(cache_nelems)) cache_nelems = nelems1 if (present(cache_preemption)) cache_preemption = preemption1 endif + + ! And the quantization... + if (present(quantize_mode)) then + nf90_inquire_variable = nf_inq_var_quantize(ncid, varid, quantize_mode, nsd) + if (nf90_inquire_variable .ne. nf90_noerr) return + endif + end function nf90_inquire_variable ! ----- function nf90_rename_var(ncid, varid, newname) diff --git a/nf03_test4/f90tst_vars5.f90 b/nf03_test4/f90tst_vars5.f90 index 013d92a1..b9a111b2 100644 --- a/nf03_test4/f90tst_vars5.f90 +++ b/nf03_test4/f90tst_vars5.f90 @@ -44,7 +44,12 @@ program f90tst_vars5 integer :: endianness_in, deflate_level_in logical :: shuffle_in, fletcher32_in, contiguous_in integer (kind = EightByteInt) :: toe_san_in - integer :: cache_size_in, cache_nelems_in, cache_preemption_in + integer :: cache_size_in, cache_nelems_in, cache_preemption_in + integer :: quantize_mode_in, nsd_in + real real_data_in(DIM_LEN_5) + real*8 double_data_in(DIM_LEN_5) + real real_data_expected(DIM_LEN_5) + real*8 double_data_expected(DIM_LEN_5) print *, '' print *,'*** Testing use of quantize feature on netCDF-4 vars from Fortran 90.' @@ -77,73 +82,54 @@ program f90tst_vars5 & varid2, contiguous = .TRUE., quantize_mode =& & nf90_quantize_bitgroom, nsd = 3)) - ! ! Write the pretend data to the file. - ! call check(nf90_put_var(ncid, varid1, data_out)) - ! call check(nf90_put_var(ncid, varid2, data_out)) - ! call check(nf90_put_var(ncid, varid3, TOE_SAN_VALUE)) - ! call check(nf90_put_var(ncid, varid4, data_out_1d)) - ! call check(nf90_put_var(ncid, varid5, data_out)) + ! Write the pretend data to the file. + call check(nf90_put_var(ncid, varid1, real_data)) + call check(nf90_put_var(ncid, varid2, double_data)) ! Close the file. call check(nf90_close(ncid)) + ! What we expect to get back. + real_data_expected(1) = 1.11084 + real_data_expected(2) = 1.000488 + real_data_expected(3) = 10 + real_data_expected(4) = 12348 + real_data_expected(5) = 0.1234436 + double_data_expected(1) = 1.11083984375 + double_data_expected(2) = 1.00048828125 + double_data_expected(3) = 10 + double_data_expected(4) = 1234698240 + double_data_expected(5) = 1234173952 + ! Reopen the file. call check(nf90_open(FILE_NAME, nf90_nowrite, ncid)) - ! ! Check some stuff out. - ! call check(nf90_inquire(ncid, ndims, nvars, ngatts, unlimdimid, file_format)) - ! if (ndims /= 2 .or. nvars /= 5 .or. ngatts /= 0 .or. unlimdimid /= -1 .or. & - ! file_format /= nf90_format_netcdf4) stop 2 - - ! ! Get varids. - ! call check(nf90_inq_varid(ncid, VAR1_NAME, varid1_in)) - ! call check(nf90_inq_varid(ncid, VAR2_NAME, varid2_in)) - ! call check(nf90_inq_varid(ncid, VAR3_NAME, varid3_in)) - ! call check(nf90_inq_varid(ncid, VAR4_NAME, varid4_in)) - ! call check(nf90_inq_varid(ncid, VAR5_NAME, varid5_in)) - - ! ! Check variable 1. - ! call check(nf90_inquire_variable(ncid, varid1_in, name_in, xtype_in, ndims_in, dimids_in, & - ! natts_in, chunksizes = chunksizes_in, endianness = endianness_in, fletcher32 = fletcher32_in, & - ! deflate_level = deflate_level_in, shuffle = shuffle_in, cache_size = cache_size_in, & - ! cache_nelems = cache_nelems_in, cache_preemption = cache_preemption_in)) - ! if (name_in .ne. VAR1_NAME .or. xtype_in .ne. NF90_INT .or. ndims_in .ne. NDIM1 .or. & - ! natts_in .ne. 0 .or. dimids_in(1) .ne. dimids(1) .or. dimids_in(2) .ne. dimids(2)) stop 3 - ! if (chunksizes_in(1) /= chunksizes(1) .or. chunksizes_in(2) /= chunksizes(2)) & - ! stop 4 - ! if (endianness_in .ne. nf90_endian_big) stop 5 - - ! ! This test code commented out because it fails parallel builds, - ! ! which don't use the cache, so don't get the same size - ! ! settings. Since we are not (yet) using a preprocessor on the - ! ! fortran code, there's no way to ifdef out these tests. - ! ! print *, cache_size_in, cache_nelems_in, cache_preemption - ! ! if (cache_size_in .ne. 16 .or. cache_nelems_in .ne. 4133 .or. & - ! ! cache_preemption .ne. CACHE_PREEMPTION) stop 555 - - ! ! Check variable 2. - ! call check(nf90_inquire_variable(ncid, varid2_in, name_in, xtype_in, ndims_in, dimids_in, & - ! natts_in, contiguous = contiguous_in, endianness = endianness_in, fletcher32 = fletcher32_in, & - ! deflate_level = deflate_level_in, shuffle = shuffle_in)) - ! if (name_in .ne. VAR2_NAME .or. xtype_in .ne. NF90_INT .or. ndims_in .ne. NDIM1 .or. & - ! natts_in .ne. 0 .or. dimids_in(1) .ne. dimids(1) .or. dimids_in(2) .ne. dimids(2)) stop 6 - ! if (deflate_level_in .ne. 0 .or. .not. contiguous_in .or. fletcher32_in .or. shuffle_in) stop 7 - - ! ! Check variable 3. - ! call check(nf90_inquire_variable(ncid, varid3_in, name_in, xtype_in, ndims_in, dimids_in, & - ! natts_in, contiguous = contiguous_in, endianness = endianness_in, fletcher32 = fletcher32_in, & - ! deflate_level = deflate_level_in, shuffle = shuffle_in)) - ! if (name_in .ne. VAR3_NAME .or. xtype_in .ne. NF90_INT64 .or. ndims_in .ne. 0 .or. & - ! natts_in .ne. 0) stop 8 - ! if (deflate_level_in .ne. 0 .or. .not. contiguous_in .or. fletcher32_in .or. shuffle_in) stop 9 - - ! ! Check variable 4. - ! call check(nf90_inquire_variable(ncid, varid4_in, name_in, xtype_in, ndims_in, dimids_in, & - ! natts_in, contiguous = contiguous_in, endianness = endianness_in, fletcher32 = fletcher32_in, & - ! deflate_level = deflate_level_in, shuffle = shuffle_in)) - ! if (name_in .ne. VAR4_NAME .or. xtype_in .ne. NF90_INT .or. ndims_in .ne. 1 .or. & - ! natts_in .ne. 0 .or. dimids_in(1) .ne. x_dimid) stop 10 - ! if (deflate_level_in .ne. 0 .or. .not. contiguous_in .or. fletcher32_in .or. shuffle_in) stop 11 + ! Check some stuff out. + call check(nf90_inquire(ncid, ndims, nvars, ngatts, unlimdimid, file_format)) + if (ndims /= 1 .or. nvars /= 2 .or. ngatts /= 0 .or. unlimdimid /= -1 .or. & + file_format /= nf90_format_netcdf4) stop 2 + + ! Get varids. + call check(nf90_inq_varid(ncid, VAR1_NAME, varid1_in)) + call check(nf90_inq_varid(ncid, VAR2_NAME, varid2_in)) + + ! Check variable 1. + call check(nf90_inquire_variable(ncid, varid1_in, name_in, xtype_in, ndims_in, dimids_in, & + natts_in, chunksizes = chunksizes_in, endianness = endianness_in, fletcher32 = fletcher32_in, & + deflate_level = deflate_level_in, shuffle = shuffle_in, cache_size = cache_size_in, & + cache_nelems = cache_nelems_in, cache_preemption = cache_preemption_in, & + quantize_mode = quantize_mode_in, nsd = nsd_in)) + if (name_in .ne. VAR1_NAME .or. xtype_in .ne. NF90_FLOAT .or. ndims_in .ne. NDIM1 .or. & + natts_in .ne. 1 .or. dimids_in(1) .ne. dimids(1)) stop 3 + + ! Check variable 2. + call check(nf90_inquire_variable(ncid, varid2_in, name_in, xtype_in, ndims_in, dimids_in, & + natts_in, contiguous = contiguous_in, endianness = endianness_in, fletcher32 = fletcher32_in, & + deflate_level = deflate_level_in, shuffle = shuffle_in, & + quantize_mode = quantize_mode_in, nsd = nsd_in)) + if (name_in .ne. VAR2_NAME .or. xtype_in .ne. NF90_DOUBLE .or. ndims_in .ne. NDIM1 .or. & + natts_in .ne. 1 .or. dimids_in(1) .ne. dimids(1)) stop 6 + if (deflate_level_in .ne. 0 .or. .not. contiguous_in .or. fletcher32_in .or. shuffle_in) stop 7 ! ! Check the data. ! call check(nf90_get_var(ncid, varid1_in, data_in)) diff --git a/nf_test4/ftst_vars7.F b/nf_test4/ftst_vars7.F index 4937a332..23c6473f 100644 --- a/nf_test4/ftst_vars7.F +++ b/nf_test4/ftst_vars7.F @@ -97,17 +97,17 @@ program ftst_vars7 retval = nf_close(ncid) if (retval .ne. nf_noerr) stop 4 -c$$$C Reopen the file. -c$$$ retval = nf_open(FILE_NAME, NF_NOWRITE, ncid) -c$$$ if (retval .ne. nf_noerr) stop 5 -c$$$ -c$$$C Check it out. -c$$$ retval = check_file(ncid, var_name, var_type, dim_name) -c$$$ if (retval .ne. 0) stop 6 -c$$$ -c$$$C Close the file. -c$$$ retval = nf_close(ncid) -c$$$ if (retval .ne. nf_noerr) stop 7 +C Reopen the file. + retval = nf_open(FILE_NAME, NF_NOWRITE, ncid) + if (retval .ne. nf_noerr) stop 5 + +C Check it out. + retval = check_file(ncid, var_name, var_type, dim_name) + if (retval .ne. 0) stop 6 + +C Close the file. + retval = nf_close(ncid) + if (retval .ne. nf_noerr) stop 7 print *,'*** SUCCESS!' end From 14fd1f7674920986a2b34a37832b87d9706bce09 Mon Sep 17 00:00:00 2001 From: Edward Hartnett Date: Sat, 4 Sep 2021 05:00:57 -0600 Subject: [PATCH 06/16] now testing quantize from f90 API --- nf03_test4/f90tst_vars5.f90 | 46 +++++++++++++++---------------------- 1 file changed, 19 insertions(+), 27 deletions(-) diff --git a/nf03_test4/f90tst_vars5.f90 b/nf03_test4/f90tst_vars5.f90 index b9a111b2..05114427 100644 --- a/nf03_test4/f90tst_vars5.f90 +++ b/nf03_test4/f90tst_vars5.f90 @@ -50,6 +50,8 @@ program f90tst_vars5 real*8 double_data_in(DIM_LEN_5) real real_data_expected(DIM_LEN_5) real*8 double_data_expected(DIM_LEN_5) + real diff + real, parameter :: EPSILON = .01 print *, '' print *,'*** Testing use of quantize feature on netCDF-4 vars from Fortran 90.' @@ -121,6 +123,7 @@ program f90tst_vars5 quantize_mode = quantize_mode_in, nsd = nsd_in)) if (name_in .ne. VAR1_NAME .or. xtype_in .ne. NF90_FLOAT .or. ndims_in .ne. NDIM1 .or. & natts_in .ne. 1 .or. dimids_in(1) .ne. dimids(1)) stop 3 + if (quantize_mode_in .ne. nf90_quantize_bitgroom .or. nsd_in .ne. 3) stop 3 ! Check variable 2. call check(nf90_inquire_variable(ncid, varid2_in, name_in, xtype_in, ndims_in, dimids_in, & @@ -130,33 +133,22 @@ program f90tst_vars5 if (name_in .ne. VAR2_NAME .or. xtype_in .ne. NF90_DOUBLE .or. ndims_in .ne. NDIM1 .or. & natts_in .ne. 1 .or. dimids_in(1) .ne. dimids(1)) stop 6 if (deflate_level_in .ne. 0 .or. .not. contiguous_in .or. fletcher32_in .or. shuffle_in) stop 7 - - ! ! Check the data. - ! call check(nf90_get_var(ncid, varid1_in, data_in)) - ! do x = 1, DIM_LEN_5 - ! do y = 1, NY - ! if (data_out(y, x) .ne. data_in(y, x)) stop 12 - ! end do - ! end do - ! call check(nf90_get_var(ncid, varid2_in, data_in)) - ! do x = 1, DIM_LEN_5 - ! do y = 1, NY - ! if (data_out(y, x) .ne. data_in(y, x)) stop 13 - ! end do - ! end do - ! call check(nf90_get_var(ncid, varid3_in, toe_san_in)) - ! if (toe_san_in .ne. TOE_SAN_VALUE) stop 14 - ! call check(nf90_get_var(ncid, varid4_in, data_in_1d)) - ! do x = 1, DIM_LEN_5 - ! if (data_out_1d(x) .ne. data_in_1d(x)) stop 15 - ! end do - ! call check(nf90_get_var(ncid, varid5_in, data_in)) - ! do x = 1, DIM_LEN_5 - ! do y = 1, NY - ! if (data_out(y, x) .ne. data_in(y, x)) stop 12 - ! end do - ! end do - + if (quantize_mode_in .ne. nf90_quantize_bitgroom .or. nsd_in .ne. 3) stop 3 + + ! Check the data. + call check(nf90_get_var(ncid, varid1_in, real_data_in)) + call check(nf90_get_var(ncid, varid2_in, double_data_in)) + + ! Check the data. + do x = 1, DIM_LEN_5 + diff = abs(real_data_in(x) - real_data_expected(x)) + if (diff .gt. EPSILON) stop 23 + diff = abs(double_data_in(x) - double_data_expected(x)) + ! print *, double_data_in(x), double_data_expected(x) + ! print *, 'x = ', x, ' diff = ', diff + if (diff .gt. EPSILON) stop 24 + end do + ! Close the file. call check(nf90_close(ncid)) From dacf69a5a239952aea794b1c5f5eff40252f074c Mon Sep 17 00:00:00 2001 From: Edward Hartnett Date: Sat, 4 Sep 2021 05:09:12 -0600 Subject: [PATCH 07/16] updated fortran docs with quantize info --- docs/netcdf-f90-sec6-variables.md | 32 ++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/docs/netcdf-f90-sec6-variables.md b/docs/netcdf-f90-sec6-variables.md index cc1f39d6..b8f224a1 100644 --- a/docs/netcdf-f90-sec6-variables.md +++ b/docs/netcdf-f90-sec6-variables.md @@ -127,7 +127,8 @@ in a netCDF-4/HDF5 file. integer, optional, intent(in) :: deflate_level logical, optional, intent(in) :: shuffle, fletcher32 integer, optional, intent(in) :: endianness - integer, optional, intent(in) :: cache_size, cache_nelems, cache_preemption + integer, optional, intent(in) :: cache_size, cache_nelems, cache_preemption + integer, optional, intent(in) :: quantize_mode, nsd integer :: nf90_def_var @@ -250,7 +251,19 @@ in a netCDF-4/HDF5 file. while a value of 100 means fully read chunks are always preempted before other chunks. +`quantize_mode` +: Set to nf90_noquantize (0) for no quantization (the default), or + nf90_quantize_bitgroom (1) for bitgroom quantization. Quantization + allows users to specify a number of significant digits (NSD) in + nf90_float and nf90_double variables. Bits beyond those needed to + maintain the NSD are set to zero or one (alternating). This causes + the data to compress better when compression is applied. + +`nsd` + +: If quantize_mode is set to nf90_quantize_bitgroom, then this is the + number of significant digits to be maintained in the data. ## Return Codes @@ -723,6 +736,7 @@ netCDF-4/HDF5 file. integer, optional, intent(out) :: deflate_level logical, optional, intent(out) :: shuffle, fletcher32 integer, optional, intent(out) :: endianness + integer, optional, intent(in) :: quantize_mode, nsd integer :: nf90_inquire_variable @@ -798,6 +812,22 @@ netCDF-4/HDF5 file. big-endian format, and NF90\_ENDIAN\_NATIVE if the endianness is not set, and the variable is not created yet. +`quantize_mode` + +: Will be set to nf90_noquantize (0) for no quantization, or + nf90_quantize_bitgroom (1) for bitgroom quantization. Quantization + allows users to specify a number of significant digits (NSD) in + nf90_float and nf90_double variables. Bits beyond those needed to + maintain the NSD are set to zero or one (alternating). This causes + the data to compress better when compression is applied. + +`nsd` + +: If quantize_mode is set to nf90_quantize_bitgroom, then this will be + set to the number of significant digits to be maintained in the + data. + + These functions return the value NF90\_NOERR if no errors occurred. Otherwise, the returned status indicates an error. Possible causes of errors include: From 01a4d3ee015e777a7b6baf7a5491eaed1d3ad450 Mon Sep 17 00:00:00 2001 From: Edward Hartnett Date: Sat, 4 Sep 2021 05:39:14 -0600 Subject: [PATCH 08/16] adding github build --- .github/workflows/build.yml | 80 +++++++++++++++++++++++++++++++++++++ .gitignore | 1 - 2 files changed, 80 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/build.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 00000000..a4a20a68 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,80 @@ +name: build + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + +jobs: + build: + + runs-on: ubuntu-latest + + env: + CPPFLAGS: "-I/home/runner/hdf5/include -I/home/runner/netcdf-c/include" + LDFLAGS: "-L/home/runner/hdf5/lib -L/home/runner/netcdf-c/lib" + + steps: + - uses: actions/checkout@v2 + - name: Installs + run: | + sudo apt-get install doxygen graphviz wget gfortran libjpeg-dev libz-dev + - name: cache-hdf5 + id: cache-hdf5 + uses: actions/cache@v2 + with: + path: ~/hdf5 + key: hdf5-${{ runner.os }}-1.10.7 + + - name: build-hdf5 + if: steps.cache-hdf5.outputs.cache-hit != 'true' + run: | + set -x + wget https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-1.10/hdf5-1.10.7/src/hdf5-1.10.7.tar.gz &> /dev/null + tar -xzf hdf5-1.10.7.tar.gz + pushd hdf5-1.10.7 + ./configure --prefix=/home/runner/hdf5 --disable-tools --disable-fortran --disable-cxx + make + sudo make install + popd + - name: cache-netcdf-c + id: cache-netcdf-c + uses: actions/cache@v2 + with: + path: ~/netcdf-c + key: netcdf-c-${{ runner.os }}-4.7.4 + + - name: build-netcdf-c + if: steps.cache-netcdf-c.outputs.cache-hit != 'true' + run: | + set -x + wget https://www.unidata.ucar.edu/downloads/netcdf/ftp/netcdf-c-4.7.4.tar.gz &> /dev/null + tar -xzf netcdf-c-4.7.4.tar.gz + pushd netcdf-c-4.7.4 + ./configure --prefix=/home/runner/netcdf-c --disable-dap --disable-utilities + make -j + sudo make install + popd + - name: cmake build + run: | + set -x + gcc --version + source .bashrc + export LD_LIBRARY_PATH="/home/runner/netcdf-c/lib:/home/runner/hdf5/lib:$LD_LIBRARY_PATH" + mkdir build + cd build + cmake -Wno-dev -DNetCDF_C_LIBRARY=/home/runner/netcdf-c/lib/libnetcdf.so -DNetCDF_C_INCLUDE_DIR=/home/runner/netcdf-c/include .. || (cat CMakeFiles/CMakeOutput.log && cat CMakeFiles/CMakeError.log) + make VERBOSE=1 + make tests VERBOSE=1 +# ctest -VV + - name: autotools build + run: | + set -x + gcc --version + source .bashrc + autoreconf -i + ./configure + make -j check + + diff --git a/.gitignore b/.gitignore index e138aa34..e29baa8f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,4 @@ CTestConfig.cmake -build* \#.\# *.*~ html From 57e28ff4d85c0455a31717b0cf0246ae31651df5 Mon Sep 17 00:00:00 2001 From: Edward Hartnett Date: Sat, 4 Sep 2021 07:10:53 -0600 Subject: [PATCH 09/16] working on workflow --- .github/workflows/build.yml | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a4a20a68..c76975a9 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -38,20 +38,11 @@ jobs: make sudo make install popd - - name: cache-netcdf-c - id: cache-netcdf-c - uses: actions/cache@v2 - with: - path: ~/netcdf-c - key: netcdf-c-${{ runner.os }}-4.7.4 - - name: build-netcdf-c - if: steps.cache-netcdf-c.outputs.cache-hit != 'true' run: | set -x - wget https://www.unidata.ucar.edu/downloads/netcdf/ftp/netcdf-c-4.7.4.tar.gz &> /dev/null - tar -xzf netcdf-c-4.7.4.tar.gz - pushd netcdf-c-4.7.4 + git clone https://github.com/Unidata/netcdf-c.git + pushd netcdf-c ./configure --prefix=/home/runner/netcdf-c --disable-dap --disable-utilities make -j sudo make install @@ -60,7 +51,6 @@ jobs: run: | set -x gcc --version - source .bashrc export LD_LIBRARY_PATH="/home/runner/netcdf-c/lib:/home/runner/hdf5/lib:$LD_LIBRARY_PATH" mkdir build cd build From eef2f2b5b738bbd183d6d25e3aa081e02fd5b365 Mon Sep 17 00:00:00 2001 From: Edward Hartnett Date: Sat, 4 Sep 2021 07:16:11 -0600 Subject: [PATCH 10/16] working on workflow --- .github/workflows/build.yml | 45 ++++++++++++++++++------------------- 1 file changed, 22 insertions(+), 23 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c76975a9..27383264 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -38,33 +38,32 @@ jobs: make sudo make install popd - - name: build-netcdf-c - run: | - set -x - git clone https://github.com/Unidata/netcdf-c.git - pushd netcdf-c - ./configure --prefix=/home/runner/netcdf-c --disable-dap --disable-utilities - make -j - sudo make install - popd - - name: cmake build - run: | - set -x - gcc --version - export LD_LIBRARY_PATH="/home/runner/netcdf-c/lib:/home/runner/hdf5/lib:$LD_LIBRARY_PATH" - mkdir build - cd build - cmake -Wno-dev -DNetCDF_C_LIBRARY=/home/runner/netcdf-c/lib/libnetcdf.so -DNetCDF_C_INCLUDE_DIR=/home/runner/netcdf-c/include .. || (cat CMakeFiles/CMakeOutput.log && cat CMakeFiles/CMakeError.log) - make VERBOSE=1 - make tests VERBOSE=1 + # - name: build-netcdf-c + # run: | + # set -x + # git clone https://github.com/Unidata/netcdf-c.git + # pushd netcdf-c + # ./configure --prefix=/home/runner/netcdf-c --disable-dap --disable-utilities + # make -j + # sudo make install + # popd + # - name: cmake build + # run: | + # set -x + # gcc --version + # export LD_LIBRARY_PATH="/home/runner/netcdf-c/lib:/home/runner/hdf5/lib:$LD_LIBRARY_PATH" + # mkdir build + # cd build + # cmake -Wno-dev -DNetCDF_C_LIBRARY=/home/runner/netcdf-c/lib/libnetcdf.so -DNetCDF_C_INCLUDE_DIR=/home/runner/netcdf-c/include .. || (cat CMakeFiles/CMakeOutput.log && cat CMakeFiles/CMakeError.log) + # make VERBOSE=1 + # make tests VERBOSE=1 # ctest -VV - name: autotools build run: | set -x gcc --version - source .bashrc - autoreconf -i - ./configure - make -j check + # autoreconf -i + # ./configure + # make -j check From eb7ee52d639dcc450448c32471c93af3f033053a Mon Sep 17 00:00:00 2001 From: Edward Hartnett Date: Sat, 4 Sep 2021 07:38:30 -0600 Subject: [PATCH 11/16] working on workflow --- .github/workflows/build.yml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 27383264..405c23fd 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -38,15 +38,15 @@ jobs: make sudo make install popd - # - name: build-netcdf-c - # run: | - # set -x - # git clone https://github.com/Unidata/netcdf-c.git - # pushd netcdf-c - # ./configure --prefix=/home/runner/netcdf-c --disable-dap --disable-utilities - # make -j - # sudo make install - # popd + - name: build-netcdf-c + run: | + set -x + git clone https://github.com/Unidata/netcdf-c.git + pushd netcdf-c + ./configure --prefix=/home/runner/netcdf-c --disable-dap --disable-utilities + make -j + sudo make install + popd # - name: cmake build # run: | # set -x @@ -62,8 +62,8 @@ jobs: run: | set -x gcc --version - # autoreconf -i - # ./configure - # make -j check + autoreconf -i + ./configure + make -j check From 77fc99e190b1fecbd9627b10f6aeb3cd5fe46530 Mon Sep 17 00:00:00 2001 From: Edward Hartnett Date: Sat, 4 Sep 2021 07:40:16 -0600 Subject: [PATCH 12/16] working on workflow --- .github/workflows/build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 405c23fd..117e45e2 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -43,6 +43,7 @@ jobs: set -x git clone https://github.com/Unidata/netcdf-c.git pushd netcdf-c + autoreconf -i ./configure --prefix=/home/runner/netcdf-c --disable-dap --disable-utilities make -j sudo make install From 774cc7b36aa13d64fb07c2a20382fc0abda420d2 Mon Sep 17 00:00:00 2001 From: Edward Hartnett Date: Sat, 4 Sep 2021 07:42:15 -0600 Subject: [PATCH 13/16] working on workflow --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 117e45e2..297ec60e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -41,7 +41,7 @@ jobs: - name: build-netcdf-c run: | set -x - git clone https://github.com/Unidata/netcdf-c.git + git clone -b ejh_quantize https://github.com/Unidata/netcdf-c.git pushd netcdf-c autoreconf -i ./configure --prefix=/home/runner/netcdf-c --disable-dap --disable-utilities From 1ae04d7c3b6a8481a8e57b9769a251ad2fa5dde7 Mon Sep 17 00:00:00 2001 From: Edward Hartnett Date: Sat, 4 Sep 2021 07:43:19 -0600 Subject: [PATCH 14/16] working on workflow --- .github/workflows/build.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 297ec60e..9cb496cb 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -41,7 +41,8 @@ jobs: - name: build-netcdf-c run: | set -x - git clone -b ejh_quantize https://github.com/Unidata/netcdf-c.git + git clone -b ejh_quantize https://github.com/edwardhartnett/netcdf-c.git +# git clone https://github.com/Unidata/netcdf-c.git pushd netcdf-c autoreconf -i ./configure --prefix=/home/runner/netcdf-c --disable-dap --disable-utilities From c46a1012eb2443ed5d6dcc271207e6149ff1bc52 Mon Sep 17 00:00:00 2001 From: Edward Hartnett Date: Sat, 4 Sep 2021 07:44:43 -0600 Subject: [PATCH 15/16] working on workflow --- .github/workflows/build.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9cb496cb..8a526b5b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -42,7 +42,6 @@ jobs: run: | set -x git clone -b ejh_quantize https://github.com/edwardhartnett/netcdf-c.git -# git clone https://github.com/Unidata/netcdf-c.git pushd netcdf-c autoreconf -i ./configure --prefix=/home/runner/netcdf-c --disable-dap --disable-utilities From 8c30b2077ab5afa67e97587341a34a33ad6daae7 Mon Sep 17 00:00:00 2001 From: Edward Hartnett Date: Sat, 4 Sep 2021 07:45:34 -0600 Subject: [PATCH 16/16] working on workflow --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8a526b5b..b4dd7447 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -41,7 +41,7 @@ jobs: - name: build-netcdf-c run: | set -x - git clone -b ejh_quantize https://github.com/edwardhartnett/netcdf-c.git + git clone -b ejh_quantize_2 https://github.com/edwardhartnett/netcdf-c.git pushd netcdf-c autoreconf -i ./configure --prefix=/home/runner/netcdf-c --disable-dap --disable-utilities