Merge pull request ESMCI#1738 from NCAR/ejh_more_testing

adding Fortran test for chunksize setting, also turning on more GitHub workflow tests
jedwards4b · Aug 28, 2020 · 3c02032 · 3c02032
2 parents f6feb76 + eb23c9f
commit 3c02032
Show file tree

Hide file tree

Showing 11 changed files with 215 additions and 12 deletions.
diff --git a/.github/workflows/a3.yml b/.github/workflows/a3.yml
@@ -126,7 +126,7 @@ jobs:
         export CC=/home/runner/openmpi/bin/mpicc
         autoreconf -i
         ./configure --with-mpiexec='mpiexec --oversubscribe'
-        which mpiexec
+        cat config.h
         make check
     - name: cmake build
       run: |
@@ -138,6 +138,7 @@ jobs:
         cd build
         export LD_LIBRARY_PATH="/home/runner/netcdf-c/lib:/home/runner/pnetcdf/lib:/home/runner/hdf5/lib:/home/runner/openmpi/lib:$LD_LIBRARY_PATH"
         cmake -Wno-dev -DWITH_MPIEXEC='/home/runner/openmpi/bin/mpiexec;--oversubscribe' -DNetCDF_C_LIBRARY=/home/runner/netcdf-c/lib/libnetcdf.so -DNetCDF_C_INCLUDE_DIR=/home/runner/netcdf-c/include -DPnetCDF_PATH='/home/runner/pnetcdf' -DPIO_ENABLE_FORTRAN=Off -DPIO_HDF5_LOGGING=On -DPIO_USE_MALLOC=On -DPIO_ENABLE_LOGGING=On -DPIO_ENABLE_TIMING=Off .. || (cat CMakeFiles/CMakeOutput.log && cat CMakeFiles/CMakeError.log)
+        cat config.h
         make VERBOSE=1
         make tests VERBOSE=1
         ctest -VV

diff --git a/configure.ac b/configure.ac
@@ -403,6 +403,7 @@ AC_CONFIG_FILES([tests/general/pio_tutil.F90:tests/general/util/pio_tutil.F90])
 AC_CONFIG_FILES([tests/cunit/run_tests.sh], [chmod ugo+x tests/cunit/run_tests.sh])
 AC_CONFIG_FILES([tests/ncint/run_tests.sh], [chmod ugo+x tests/ncint/run_tests.sh])
 AC_CONFIG_FILES([tests/ncint/run_perf.sh], [chmod ugo+x tests/ncint/run_perf.sh])
+AC_CONFIG_FILES([tests/unit/run_tests.sh], [chmod ugo+x tests/unit/run_tests.sh])
 AC_CONFIG_FILES([examples/c/run_tests.sh], [chmod ugo+x examples/c/run_tests.sh])
 
 # Args:

diff --git a/src/flib/pio_nf.F90 b/src/flib/pio_nf.F90
@@ -61,6 +61,7 @@ module pio_nf
        pio_inq_vardimid                                     , &
        pio_inq_varnatts                                     , &
        pio_inq_var_deflate                                  , &
+       pio_inq_var_chunking                                 , &
        pio_inquire_variable                                 , &
        pio_inquire_dimension                                , &
        pio_inq_dimname                                      , &
@@ -92,7 +93,7 @@ module pio_nf
   end interface pio_def_var_deflate
   interface pio_def_var_chunking
      module procedure &
-          def_var_chunking
+          def_var_chunking_desc
   end interface pio_def_var_chunking
   interface pio_inq_attname
      module procedure &
@@ -156,6 +157,12 @@ module pio_nf
           inq_var_deflate_vid                                  , &
           inq_var_deflate_id
   end interface pio_inq_var_deflate
+  interface pio_inq_var_chunking
+     module procedure &
+          inq_var_chunking_desc                                 , &
+          inq_var_chunking_vid                                  , &
+          inq_var_chunking_id
+  end interface pio_inq_var_chunking
   interface pio_inquire_dimension
      module procedure &
           inquire_dimension_desc                            , &
@@ -1242,6 +1249,77 @@ end function PIOc_inq_var_deflate
     ierr = PIOc_inq_var_deflate(ncid, varid-1, shuffle, deflate, deflate_level)
   end function inq_var_deflate_id
 
+  !>
+  !! @public
+  !! @ingroup PIO_inquire_variable
+  !! Gets metadata information for netcdf file.
+  !!
+  !! @param File @copydoc file_desc_t
+  !! @param vardesc @copydoc var_desc_t
+  !! @param storage 0 for chunked, 1 for contiguous
+  !! @param chunksizes Array of chunk sizes.
+  !! @retval ierr @copydoc error_return
+  !! @author Ed Hartnett
+  !<
+  integer function inq_var_chunking_desc(File, vardesc, storage, chunksizes) result(ierr)
+
+    type (File_desc_t), intent(in) :: File
+    type (Var_desc_t), intent(in) :: vardesc
+    integer, intent(out) :: storage
+    integer (kind=PIO_OFFSET_KIND), intent(out) :: chunksizes(*)
+
+    ierr = pio_inq_var_chunking(File%fh, vardesc%varid, storage, chunksizes)
+  end function inq_var_chunking_desc
+
+  !>
+  !! @public
+  !! @ingroup PIO_inquire_variable
+  !! Gets metadata information for netcdf file.
+  !! @author Ed Hartnett
+  !<
+  integer function inq_var_chunking_vid(File, varid, storage, chunksizes) result(ierr)
+
+    type (File_desc_t), intent(in) :: File
+    integer, intent(in) :: varid
+    integer, intent(out) :: storage
+    integer (kind=PIO_OFFSET_KIND), intent(out) :: chunksizes(*)
+
+    ierr = pio_inq_var_chunking(File%fh, varid, storage, chunksizes)
+  end function inq_var_chunking_vid
+
+  !>
+  !! @public
+  !! @ingroup PIO_inquire_variable
+  !! Gets metadata information for netcdf file.
+  !! @author Ed Hartnett
+  !<
+  integer function inq_var_chunking_id(ncid, varid, storage, chunksizes) result(ierr)
+    integer, intent(in) :: ncid
+    integer, intent(in) :: varid
+    integer, intent(out) :: storage
+    integer (kind=PIO_OFFSET_KIND), intent(out) :: chunksizes(*)
+    integer(kind=PIO_OFFSET_KIND) :: cchunksizes(PIO_MAX_VAR_DIMS)
+    integer :: ndims, i
+
+    interface
+       integer(C_INT) function PIOc_inq_var_chunking(ncid, varid, storage, cchunksizes) &
+            bind(C, name="PIOc_inq_var_chunking")
+         use iso_c_binding
+         integer(C_INT), value :: ncid
+         integer(C_INT), value :: varid
+         integer(C_INT) :: storage
+         integer(C_SIZE_T) :: cchunksizes(*)
+       end function PIOc_inq_var_chunking
+    end interface
+
+    ierr = PIOc_inq_var_chunking(ncid, varid-1, storage, cchunksizes)
+    ierr = pio_inq_varndims(ncid, varid, ndims)
+    do i = 1, ndims
+       chunksizes(i) = cchunksizes(ndims - i + 1)
+    enddo
+
+  end function inq_var_chunking_id
+
   !>
   !! @public
   !! @ingroup PIO_inquire_variable
@@ -1705,12 +1783,12 @@ end function def_var_deflate_desc
   !! Changes chunking settings for a netCDF-4/HDF5 variable.
   !! @author Ed Hartnett
   !<
-  integer function def_var_chunking(file, vardesc, storage, chunksizes) result(ierr)
+  integer function def_var_chunking_desc(file, vardesc, storage, chunksizes) result(ierr)
     type (File_desc_t), intent(in)  :: file
     type (var_desc_t), intent(in) :: vardesc
     integer, intent(in) :: storage
     integer, intent(in) :: chunksizes(:)
-    integer(C_INT) :: cchunksizes(PIO_MAX_VAR_DIMS)
+    integer(kind=PIO_OFFSET_KIND) :: cchunksizes(PIO_MAX_VAR_DIMS)
     integer :: ndims, i
 
     interface
@@ -1720,16 +1798,16 @@ integer (C_INT) function PIOc_def_var_chunking(ncid, varid, storage, chunksizes)
          integer(c_int), value :: ncid
          integer(c_int), value :: varid
          integer(c_int), value :: storage
-         integer(c_int) :: chunksizes(*)
+         integer(c_size_t) :: chunksizes(*)
        end function PIOc_def_var_chunking
     end interface
     ndims = size(chunksizes)
     do i=1,ndims
-       cchunksizes(i) = chunksizes(ndims-i+1)-1
+       cchunksizes(i) = chunksizes(ndims-i+1)
     enddo
 
     ierr = PIOc_def_var_chunking(file%fh, vardesc%varid-1, storage, cchunksizes)
-  end function def_var_chunking
+  end function def_var_chunking_desc
 
   !>
   !! @ingroup PIO_set_chunk_cache

diff --git a/tests/unit/CMakeLists.txt b/tests/unit/CMakeLists.txt
@@ -1,3 +1,6 @@
+# This is the CMake build file for the tests/unit tests of the PIO library.
+
+# Jim Edwards
 include (LibMPI)
 
 include_directories("${CMAKE_SOURCE_DIR}/tests/unit")
@@ -35,6 +38,11 @@ endif ()
 
 add_dependencies (tests pio_unit_test)
 
+# Add ftst_vars.F90.
+add_executable (ftst_vars_chunking EXCLUDE_FROM_ALL ftst_vars_chunking.F90)
+target_link_libraries (ftst_vars_chunking piof)
+add_dependencies (tests ftst_vars_chunking)
+
 # Test Timeout in seconds.
 set (DEFAULT_TEST_TIMEOUT 60)
 
@@ -55,6 +63,10 @@ else ()
     EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_unit_test
     NUMPROCS 4
     TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_mpi_test(ftst_vars_chunking
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/ftst_vars_chunking
+    NUMPROCS 4
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
 endif ()
 
 if (PIO_HDF5_LOGGING)

diff --git a/tests/unit/Makefile.am b/tests/unit/Makefile.am
@@ -9,11 +9,17 @@
 # Find the pio.mod file.
 AM_CPPFLAGS = -I$(top_srcdir)/src/flib
 
+# Link to the PIO C and Fortran libraries.
+LDADD = ${top_builddir}/src/flib/libpiof.la	\
+${top_builddir}/src/clib/libpioc.la
+
 # Build the test for make check.
-check_PROGRAMS = pio_unit_test_driver
+check_PROGRAMS = pio_unit_test_driver ftst_vars_chunking
 pio_unit_test_driver_SOURCES = driver.F90
 pio_unit_test_driver_LDADD = libglobal_vars.la libncdf_tests.la	\
-libbasic_tests.la ${top_builddir}/src/flib/libpiof.la ${top_builddir}/src/clib/libpioc.la
+libbasic_tests.la ${top_builddir}/src/flib/libpiof.la		\
+${top_builddir}/src/clib/libpioc.la
+ftst_vars_chunking_SOURCES = ftst_vars_chunking.F90
 
 # Build these uninstalled convenience libraries.
 noinst_LTLIBRARIES = libglobal_vars.la libncdf_tests.la	\
@@ -30,7 +36,10 @@ TESTS = run_tests.sh
 endif # RUN_TESTS
 
 # Distribute the test script.
-EXTRA_DIST = CMakeLists.txt run_tests.sh input.nl not_netcdf.ieee
+EXTRA_DIST = CMakeLists.txt run_tests.sh input.nl not_netcdf.ieee	\
+run_tests.sh.in
 
 # Clean up files produced during testing.
 CLEANFILES = *.nc *.log *.mod
+
+DISTCLEANFILES = run_tests.sh
diff --git a/tests/unit/basic_tests.F90 b/tests/unit/basic_tests.F90
@@ -3,6 +3,7 @@
 !! @brief Module containing basic unit tests that are run for both
 !!        binary and netcdf file types.
 !<
+#include "config.h"
 
 module basic_tests
 

diff --git a/tests/unit/driver.F90 b/tests/unit/driver.F90
@@ -2,6 +2,7 @@
 !! @file
 !! @brief The driver for PIO unit tests
 !<
+#include "config.h"
 
 Program pio_unit_test_driver
   use pio

diff --git a/tests/unit/ftst_vars_chunking.F90 b/tests/unit/ftst_vars_chunking.F90
@@ -0,0 +1,98 @@
+  ! This is a test of the PIO Fortran library.
+
+  ! This tests var functions.
+
+  ! Ed Hartnett, 8/28/20
+#include "config.h"
+
+program ftst_vars_chunking
+  use mpi
+  use pio
+  use pio_nf
+
+  integer, parameter :: NUM_IOTYPES = 2
+  integer, parameter :: NDIM2 = 2
+
+  type(iosystem_desc_t) :: pio_iosystem
+  type(file_desc_t)     :: pio_file
+  type(var_desc_t)      :: pio_var  
+  integer :: my_rank, ntasks
+  integer :: niotasks = 1, stride = 1
+  character(len=64) :: filename = 'ftst_vars_chunking.nc'
+  character(len=64) :: dim_name_1 = 'influence_on_Roman_history'
+  character(len=64) :: dim_name_2 = 'age_at_death'
+  character(len=64) :: var_name = 'Caesar'
+  integer :: dimid1, dimid2, dim_len1 = 40, dim_len2 = 80
+  integer :: chunksize1 = 10, chunksize2 = 20
+  integer :: storage_in
+  integer (kind=PIO_OFFSET_KIND) :: chunksizes_in(NDIM2)
+  integer :: iotype(NUM_IOTYPES) = (/ PIO_iotype_netcdf4c, PIO_iotype_netcdf4p /)
+  integer :: iotype_idx, ierr
+
+  ! Set up MPI
+  call MPI_Init(ierr)
+  call MPI_Comm_rank(MPI_COMM_WORLD, my_rank, ierr)
+  call MPI_Comm_size(MPI_COMM_WORLD, ntasks , ierr)
+
+  ! This whole test only works for netCDF/HDF5 files, because it is
+  ! about chunking.
+#ifdef _NETCDF4
+  if (my_rank .eq. 0) print *,'Testing variables...'
+
+  ! Initialize PIO.
+  call PIO_init(my_rank, MPI_COMM_WORLD, niotasks, 0, stride, &
+       PIO_rearr_subset, pio_iosystem, base=1)
+
+  ! Set error handling for test.
+  call PIO_seterrorhandling(pio_iosystem, PIO_RETURN_ERROR)  
+  call PIO_seterrorhandling(PIO_DEFAULT, PIO_RETURN_ERROR)
+
+  ! Uncomment (and build with --enable-logging) to turn on logging.
+  !ret_val = PIO_set_log_level(3)
+
+  ! Try this test for NETCDF4C and NETCDF4P.
+  do iotype_idx = 1, NUM_IOTYPES
+
+     ! Create a file.
+     ierr = PIO_createfile(pio_iosystem, pio_file, iotype(iotype_idx), filename)
+     if (ierr .ne. PIO_NOERR) stop 3
+
+     ! Define dims.
+     ret_val = PIO_def_dim(pio_file, dim_name_1, dim_len1, dimid1)
+     if (ierr .ne. PIO_NOERR) stop 5
+     ret_val = PIO_def_dim(pio_file, dim_name_2, dim_len2, dimid2)
+     if (ierr .ne. PIO_NOERR) stop 6
+
+     ! Define a var.
+     ret_val = PIO_def_var(pio_file, var_name, PIO_int, (/dimid1, dimid2/), pio_var)
+     if (ierr .ne. PIO_NOERR) stop 7
+
+     ! Define chunking for var.
+     ret_val = PIO_def_var_chunking(pio_file, pio_var, 0, (/chunksize1, chunksize2/))
+     if (ierr .ne. PIO_NOERR) stop 9
+
+     ! Close the file.
+     call PIO_closefile(pio_file)
+
+     ! Open the file.
+     ret_val = PIO_openfile(pio_iosystem, pio_file, iotype(iotype_idx), filename, PIO_nowrite)  
+     if (ierr .ne. PIO_NOERR) stop 23
+
+     ! Find var chunksizes using varid.
+     ret_val = PIO_inq_var_chunking(pio_file, 1, storage_in, chunksizes_in)
+     if (ierr .ne. PIO_NOERR) stop 25
+     if (chunksizes_in(1) .ne. chunksize1) stop 26
+     if (chunksizes_in(2) .ne. chunksize2) stop 26
+
+     ! Close the file.
+     call PIO_closefile(pio_file)
+
+  end do ! next IOTYPE
+
+  ! Finalize PIO.
+  call PIO_finalize(pio_iosystem, ierr)
+
+  if (my_rank .eq. 0) print *,'SUCCESS!'
+#endif 
+  call MPI_Finalize(ierr)        
+end program ftst_vars_chunking
diff --git a/tests/unit/global_vars.F90 b/tests/unit/global_vars.F90
@@ -2,6 +2,7 @@
 !! @file
 !! @brief Module containing variables used across all unit test files
 !<
+#include "config.h"
 
 module global_vars
 

diff --git a/tests/unit/ncdf_tests.F90 b/tests/unit/ncdf_tests.F90
@@ -2,6 +2,7 @@
 !! @file
 !! @brief Module containing netcdf-specific PIO unit tests
 !<
+#include "config.h"
 
 module ncdf_tests
 

diff --git a/tests/unit/run_tests.sh → tests/unit/run_tests.sh.in b/tests/unit/run_tests.sh → tests/unit/run_tests.sh.in
@@ -10,14 +10,14 @@ trap exit INT TERM
 
 printf 'running PIO tests...\n'
 
-PIO_TESTS='pio_unit_test_driver'
+PIO_TESTS='pio_unit_test_driver ftst_vars_chunking'
 
 success1=true
 for TEST in $PIO_TESTS
 do
     success1=false
     echo "running ${TEST}"
-    mpiexec -n 4 ./${TEST} && success1=true
+    @WITH_MPIEXEC@ -n 4 ./${TEST} && success1=true
     if test $success1 = false; then
         break
     fi