diff --git a/.github/workflows/a3.yml b/.github/workflows/a3.yml index 6f326b86a3a..53121bb8acb 100644 --- a/.github/workflows/a3.yml +++ b/.github/workflows/a3.yml @@ -126,7 +126,7 @@ jobs: export CC=/home/runner/openmpi/bin/mpicc autoreconf -i ./configure --with-mpiexec='mpiexec --oversubscribe' - which mpiexec + cat config.h make check - name: cmake build run: | @@ -138,6 +138,7 @@ jobs: cd build export LD_LIBRARY_PATH="/home/runner/netcdf-c/lib:/home/runner/pnetcdf/lib:/home/runner/hdf5/lib:/home/runner/openmpi/lib:$LD_LIBRARY_PATH" cmake -Wno-dev -DWITH_MPIEXEC='/home/runner/openmpi/bin/mpiexec;--oversubscribe' -DNetCDF_C_LIBRARY=/home/runner/netcdf-c/lib/libnetcdf.so -DNetCDF_C_INCLUDE_DIR=/home/runner/netcdf-c/include -DPnetCDF_PATH='/home/runner/pnetcdf' -DPIO_ENABLE_FORTRAN=Off -DPIO_HDF5_LOGGING=On -DPIO_USE_MALLOC=On -DPIO_ENABLE_LOGGING=On -DPIO_ENABLE_TIMING=Off .. || (cat CMakeFiles/CMakeOutput.log && cat CMakeFiles/CMakeError.log) + cat config.h make VERBOSE=1 make tests VERBOSE=1 ctest -VV diff --git a/configure.ac b/configure.ac index 429a14e1e9d..ec0262495f9 100644 --- a/configure.ac +++ b/configure.ac @@ -403,6 +403,7 @@ AC_CONFIG_FILES([tests/general/pio_tutil.F90:tests/general/util/pio_tutil.F90]) AC_CONFIG_FILES([tests/cunit/run_tests.sh], [chmod ugo+x tests/cunit/run_tests.sh]) AC_CONFIG_FILES([tests/ncint/run_tests.sh], [chmod ugo+x tests/ncint/run_tests.sh]) AC_CONFIG_FILES([tests/ncint/run_perf.sh], [chmod ugo+x tests/ncint/run_perf.sh]) +AC_CONFIG_FILES([tests/unit/run_tests.sh], [chmod ugo+x tests/unit/run_tests.sh]) AC_CONFIG_FILES([examples/c/run_tests.sh], [chmod ugo+x examples/c/run_tests.sh]) # Args: diff --git a/src/flib/pio_nf.F90 b/src/flib/pio_nf.F90 index a8352b294b2..459a6355d01 100644 --- a/src/flib/pio_nf.F90 +++ b/src/flib/pio_nf.F90 @@ -61,6 +61,7 @@ module pio_nf pio_inq_vardimid , & pio_inq_varnatts , & pio_inq_var_deflate , & + pio_inq_var_chunking , & pio_inquire_variable , & pio_inquire_dimension , & pio_inq_dimname , & @@ -92,7 +93,7 @@ module pio_nf end interface pio_def_var_deflate interface pio_def_var_chunking module procedure & - def_var_chunking + def_var_chunking_desc end interface pio_def_var_chunking interface pio_inq_attname module procedure & @@ -156,6 +157,12 @@ module pio_nf inq_var_deflate_vid , & inq_var_deflate_id end interface pio_inq_var_deflate + interface pio_inq_var_chunking + module procedure & + inq_var_chunking_desc , & + inq_var_chunking_vid , & + inq_var_chunking_id + end interface pio_inq_var_chunking interface pio_inquire_dimension module procedure & inquire_dimension_desc , & @@ -1242,6 +1249,77 @@ end function PIOc_inq_var_deflate ierr = PIOc_inq_var_deflate(ncid, varid-1, shuffle, deflate, deflate_level) end function inq_var_deflate_id + !> + !! @public + !! @ingroup PIO_inquire_variable + !! Gets metadata information for netcdf file. + !! + !! @param File @copydoc file_desc_t + !! @param vardesc @copydoc var_desc_t + !! @param storage 0 for chunked, 1 for contiguous + !! @param chunksizes Array of chunk sizes. + !! @retval ierr @copydoc error_return + !! @author Ed Hartnett + !< + integer function inq_var_chunking_desc(File, vardesc, storage, chunksizes) result(ierr) + + type (File_desc_t), intent(in) :: File + type (Var_desc_t), intent(in) :: vardesc + integer, intent(out) :: storage + integer (kind=PIO_OFFSET_KIND), intent(out) :: chunksizes(*) + + ierr = pio_inq_var_chunking(File%fh, vardesc%varid, storage, chunksizes) + end function inq_var_chunking_desc + + !> + !! @public + !! @ingroup PIO_inquire_variable + !! Gets metadata information for netcdf file. + !! @author Ed Hartnett + !< + integer function inq_var_chunking_vid(File, varid, storage, chunksizes) result(ierr) + + type (File_desc_t), intent(in) :: File + integer, intent(in) :: varid + integer, intent(out) :: storage + integer (kind=PIO_OFFSET_KIND), intent(out) :: chunksizes(*) + + ierr = pio_inq_var_chunking(File%fh, varid, storage, chunksizes) + end function inq_var_chunking_vid + + !> + !! @public + !! @ingroup PIO_inquire_variable + !! Gets metadata information for netcdf file. + !! @author Ed Hartnett + !< + integer function inq_var_chunking_id(ncid, varid, storage, chunksizes) result(ierr) + integer, intent(in) :: ncid + integer, intent(in) :: varid + integer, intent(out) :: storage + integer (kind=PIO_OFFSET_KIND), intent(out) :: chunksizes(*) + integer(kind=PIO_OFFSET_KIND) :: cchunksizes(PIO_MAX_VAR_DIMS) + integer :: ndims, i + + interface + integer(C_INT) function PIOc_inq_var_chunking(ncid, varid, storage, cchunksizes) & + bind(C, name="PIOc_inq_var_chunking") + use iso_c_binding + integer(C_INT), value :: ncid + integer(C_INT), value :: varid + integer(C_INT) :: storage + integer(C_SIZE_T) :: cchunksizes(*) + end function PIOc_inq_var_chunking + end interface + + ierr = PIOc_inq_var_chunking(ncid, varid-1, storage, cchunksizes) + ierr = pio_inq_varndims(ncid, varid, ndims) + do i = 1, ndims + chunksizes(i) = cchunksizes(ndims - i + 1) + enddo + + end function inq_var_chunking_id + !> !! @public !! @ingroup PIO_inquire_variable @@ -1705,12 +1783,12 @@ end function def_var_deflate_desc !! Changes chunking settings for a netCDF-4/HDF5 variable. !! @author Ed Hartnett !< - integer function def_var_chunking(file, vardesc, storage, chunksizes) result(ierr) + integer function def_var_chunking_desc(file, vardesc, storage, chunksizes) result(ierr) type (File_desc_t), intent(in) :: file type (var_desc_t), intent(in) :: vardesc integer, intent(in) :: storage integer, intent(in) :: chunksizes(:) - integer(C_INT) :: cchunksizes(PIO_MAX_VAR_DIMS) + integer(kind=PIO_OFFSET_KIND) :: cchunksizes(PIO_MAX_VAR_DIMS) integer :: ndims, i interface @@ -1720,16 +1798,16 @@ integer (C_INT) function PIOc_def_var_chunking(ncid, varid, storage, chunksizes) integer(c_int), value :: ncid integer(c_int), value :: varid integer(c_int), value :: storage - integer(c_int) :: chunksizes(*) + integer(c_size_t) :: chunksizes(*) end function PIOc_def_var_chunking end interface ndims = size(chunksizes) do i=1,ndims - cchunksizes(i) = chunksizes(ndims-i+1)-1 + cchunksizes(i) = chunksizes(ndims-i+1) enddo ierr = PIOc_def_var_chunking(file%fh, vardesc%varid-1, storage, cchunksizes) - end function def_var_chunking + end function def_var_chunking_desc !> !! @ingroup PIO_set_chunk_cache diff --git a/tests/unit/CMakeLists.txt b/tests/unit/CMakeLists.txt index eaf7c0f5694..27298336b54 100644 --- a/tests/unit/CMakeLists.txt +++ b/tests/unit/CMakeLists.txt @@ -1,3 +1,6 @@ +# This is the CMake build file for the tests/unit tests of the PIO library. + +# Jim Edwards include (LibMPI) include_directories("${CMAKE_SOURCE_DIR}/tests/unit") @@ -35,6 +38,11 @@ endif () add_dependencies (tests pio_unit_test) +# Add ftst_vars.F90. +add_executable (ftst_vars_chunking EXCLUDE_FROM_ALL ftst_vars_chunking.F90) +target_link_libraries (ftst_vars_chunking piof) +add_dependencies (tests ftst_vars_chunking) + # Test Timeout in seconds. set (DEFAULT_TEST_TIMEOUT 60) @@ -55,6 +63,10 @@ else () EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_unit_test NUMPROCS 4 TIMEOUT ${DEFAULT_TEST_TIMEOUT}) + add_mpi_test(ftst_vars_chunking + EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/ftst_vars_chunking + NUMPROCS 4 + TIMEOUT ${DEFAULT_TEST_TIMEOUT}) endif () if (PIO_HDF5_LOGGING) diff --git a/tests/unit/Makefile.am b/tests/unit/Makefile.am index 1eaea6e77b3..867857ef97e 100644 --- a/tests/unit/Makefile.am +++ b/tests/unit/Makefile.am @@ -9,11 +9,17 @@ # Find the pio.mod file. AM_CPPFLAGS = -I$(top_srcdir)/src/flib +# Link to the PIO C and Fortran libraries. +LDADD = ${top_builddir}/src/flib/libpiof.la \ +${top_builddir}/src/clib/libpioc.la + # Build the test for make check. -check_PROGRAMS = pio_unit_test_driver +check_PROGRAMS = pio_unit_test_driver ftst_vars_chunking pio_unit_test_driver_SOURCES = driver.F90 pio_unit_test_driver_LDADD = libglobal_vars.la libncdf_tests.la \ -libbasic_tests.la ${top_builddir}/src/flib/libpiof.la ${top_builddir}/src/clib/libpioc.la +libbasic_tests.la ${top_builddir}/src/flib/libpiof.la \ +${top_builddir}/src/clib/libpioc.la +ftst_vars_chunking_SOURCES = ftst_vars_chunking.F90 # Build these uninstalled convenience libraries. noinst_LTLIBRARIES = libglobal_vars.la libncdf_tests.la \ @@ -30,7 +36,10 @@ TESTS = run_tests.sh endif # RUN_TESTS # Distribute the test script. -EXTRA_DIST = CMakeLists.txt run_tests.sh input.nl not_netcdf.ieee +EXTRA_DIST = CMakeLists.txt run_tests.sh input.nl not_netcdf.ieee \ +run_tests.sh.in # Clean up files produced during testing. CLEANFILES = *.nc *.log *.mod + +DISTCLEANFILES = run_tests.sh diff --git a/tests/unit/basic_tests.F90 b/tests/unit/basic_tests.F90 index e48c99a9047..b9f72692be4 100644 --- a/tests/unit/basic_tests.F90 +++ b/tests/unit/basic_tests.F90 @@ -3,6 +3,7 @@ !! @brief Module containing basic unit tests that are run for both !! binary and netcdf file types. !< +#include "config.h" module basic_tests diff --git a/tests/unit/driver.F90 b/tests/unit/driver.F90 index 6764b875595..6c3975d9b47 100644 --- a/tests/unit/driver.F90 +++ b/tests/unit/driver.F90 @@ -2,6 +2,7 @@ !! @file !! @brief The driver for PIO unit tests !< +#include "config.h" Program pio_unit_test_driver use pio diff --git a/tests/unit/ftst_vars_chunking.F90 b/tests/unit/ftst_vars_chunking.F90 new file mode 100644 index 00000000000..af04112d5fa --- /dev/null +++ b/tests/unit/ftst_vars_chunking.F90 @@ -0,0 +1,98 @@ + ! This is a test of the PIO Fortran library. + + ! This tests var functions. + + ! Ed Hartnett, 8/28/20 +#include "config.h" + +program ftst_vars_chunking + use mpi + use pio + use pio_nf + + integer, parameter :: NUM_IOTYPES = 2 + integer, parameter :: NDIM2 = 2 + + type(iosystem_desc_t) :: pio_iosystem + type(file_desc_t) :: pio_file + type(var_desc_t) :: pio_var + integer :: my_rank, ntasks + integer :: niotasks = 1, stride = 1 + character(len=64) :: filename = 'ftst_vars_chunking.nc' + character(len=64) :: dim_name_1 = 'influence_on_Roman_history' + character(len=64) :: dim_name_2 = 'age_at_death' + character(len=64) :: var_name = 'Caesar' + integer :: dimid1, dimid2, dim_len1 = 40, dim_len2 = 80 + integer :: chunksize1 = 10, chunksize2 = 20 + integer :: storage_in + integer (kind=PIO_OFFSET_KIND) :: chunksizes_in(NDIM2) + integer :: iotype(NUM_IOTYPES) = (/ PIO_iotype_netcdf4c, PIO_iotype_netcdf4p /) + integer :: iotype_idx, ierr + + ! Set up MPI + call MPI_Init(ierr) + call MPI_Comm_rank(MPI_COMM_WORLD, my_rank, ierr) + call MPI_Comm_size(MPI_COMM_WORLD, ntasks , ierr) + + ! This whole test only works for netCDF/HDF5 files, because it is + ! about chunking. +#ifdef _NETCDF4 + if (my_rank .eq. 0) print *,'Testing variables...' + + ! Initialize PIO. + call PIO_init(my_rank, MPI_COMM_WORLD, niotasks, 0, stride, & + PIO_rearr_subset, pio_iosystem, base=1) + + ! Set error handling for test. + call PIO_seterrorhandling(pio_iosystem, PIO_RETURN_ERROR) + call PIO_seterrorhandling(PIO_DEFAULT, PIO_RETURN_ERROR) + + ! Uncomment (and build with --enable-logging) to turn on logging. + !ret_val = PIO_set_log_level(3) + + ! Try this test for NETCDF4C and NETCDF4P. + do iotype_idx = 1, NUM_IOTYPES + + ! Create a file. + ierr = PIO_createfile(pio_iosystem, pio_file, iotype(iotype_idx), filename) + if (ierr .ne. PIO_NOERR) stop 3 + + ! Define dims. + ret_val = PIO_def_dim(pio_file, dim_name_1, dim_len1, dimid1) + if (ierr .ne. PIO_NOERR) stop 5 + ret_val = PIO_def_dim(pio_file, dim_name_2, dim_len2, dimid2) + if (ierr .ne. PIO_NOERR) stop 6 + + ! Define a var. + ret_val = PIO_def_var(pio_file, var_name, PIO_int, (/dimid1, dimid2/), pio_var) + if (ierr .ne. PIO_NOERR) stop 7 + + ! Define chunking for var. + ret_val = PIO_def_var_chunking(pio_file, pio_var, 0, (/chunksize1, chunksize2/)) + if (ierr .ne. PIO_NOERR) stop 9 + + ! Close the file. + call PIO_closefile(pio_file) + + ! Open the file. + ret_val = PIO_openfile(pio_iosystem, pio_file, iotype(iotype_idx), filename, PIO_nowrite) + if (ierr .ne. PIO_NOERR) stop 23 + + ! Find var chunksizes using varid. + ret_val = PIO_inq_var_chunking(pio_file, 1, storage_in, chunksizes_in) + if (ierr .ne. PIO_NOERR) stop 25 + if (chunksizes_in(1) .ne. chunksize1) stop 26 + if (chunksizes_in(2) .ne. chunksize2) stop 26 + + ! Close the file. + call PIO_closefile(pio_file) + + end do ! next IOTYPE + + ! Finalize PIO. + call PIO_finalize(pio_iosystem, ierr) + + if (my_rank .eq. 0) print *,'SUCCESS!' +#endif + call MPI_Finalize(ierr) +end program ftst_vars_chunking diff --git a/tests/unit/global_vars.F90 b/tests/unit/global_vars.F90 index f3347ba38c9..69cf0f412ac 100644 --- a/tests/unit/global_vars.F90 +++ b/tests/unit/global_vars.F90 @@ -2,6 +2,7 @@ !! @file !! @brief Module containing variables used across all unit test files !< +#include "config.h" module global_vars diff --git a/tests/unit/ncdf_tests.F90 b/tests/unit/ncdf_tests.F90 index 831a06d28fb..765132751e3 100644 --- a/tests/unit/ncdf_tests.F90 +++ b/tests/unit/ncdf_tests.F90 @@ -2,6 +2,7 @@ !! @file !! @brief Module containing netcdf-specific PIO unit tests !< +#include "config.h" module ncdf_tests diff --git a/tests/unit/run_tests.sh b/tests/unit/run_tests.sh.in similarity index 81% rename from tests/unit/run_tests.sh rename to tests/unit/run_tests.sh.in index 0f7c61ee178..77511ff974a 100755 --- a/tests/unit/run_tests.sh +++ b/tests/unit/run_tests.sh.in @@ -10,14 +10,14 @@ trap exit INT TERM printf 'running PIO tests...\n' -PIO_TESTS='pio_unit_test_driver' +PIO_TESTS='pio_unit_test_driver ftst_vars_chunking' success1=true for TEST in $PIO_TESTS do success1=false echo "running ${TEST}" - mpiexec -n 4 ./${TEST} && success1=true + @WITH_MPIEXEC@ -n 4 ./${TEST} && success1=true if test $success1 = false; then break fi