From ed9f6c5386e507c08a4f7890920e3c46a21a0a37 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Sun, 7 Jul 2019 07:08:15 -0600 Subject: [PATCH 01/17] adding directories and Makefile.am changes to support fortran netcdf integration --- configure.ac | 2 ++ src/Makefile.am | 7 +++++-- src/fncint/Makefile.am | 12 ++++++++++++ tests/Makefile.am | 5 ++++- tests/fncint/Makefile.am | 27 +++++++++++++++++++++++++++ 5 files changed, 50 insertions(+), 3 deletions(-) create mode 100644 src/fncint/Makefile.am create mode 100644 tests/fncint/Makefile.am diff --git a/configure.ac b/configure.ac index 8ad88a66135..e9efb137784 100644 --- a/configure.ac +++ b/configure.ac @@ -296,11 +296,13 @@ AC_OUTPUT(Makefile src/Makefile src/clib/Makefile src/ncint/Makefile + src/fncint/Makefile src/flib/Makefile src/gptl/Makefile tests/Makefile tests/cunit/Makefile tests/ncint/Makefile + tests/fncint/Makefile tests/unit/Makefile tests/general/Makefile tests/general/util/Makefile diff --git a/src/Makefile.am b/src/Makefile.am index 4eaa356d1c8..4a194379393 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -5,7 +5,7 @@ # Does the user want to build fortran? if BUILD_FORTRAN FLIB = flib -endif +endif # BUILD_FORTRAN # Are we building with the GPTL timing library? if USE_GPTL @@ -15,9 +15,12 @@ endif # Are we building with netCDF integration? if BUILD_NCINT NCINT = ncint +if BUILD_FORTRAN +FNCINT = fncint +endif # BUILD_FORTRAN endif # BUILD_NCINT # Build these subdirectories. -SUBDIRS = ${NCINT} clib ${GPTL} $(FLIB) +SUBDIRS = ${NCINT} clib ${GPTL} $(FNCINT) $(FLIB) EXTRA_DIST = CMakeLists.txt diff --git a/src/fncint/Makefile.am b/src/fncint/Makefile.am new file mode 100644 index 00000000000..d635e6e86b3 --- /dev/null +++ b/src/fncint/Makefile.am @@ -0,0 +1,12 @@ +## This is the automake file to build the PIO netCDF integration +## layer for the PIO fortran library. +# Ed Hartnett 7/7/19 + +# Find pio.h. +AM_CPPFLAGS = -I$(top_srcdir)/src/clib + +# This is our output. The ncint convenience library. +#noinst_LTLIBRARIES = libfncint.la + +# The source files. +#libfncint_la_SOURCES = ncintdispatch.c ncintdispatch.h ncint_pio.c diff --git a/tests/Makefile.am b/tests/Makefile.am index 8c58db5bd99..502597242d0 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -17,9 +17,12 @@ endif # BUILD_FORTRAN # Are we building with netCDF integration? if BUILD_NCINT NCINT = ncint +if BUILD_FORTRAN +FNCINT = fncint +endif # BUILD_FORTRAN endif # BUILD_NCINT # Build in these subdirs. -SUBDIRS = cunit ${UNIT} ${NCINT} ${GENERAL} ${PERFORMANCE} +SUBDIRS = cunit ${UNIT} ${NCINT} ${GENERAL} ${PERFORMANCE} ${FNCINT} EXTRA_DIST = CMakeLists.txt diff --git a/tests/fncint/Makefile.am b/tests/fncint/Makefile.am new file mode 100644 index 00000000000..e04bf067cbc --- /dev/null +++ b/tests/fncint/Makefile.am @@ -0,0 +1,27 @@ +## This is the automake file for building the netCDF integration layer +## tests. + +# Ed Hartnett 7/3/19 + +# Put together AM_CPPFLAGS and AM_LDFLAGS. +AM_CPPFLAGS = -I$(top_srcdir)/src/clib +LDADD = ${top_builddir}/src/clib/libpioc.la + +# Build the test for make check. +#check_PROGRAMS = tst_pio_udf + +#if RUN_TESTS +# Tests will run from a bash script. +#TESTS = run_tests.sh +#endif # RUN_TESTS + +# if RUN_TESTS +# # Tests will run from a bash script. +# TESTS = run_tests.sh +# endif # RUN_TESTS + +# Distribute the test script. +#EXTRA_DIST = run_tests.sh + +# Clean up files produced during testing. +#CLEANFILES = *.nc *.log From 116f7de06aa4ce200dee4e7dbe16b8812b07efe0 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Mon, 8 Jul 2019 05:49:39 -0600 Subject: [PATCH 02/17] starting to test ncint in fortran --- src/flib/piolib_mod.F90 | 18 ++++++++++++++++++ tests/fncint/Makefile.am | 5 +++-- tests/fncint/ftst_pio.f90 | 9 +++++++++ tests/ncint/tst_pio_udf.c | 2 +- 4 files changed, 31 insertions(+), 3 deletions(-) create mode 100644 tests/fncint/ftst_pio.f90 diff --git a/src/flib/piolib_mod.F90 b/src/flib/piolib_mod.F90 index 11319bb603c..8fa5e0123b9 100644 --- a/src/flib/piolib_mod.F90 +++ b/src/flib/piolib_mod.F90 @@ -190,6 +190,24 @@ module piolib_mod module procedure PIO_initdecomp_bc end interface PIO_initdecomp + !> + !! PIO_initdecomp is an overload interface the models decomposition to pio. + !! initdecomp_1dof_bin_i8, initdecomp_1dof_nf_i4, initdecomp_2dof_bin_i4, + !! and initdecomp_2dof_nf_i4 are all depreciated, but supported for backwards + !! compatibility. + !< + ! interface nf_init_decomp + ! module procedure PIO_initdecomp_dof_i4 ! previous name: initdecomop_1dof_nf_box + ! module procedure PIO_initdecomp_dof_i8 ! previous name: initdecomop_1dof_nf_box + ! module procedure initdecomp_1dof_nf_i4 + ! module procedure initdecomp_1dof_nf_i8 + ! module procedure initdecomp_1dof_bin_i4 + ! module procedure initdecomp_1dof_bin_i8 + ! module procedure initdecomp_2dof_nf_i4 + ! module procedure initdecomp_2dof_nf_i8 + ! module procedure PIO_initdecomp_bc + ! end interface nf_init_decomp + !> !! Return the actual number of IO-tasks used. PIO will reset the !! total number of IO-tasks if certain conditions are meet. diff --git a/tests/fncint/Makefile.am b/tests/fncint/Makefile.am index e04bf067cbc..6e73a038531 100644 --- a/tests/fncint/Makefile.am +++ b/tests/fncint/Makefile.am @@ -4,11 +4,12 @@ # Ed Hartnett 7/3/19 # Put together AM_CPPFLAGS and AM_LDFLAGS. -AM_CPPFLAGS = -I$(top_srcdir)/src/clib +AM_FCFLAGS = -I$(top_srcdir)/src/flib LDADD = ${top_builddir}/src/clib/libpioc.la # Build the test for make check. -#check_PROGRAMS = tst_pio_udf +check_PROGRAMS = ftst_pio +ftst_pio_SOURCES = ftst_pio.f90 #if RUN_TESTS # Tests will run from a bash script. diff --git a/tests/fncint/ftst_pio.f90 b/tests/fncint/ftst_pio.f90 new file mode 100644 index 00000000000..f523370bd19 --- /dev/null +++ b/tests/fncint/ftst_pio.f90 @@ -0,0 +1,9 @@ + !> This is a test program for the Fortran API use of the netCDF + !! integration layer. + +program ftst_pio + use pio + implicit none + + print *, '*** SUCCESS running ftst_pio!' +end program ftst_pio diff --git a/tests/ncint/tst_pio_udf.c b/tests/ncint/tst_pio_udf.c index 4a6c7fb1be6..dda4cdf55af 100644 --- a/tests/ncint/tst_pio_udf.c +++ b/tests/ncint/tst_pio_udf.c @@ -51,7 +51,7 @@ main(int argc, char **argv) int i; /* Turn on logging for PIO library. */ - PIOc_set_log_level(3); + /* PIOc_set_log_level(3); */ /* Initialize the intracomm. */ if (nc_init_intracomm(MPI_COMM_WORLD, 1, 1, 0, 0, &iosysid)) ERR; From e03d43f8699c1d25bda9a6397dd97a39a77013ae Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Mon, 8 Jul 2019 06:07:16 -0600 Subject: [PATCH 03/17] adding run_tests.sh script in fncint test directory --- tests/fncint/Makefile.am | 17 ++++++----------- tests/fncint/ftst_pio.f90 | 13 ++++++++++++- tests/fncint/run_tests.sh | 30 ++++++++++++++++++++++++++++++ 3 files changed, 48 insertions(+), 12 deletions(-) create mode 100755 tests/fncint/run_tests.sh diff --git a/tests/fncint/Makefile.am b/tests/fncint/Makefile.am index 6e73a038531..6b6d7568547 100644 --- a/tests/fncint/Makefile.am +++ b/tests/fncint/Makefile.am @@ -5,24 +5,19 @@ # Put together AM_CPPFLAGS and AM_LDFLAGS. AM_FCFLAGS = -I$(top_srcdir)/src/flib -LDADD = ${top_builddir}/src/clib/libpioc.la +AM_LDADD = ${top_builddir}/src/clib/libpioc.la # Build the test for make check. check_PROGRAMS = ftst_pio ftst_pio_SOURCES = ftst_pio.f90 -#if RUN_TESTS +if RUN_TESTS # Tests will run from a bash script. -#TESTS = run_tests.sh -#endif # RUN_TESTS - -# if RUN_TESTS -# # Tests will run from a bash script. -# TESTS = run_tests.sh -# endif # RUN_TESTS +TESTS = run_tests.sh +endif # RUN_TESTS # Distribute the test script. -#EXTRA_DIST = run_tests.sh +EXTRA_DIST = run_tests.sh # Clean up files produced during testing. -#CLEANFILES = *.nc *.log +CLEANFILES = *.nc *.log diff --git a/tests/fncint/ftst_pio.f90 b/tests/fncint/ftst_pio.f90 index f523370bd19..04c589bc219 100644 --- a/tests/fncint/ftst_pio.f90 +++ b/tests/fncint/ftst_pio.f90 @@ -4,6 +4,17 @@ program ftst_pio use pio implicit none + include 'mpif.h' - print *, '*** SUCCESS running ftst_pio!' + integer :: myRank, ntasks + integer :: ierr + + call MPI_Init(ierr) + call MPI_Comm_rank(MPI_COMM_WORLD, myRank, ierr) + call MPI_Comm_size(MPI_COMM_WORLD, ntasks, ierr) + + call MPI_Finalize(ierr) + if (myRank .eq. 0) then + print *, '*** SUCCESS running ftst_pio!' + endif end program ftst_pio diff --git a/tests/fncint/run_tests.sh b/tests/fncint/run_tests.sh new file mode 100755 index 00000000000..fbe296e54a9 --- /dev/null +++ b/tests/fncint/run_tests.sh @@ -0,0 +1,30 @@ +#!/bin/sh +# This is a test script for PIO. +# Ed Hartnett + +# Stop execution of script if error is returned. +set -e + +# Stop loop if ctrl-c is pressed. +trap exit INT TERM + +printf 'running Fortran tests for PIO netCDF integration...\n' + +PIO_TESTS='ftst_pio' + +success1=true +for TEST in $PIO_TESTS +do + success1=false + echo "running ${TEST}" + mpiexec -n 4 ./${TEST} && success1=true + if test $success1 = false; then + break + fi +done + +# Did we succeed? +if test x$success1 = xtrue; then + exit 0 +fi +exit 1 From 259622303d463acc9ed8bd3aa1a0bc453e832f04 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Mon, 8 Jul 2019 06:29:36 -0600 Subject: [PATCH 04/17] developing fortran test of ncint --- tests/fncint/Makefile.am | 8 +++++--- tests/fncint/ftst_pio.f90 | 6 ++++++ 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/tests/fncint/Makefile.am b/tests/fncint/Makefile.am index 6b6d7568547..2493c4b7007 100644 --- a/tests/fncint/Makefile.am +++ b/tests/fncint/Makefile.am @@ -3,9 +3,11 @@ # Ed Hartnett 7/3/19 -# Put together AM_CPPFLAGS and AM_LDFLAGS. -AM_FCFLAGS = -I$(top_srcdir)/src/flib -AM_LDADD = ${top_builddir}/src/clib/libpioc.la +# Link to the PIO Fortran and C libraries. +LDADD = ${top_builddir}/src/flib/libpiof.la ${top_builddir}/src/clib/libpioc.la + +# Find the pio.mod file. +AM_FCFLAGS = -I${top_builddir}/src/flib # Build the test for make check. check_PROGRAMS = ftst_pio diff --git a/tests/fncint/ftst_pio.f90 b/tests/fncint/ftst_pio.f90 index 04c589bc219..efdb30b9405 100644 --- a/tests/fncint/ftst_pio.f90 +++ b/tests/fncint/ftst_pio.f90 @@ -7,12 +7,18 @@ program ftst_pio include 'mpif.h' integer :: myRank, ntasks + type(iosystem_desc_t) :: ioSystem + integer :: niotasks = 1, numAggregator = 0, stride = 1, base = 0 integer :: ierr call MPI_Init(ierr) call MPI_Comm_rank(MPI_COMM_WORLD, myRank, ierr) call MPI_Comm_size(MPI_COMM_WORLD, ntasks, ierr) + call PIO_init(myRank, MPI_COMM_WORLD, niotasks, numAggregator, & + stride, PIO_rearr_subset, ioSystem, base) + + call PIO_finalize(ioSystem, ierr) call MPI_Finalize(ierr) if (myRank .eq. 0) then print *, '*** SUCCESS running ftst_pio!' From b0649bac26c3fbe6aa0c76cd761715b0235eaf26 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Mon, 8 Jul 2019 07:41:41 -0600 Subject: [PATCH 05/17] further test development --- tests/fncint/ftst_pio.f90 | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/fncint/ftst_pio.f90 b/tests/fncint/ftst_pio.f90 index efdb30b9405..fce640a994a 100644 --- a/tests/fncint/ftst_pio.f90 +++ b/tests/fncint/ftst_pio.f90 @@ -15,6 +15,7 @@ program ftst_pio call MPI_Comm_rank(MPI_COMM_WORLD, myRank, ierr) call MPI_Comm_size(MPI_COMM_WORLD, ntasks, ierr) + ierr = pio_set_log_level(1) call PIO_init(myRank, MPI_COMM_WORLD, niotasks, numAggregator, & stride, PIO_rearr_subset, ioSystem, base) From c08f6918ba267063b918f7fae0338cca012fc98a Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Mon, 8 Jul 2019 10:03:03 -0600 Subject: [PATCH 06/17] now linking fncint test to netcdf fortran library --- tests/fncint/Makefile.am | 5 ++++- tests/fncint/ftst_pio.f90 | 5 +++++ tests/ncint/Makefile.am | 2 +- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/tests/fncint/Makefile.am b/tests/fncint/Makefile.am index 2493c4b7007..3cc3d3df955 100644 --- a/tests/fncint/Makefile.am +++ b/tests/fncint/Makefile.am @@ -6,8 +6,11 @@ # Link to the PIO Fortran and C libraries. LDADD = ${top_builddir}/src/flib/libpiof.la ${top_builddir}/src/clib/libpioc.la +# Link to the netCDF fortran library. +LDADD += -lnetcdff + # Find the pio.mod file. -AM_FCFLAGS = -I${top_builddir}/src/flib +AM_FCFLAGS = -I${top_builddir}/src/flib ${CPPFLAGS} # Build the test for make check. check_PROGRAMS = ftst_pio diff --git a/tests/fncint/ftst_pio.f90 b/tests/fncint/ftst_pio.f90 index fce640a994a..0283ed1673b 100644 --- a/tests/fncint/ftst_pio.f90 +++ b/tests/fncint/ftst_pio.f90 @@ -5,10 +5,12 @@ program ftst_pio use pio implicit none include 'mpif.h' + include 'netcdf.inc' integer :: myRank, ntasks type(iosystem_desc_t) :: ioSystem integer :: niotasks = 1, numAggregator = 0, stride = 1, base = 0 + integer :: ncid integer :: ierr call MPI_Init(ierr) @@ -19,6 +21,9 @@ program ftst_pio call PIO_init(myRank, MPI_COMM_WORLD, niotasks, numAggregator, & stride, PIO_rearr_subset, ioSystem, base) + ierr = nf_create('ftst_pio.nc', nf_clobber, ncid) + ierr = nf_close(ncid) + call PIO_finalize(ioSystem, ierr) call MPI_Finalize(ierr) if (myRank .eq. 0) then diff --git a/tests/ncint/Makefile.am b/tests/ncint/Makefile.am index e5d41d2f2b1..03ebdc02ef6 100644 --- a/tests/ncint/Makefile.am +++ b/tests/ncint/Makefile.am @@ -24,4 +24,4 @@ endif # RUN_TESTS EXTRA_DIST = run_tests.sh # Clean up files produced during testing. -#CLEANFILES = *.nc *.log +CLEANFILES = *.nc *.log From 3b08eb8fc18f4494f31ff315d9f507138941ba37 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Wed, 10 Jul 2019 13:32:05 -0600 Subject: [PATCH 07/17] finally got fortran working with netcdf integration! --- configure.ac | 1 + src/clib/Makefile.am | 1 + src/clib/pio.h | 1 + src/clib/pioc.c | 15 ++++++++++ src/flib/pio.F90 | 2 +- src/flib/piolib_mod.F90 | 58 ++++++++++++++++++++++++++++++++++++++- src/ncint/ncint_pio.c | 18 +++++++++--- src/ncint/ncintdispatch.c | 13 +++++++-- tests/fncint/ftst_pio.f90 | 9 ++++-- 9 files changed, 106 insertions(+), 12 deletions(-) diff --git a/configure.ac b/configure.ac index e9efb137784..7bdced12fd7 100644 --- a/configure.ac +++ b/configure.ac @@ -280,6 +280,7 @@ fi if test "x$enable_netcdf_integration" = xyes; then AC_DEFINE([HDF5_PARALLEL],[1],[Does HDF5 library provide parallel access]) AC_DEFINE([USE_NETCDF4],[1],[Does HDF5 library provide parallel access]) + AC_DEFINE([NETCDF_INTEGRATION],[1],[Are we building with netCDF integration]) fi AM_CONDITIONAL(BUILD_NCINT, [test "x$enable_netcdf_integration" = xyes]) diff --git a/src/clib/Makefile.am b/src/clib/Makefile.am index a938e0d867a..66c7a948825 100644 --- a/src/clib/Makefile.am +++ b/src/clib/Makefile.am @@ -7,6 +7,7 @@ lib_LTLIBRARIES = libpioc.la # Are we building with netCDF integration? if BUILD_NCINT libpioc_la_LIBADD = ../ncint/libncint.la +libpioc_la_CPPFLAGS = -I${top_srcdir}/src/ncint endif # BUILD_NCINT # These linker flags specify libtool version info. diff --git a/src/clib/pio.h b/src/clib/pio.h index f1203456ca0..56b36bd732d 100644 --- a/src/clib/pio.h +++ b/src/clib/pio.h @@ -1250,6 +1250,7 @@ extern "C" { int nc_init_intracomm(MPI_Comm comp_comm, int num_iotasks, int stride, int base, int rearr, int *iosysidp); + int nc_set_iosystem(int iosysid); int nc_free_iosystem(int iosysid); int nc_init_decomp(int iosysid, int pio_type, int ndims, const int *gdimlen, diff --git a/src/clib/pioc.c b/src/clib/pioc.c index 1f0c1ab2ff8..1c5dd32bd63 100644 --- a/src/clib/pioc.c +++ b/src/clib/pioc.c @@ -9,12 +9,20 @@ #include #include #include +#ifdef NETCDF_INTEGRATION +#include "ncintdispatch.h" +#endif /* NETCDF_INTEGRATION */ #ifdef USE_MPE /* The event numbers for MPE logging. */ extern int event_num[2][NUM_EVENTS]; #endif /* USE_MPE */ +#ifdef NETCDF_INTEGRATION +/* Have we initialized? */ +extern int ncint_initialized; +#endif /* NETCDF_INTEGRATION */ + /** * @defgroup PIO_init_c Initialize the IO System * Initialize the IOSystem, including specifying number of IO and @@ -934,6 +942,13 @@ PIOc_Init_Intracomm(MPI_Comm comp_comm, int num_iotasks, int stride, int base, if ((ret = pio_init_logging())) return pio_err(NULL, NULL, ret, __FILE__, __LINE__); +#ifdef NETCDF_INTEGRATION + PLOG((1, "Initializing netcdf integration")); + /* Initialize netCDF integration layer if we need to. */ + if (!ncint_initialized) + PIO_NCINT_initialize(); +#endif /* NETCDF_INTEGRATION */ + #ifdef USE_MPE pio_start_mpe_log(INIT); #endif /* USE_MPE */ diff --git a/src/flib/pio.F90 b/src/flib/pio.F90 index b4939e29060..89c2870063d 100644 --- a/src/flib/pio.F90 +++ b/src/flib/pio.F90 @@ -21,7 +21,7 @@ module pio pio_freedecomp, pio_syncfile, & pio_finalize, pio_set_hint, pio_getnumiotasks, pio_file_is_open, & PIO_deletefile, PIO_get_numiotasks, PIO_iotype_available, & - pio_set_rearr_opts + pio_set_rearr_opts, nf_init_intracom use pio_types, only : io_desc_t, file_desc_t, var_desc_t, iosystem_desc_t, & pio_rearr_opt_t, pio_rearr_comm_fc_opt_t, pio_rearr_comm_fc_2d_enable,& diff --git a/src/flib/piolib_mod.F90 b/src/flib/piolib_mod.F90 index 8fa5e0123b9..2b77af19cd5 100644 --- a/src/flib/piolib_mod.F90 +++ b/src/flib/piolib_mod.F90 @@ -97,7 +97,8 @@ module piolib_mod PIO_deletefile, & PIO_get_numiotasks, & PIO_iotype_available, & - PIO_set_rearr_opts + PIO_set_rearr_opts, & + nf_init_intracom !----------------------------------------------------------------------- ! @@ -133,6 +134,10 @@ module piolib_mod module procedure setframe end interface PIO_setframe + interface nf_init_intracom + module procedure nf_init_intracom + end interface nf_init_intracom + !> !! Increment the record number for a future read/write of distributed !! arrays (see @ref PIO_write_darray, @ref PIO_read_darray). @@ -989,6 +994,57 @@ end function PIOc_Init_Intracomm_from_F90 #endif end subroutine init_intracom + !> + !! @public + !! @ingroup PIO_init + !! Initialize the pio subsystem. This is a collective call. Input + !! parameters are read on comp_rank=0 values on other tasks are + !! ignored. This variation of PIO_init locates the IO tasks on a + !! subset of the compute tasks. + !! + !! @param comp_rank mpi rank of each participating task, + !! @param comp_comm the mpi communicator which defines the + !! collective. + !! @param num_iotasks the number of iotasks to define. + !! @param num_aggregator the mpi aggregator count + !! @param stride the stride in the mpi rank between io tasks. + !! @param rearr @copydoc PIO_rearr_method + !! @param iosystem a derived type which can be used in subsequent + !! pio operations (defined in PIO_types). + !! @param base @em optional argument can be used to offset the first + !! io task - default base is task 1. + !! @param rearr_opts the rearranger options. + !! @author Jim Edwards + !< + subroutine nf_init_intracom(comp_rank, comp_comm, num_iotasks, num_aggregator, stride, rearr, iosystem,base, rearr_opts) + use pio_types, only : pio_internal_error, pio_rearr_opt_t + use iso_c_binding + + integer(i4), intent(in) :: comp_rank + integer(i4), intent(in) :: comp_comm + integer(i4), intent(in) :: num_iotasks + integer(i4), intent(in) :: num_aggregator + integer(i4), intent(in) :: stride + integer(i4), intent(in) :: rearr + type (iosystem_desc_t), intent(out) :: iosystem ! io descriptor to initalize + integer(i4), intent(in),optional :: base + type (pio_rearr_opt_t), intent(in), optional :: rearr_opts + integer :: ierr + + interface + integer(C_INT) function nc_set_iosystem(iosystemid) & + bind(C,name="nc_set_iosystem") + use iso_c_binding + integer(C_INT), intent(in), value :: iosystemid + end function nc_set_iosystem + end interface + + call init_intracom(comp_rank, comp_comm, num_iotasks, num_aggregator, stride, rearr, iosystem,base, rearr_opts) + + ierr = nc_set_iosystem(iosystem%iosysid) + + end subroutine nf_init_intracom + !! @public !! @ingroup PIO_init !! Initialize the pio subsystem. This is a collective call. Input diff --git a/src/ncint/ncint_pio.c b/src/ncint/ncint_pio.c index f750e0f79e8..0c9b3a16c4a 100644 --- a/src/ncint/ncint_pio.c +++ b/src/ncint/ncint_pio.c @@ -27,10 +27,6 @@ nc_init_intracomm(MPI_Comm comp_comm, int num_iotasks, int stride, int base, int { int ret; - /* Initialize if we need to. */ - if (!ncint_initialized) - PIO_NCINT_initialize(); - /* Call the PIOc_ function to initialize the intracomm. */ if ((ret = PIOc_Init_Intracomm(comp_comm, num_iotasks, stride, base, rearr, iosysidp))) @@ -42,6 +38,20 @@ nc_init_intracomm(MPI_Comm comp_comm, int num_iotasks, int stride, int base, int return PIO_NOERR; } +/** + * Set the default iosystemID. + * + * @author Ed Hartnett + */ +int +nc_set_iosystem(int iosysid) +{ + /* Remember the io system id. */ + diosysid = iosysid; + + return PIO_NOERR; +} + /** * Same as PIOc_free_iosystem(). * diff --git a/src/ncint/ncintdispatch.c b/src/ncint/ncintdispatch.c index b4d7d4a4026..e5075b61e2e 100644 --- a/src/ncint/ncintdispatch.c +++ b/src/ncint/ncintdispatch.c @@ -128,6 +128,8 @@ PIO_NCINT_initialize(void) NCINT_dispatch_table = &NCINT_dispatcher; + PLOG((1, "Adding user-defined format for netCDF PIO integration")); + /* Add our user defined format. */ if ((ret = nc_def_user_format(NC_UDF0, &NCINT_dispatcher, NULL))) return ret; @@ -148,8 +150,13 @@ PIO_NCINT_finalize(void) return NC_NOERR; } +/** + * Create a file using PIO via netCDF's nc_create(). + * + * @author Ed Hartnett + */ int -PIO_NCINT_create(const char* path, int cmode, size_t initialsz, int basepe, +PIO_NCINT_create(const char *path, int cmode, size_t initialsz, int basepe, size_t *chunksizehintp, void *parameters, const NC_Dispatch *dispatch, NC *nc_file) { @@ -157,7 +164,7 @@ PIO_NCINT_create(const char* path, int cmode, size_t initialsz, int basepe, iosystem_desc_t *ios; /* Pointer to io system information. */ int ret; - LOG((1, "PIO_NCINT_create path = %s mode = %x", path, mode)); + PLOG((1, "PIO_NCINT_create path = %s mode = %x", path, cmode)); /* Get the IO system info from the id. */ if (!(ios = pio_get_iosystem_from_id(diosysid))) @@ -191,7 +198,7 @@ PIO_NCINT_open(const char *path, int mode, int basepe, size_t *chunksizehintp, iosystem_desc_t *ios; /* Pointer to io system information. */ int ret; - LOG((1, "PIO_NCINT_open path = %s mode = %x", path, mode)); + PLOG((1, "PIO_NCINT_open path = %s mode = %x", path, mode)); /* Get the IO system info from the id. */ if (!(ios = pio_get_iosystem_from_id(diosysid))) diff --git a/tests/fncint/ftst_pio.f90 b/tests/fncint/ftst_pio.f90 index 0283ed1673b..1172cd65534 100644 --- a/tests/fncint/ftst_pio.f90 +++ b/tests/fncint/ftst_pio.f90 @@ -11,17 +11,20 @@ program ftst_pio type(iosystem_desc_t) :: ioSystem integer :: niotasks = 1, numAggregator = 0, stride = 1, base = 0 integer :: ncid + character*(*) FILE_NAME + parameter (FILE_NAME='ftst_pio.nc') integer :: ierr call MPI_Init(ierr) call MPI_Comm_rank(MPI_COMM_WORLD, myRank, ierr) call MPI_Comm_size(MPI_COMM_WORLD, ntasks, ierr) - ierr = pio_set_log_level(1) - call PIO_init(myRank, MPI_COMM_WORLD, niotasks, numAggregator, & + ierr = pio_set_log_level(2) + ierr = nf_set_log_level(2) + call nf_init_intracom(myRank, MPI_COMM_WORLD, niotasks, numAggregator, & stride, PIO_rearr_subset, ioSystem, base) - ierr = nf_create('ftst_pio.nc', nf_clobber, ncid) + ierr = nf_create(FILE_NAME, 64, ncid) ierr = nf_close(ncid) call PIO_finalize(ioSystem, ierr) From 392fb392afa9fefd38fe35b6d09e5a79c0dbb73f Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Wed, 10 Jul 2019 13:35:02 -0600 Subject: [PATCH 08/17] cleanup --- configure.ac | 1 - src/Makefile.am | 5 +---- src/fncint/Makefile.am | 12 ------------ tests/fncint/ftst_pio.f90 | 4 ++-- 4 files changed, 3 insertions(+), 19 deletions(-) delete mode 100644 src/fncint/Makefile.am diff --git a/configure.ac b/configure.ac index 7bdced12fd7..612fc8a5f42 100644 --- a/configure.ac +++ b/configure.ac @@ -297,7 +297,6 @@ AC_OUTPUT(Makefile src/Makefile src/clib/Makefile src/ncint/Makefile - src/fncint/Makefile src/flib/Makefile src/gptl/Makefile tests/Makefile diff --git a/src/Makefile.am b/src/Makefile.am index 4a194379393..eab6ab0d2f1 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -15,12 +15,9 @@ endif # Are we building with netCDF integration? if BUILD_NCINT NCINT = ncint -if BUILD_FORTRAN -FNCINT = fncint -endif # BUILD_FORTRAN endif # BUILD_NCINT # Build these subdirectories. -SUBDIRS = ${NCINT} clib ${GPTL} $(FNCINT) $(FLIB) +SUBDIRS = ${NCINT} clib ${GPTL} $(FLIB) EXTRA_DIST = CMakeLists.txt diff --git a/src/fncint/Makefile.am b/src/fncint/Makefile.am deleted file mode 100644 index d635e6e86b3..00000000000 --- a/src/fncint/Makefile.am +++ /dev/null @@ -1,12 +0,0 @@ -## This is the automake file to build the PIO netCDF integration -## layer for the PIO fortran library. -# Ed Hartnett 7/7/19 - -# Find pio.h. -AM_CPPFLAGS = -I$(top_srcdir)/src/clib - -# This is our output. The ncint convenience library. -#noinst_LTLIBRARIES = libfncint.la - -# The source files. -#libfncint_la_SOURCES = ncintdispatch.c ncintdispatch.h ncint_pio.c diff --git a/tests/fncint/ftst_pio.f90 b/tests/fncint/ftst_pio.f90 index 1172cd65534..b55fc80454e 100644 --- a/tests/fncint/ftst_pio.f90 +++ b/tests/fncint/ftst_pio.f90 @@ -19,8 +19,8 @@ program ftst_pio call MPI_Comm_rank(MPI_COMM_WORLD, myRank, ierr) call MPI_Comm_size(MPI_COMM_WORLD, ntasks, ierr) - ierr = pio_set_log_level(2) - ierr = nf_set_log_level(2) + ! ierr = pio_set_log_level(2) + ! ierr = nf_set_log_level(2) call nf_init_intracom(myRank, MPI_COMM_WORLD, niotasks, numAggregator, & stride, PIO_rearr_subset, ioSystem, base) From e12d55a580c007e07d48a32edfd9379767bd0c12 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Wed, 10 Jul 2019 13:36:36 -0600 Subject: [PATCH 09/17] cleanup --- src/flib/piolib_mod.F90 | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/src/flib/piolib_mod.F90 b/src/flib/piolib_mod.F90 index 2b77af19cd5..2b1486f9344 100644 --- a/src/flib/piolib_mod.F90 +++ b/src/flib/piolib_mod.F90 @@ -195,24 +195,6 @@ module piolib_mod module procedure PIO_initdecomp_bc end interface PIO_initdecomp - !> - !! PIO_initdecomp is an overload interface the models decomposition to pio. - !! initdecomp_1dof_bin_i8, initdecomp_1dof_nf_i4, initdecomp_2dof_bin_i4, - !! and initdecomp_2dof_nf_i4 are all depreciated, but supported for backwards - !! compatibility. - !< - ! interface nf_init_decomp - ! module procedure PIO_initdecomp_dof_i4 ! previous name: initdecomop_1dof_nf_box - ! module procedure PIO_initdecomp_dof_i8 ! previous name: initdecomop_1dof_nf_box - ! module procedure initdecomp_1dof_nf_i4 - ! module procedure initdecomp_1dof_nf_i8 - ! module procedure initdecomp_1dof_bin_i4 - ! module procedure initdecomp_1dof_bin_i8 - ! module procedure initdecomp_2dof_nf_i4 - ! module procedure initdecomp_2dof_nf_i8 - ! module procedure PIO_initdecomp_bc - ! end interface nf_init_decomp - !> !! Return the actual number of IO-tasks used. PIO will reset the !! total number of IO-tasks if certain conditions are meet. From 946c2fcccd3e8d880a23b525ac70be7da7b1e335 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Thu, 11 Jul 2019 09:11:43 -0600 Subject: [PATCH 10/17] getting netcdf integration working in fortran --- src/flib/pio.F90 | 6 +++++- src/flib/piolib_mod.F90 | 9 +++++++-- tests/fncint/ftst_pio.f90 | 4 ++-- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/src/flib/pio.F90 b/src/flib/pio.F90 index 89c2870063d..280c9a836d5 100644 --- a/src/flib/pio.F90 +++ b/src/flib/pio.F90 @@ -7,6 +7,7 @@ !> !! @defgroup PIO_set_blocksize Box Rearranger Settings !! Set the box rearranger blocksize in Fortran. +#include "config.h" module pio ! Package all exposed variables and functions under one roof @@ -21,7 +22,10 @@ module pio pio_freedecomp, pio_syncfile, & pio_finalize, pio_set_hint, pio_getnumiotasks, pio_file_is_open, & PIO_deletefile, PIO_get_numiotasks, PIO_iotype_available, & - pio_set_rearr_opts, nf_init_intracom +!#ifdef NETCDF_INTEGRATION + nf_init_intracom, & +!#endif + pio_set_rearr_opts use pio_types, only : io_desc_t, file_desc_t, var_desc_t, iosystem_desc_t, & pio_rearr_opt_t, pio_rearr_comm_fc_opt_t, pio_rearr_comm_fc_2d_enable,& diff --git a/src/flib/piolib_mod.F90 b/src/flib/piolib_mod.F90 index 2b1486f9344..d50ad7f03cc 100644 --- a/src/flib/piolib_mod.F90 +++ b/src/flib/piolib_mod.F90 @@ -1,4 +1,5 @@ #define __PIO_FILE__ "piolib_mod.f90" +#include "config.h" !> !! @file !! Initialization Routines for PIO. @@ -97,8 +98,8 @@ module piolib_mod PIO_deletefile, & PIO_get_numiotasks, & PIO_iotype_available, & - PIO_set_rearr_opts, & - nf_init_intracom + nf_init_intracom, & + PIO_set_rearr_opts !----------------------------------------------------------------------- ! @@ -134,9 +135,11 @@ module piolib_mod module procedure setframe end interface PIO_setframe +!#ifdef NETCDF_INTEGRATION interface nf_init_intracom module procedure nf_init_intracom end interface nf_init_intracom +!#endif !> !! Increment the record number for a future read/write of distributed @@ -976,6 +979,7 @@ end function PIOc_Init_Intracomm_from_F90 #endif end subroutine init_intracom +!#ifdef NETCDF_INTEGRATION !> !! @public !! @ingroup PIO_init @@ -1026,6 +1030,7 @@ end function nc_set_iosystem ierr = nc_set_iosystem(iosystem%iosysid) end subroutine nf_init_intracom +!#endif !! @public !! @ingroup PIO_init diff --git a/tests/fncint/ftst_pio.f90 b/tests/fncint/ftst_pio.f90 index b55fc80454e..1172cd65534 100644 --- a/tests/fncint/ftst_pio.f90 +++ b/tests/fncint/ftst_pio.f90 @@ -19,8 +19,8 @@ program ftst_pio call MPI_Comm_rank(MPI_COMM_WORLD, myRank, ierr) call MPI_Comm_size(MPI_COMM_WORLD, ntasks, ierr) - ! ierr = pio_set_log_level(2) - ! ierr = nf_set_log_level(2) + ierr = pio_set_log_level(2) + ierr = nf_set_log_level(2) call nf_init_intracom(myRank, MPI_COMM_WORLD, niotasks, numAggregator, & stride, PIO_rearr_subset, ioSystem, base) From a48cd4afde4cfe2c7b20ea04cfede1302ad7441f Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Thu, 11 Jul 2019 09:13:35 -0600 Subject: [PATCH 11/17] turning off netcdf integration in Fortran if NETCDF_INTEGRATION is not set by configure.ac --- src/flib/piolib_mod.F90 | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/flib/piolib_mod.F90 b/src/flib/piolib_mod.F90 index d50ad7f03cc..014e641840c 100644 --- a/src/flib/piolib_mod.F90 +++ b/src/flib/piolib_mod.F90 @@ -98,7 +98,9 @@ module piolib_mod PIO_deletefile, & PIO_get_numiotasks, & PIO_iotype_available, & +#ifdef NETCDF_INTEGRATION nf_init_intracom, & +#endif PIO_set_rearr_opts !----------------------------------------------------------------------- @@ -135,11 +137,11 @@ module piolib_mod module procedure setframe end interface PIO_setframe -!#ifdef NETCDF_INTEGRATION +#ifdef NETCDF_INTEGRATION interface nf_init_intracom module procedure nf_init_intracom end interface nf_init_intracom -!#endif +#endif !> !! Increment the record number for a future read/write of distributed @@ -979,7 +981,7 @@ end function PIOc_Init_Intracomm_from_F90 #endif end subroutine init_intracom -!#ifdef NETCDF_INTEGRATION +#ifdef NETCDF_INTEGRATION !> !! @public !! @ingroup PIO_init @@ -1030,7 +1032,7 @@ end function nc_set_iosystem ierr = nc_set_iosystem(iosystem%iosysid) end subroutine nf_init_intracom -!#endif +#endif !! @public !! @ingroup PIO_init From a9832f836c9f0f634a006910f582bc3bef86f6f6 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Thu, 11 Jul 2019 09:13:46 -0600 Subject: [PATCH 12/17] turning off netcdf integration in Fortran if NETCDF_INTEGRATION is not set by configure.ac --- src/flib/pio.F90 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/flib/pio.F90 b/src/flib/pio.F90 index 280c9a836d5..3c3c007ed13 100644 --- a/src/flib/pio.F90 +++ b/src/flib/pio.F90 @@ -22,9 +22,9 @@ module pio pio_freedecomp, pio_syncfile, & pio_finalize, pio_set_hint, pio_getnumiotasks, pio_file_is_open, & PIO_deletefile, PIO_get_numiotasks, PIO_iotype_available, & -!#ifdef NETCDF_INTEGRATION +#ifdef NETCDF_INTEGRATION nf_init_intracom, & -!#endif +#endif pio_set_rearr_opts use pio_types, only : io_desc_t, file_desc_t, var_desc_t, iosystem_desc_t, & From f7cc7a424a51dd6737d7124024b88b5af0c466d6 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Thu, 11 Jul 2019 09:44:26 -0600 Subject: [PATCH 13/17] trying to fix cmake build --- src/flib/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/src/flib/CMakeLists.txt b/src/flib/CMakeLists.txt index 42b70edb789..d70dae8a5d5 100644 --- a/src/flib/CMakeLists.txt +++ b/src/flib/CMakeLists.txt @@ -38,6 +38,7 @@ endif () # Include flib source and binary directories (for Fortran modules) target_include_directories (piof + PUBLIC ${CMAKE_BINARY_DIR} PUBLIC ${CMAKE_CURRENT_SOURCE_DIR} PUBLIC ${CMAKE_CURRENT_BINARY_DIR}) From 22e36e0d5b0e28be4f4f76ef396cf8455811655c Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Thu, 11 Jul 2019 10:45:27 -0600 Subject: [PATCH 14/17] trying to fix cmake build --- src/flib/CMakeLists.txt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/flib/CMakeLists.txt b/src/flib/CMakeLists.txt index d70dae8a5d5..f76d22f1cbe 100644 --- a/src/flib/CMakeLists.txt +++ b/src/flib/CMakeLists.txt @@ -38,9 +38,10 @@ endif () # Include flib source and binary directories (for Fortran modules) target_include_directories (piof - PUBLIC ${CMAKE_BINARY_DIR} PUBLIC ${CMAKE_CURRENT_SOURCE_DIR} - PUBLIC ${CMAKE_CURRENT_BINARY_DIR}) + PUBLIC ${CMAKE_CURRENT_BINARY_DIR} + PUBLIC ${CMAKE_BINARY_DIR} + ) # System and compiler CPP directives target_compile_definitions (piof From 1fffdf8de047dd7b72e39b0794185698df62e826 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Thu, 11 Jul 2019 11:05:02 -0600 Subject: [PATCH 15/17] trying to fix cmake build --- src/flib/CMakeLists.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/src/flib/CMakeLists.txt b/src/flib/CMakeLists.txt index f76d22f1cbe..58c30caca1a 100644 --- a/src/flib/CMakeLists.txt +++ b/src/flib/CMakeLists.txt @@ -40,7 +40,6 @@ endif () target_include_directories (piof PUBLIC ${CMAKE_CURRENT_SOURCE_DIR} PUBLIC ${CMAKE_CURRENT_BINARY_DIR} - PUBLIC ${CMAKE_BINARY_DIR} ) # System and compiler CPP directives From e6991ab2a907ad45f2e03cf569f8887660b5e42f Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Thu, 11 Jul 2019 11:14:31 -0600 Subject: [PATCH 16/17] trying to fix cmake build --- src/flib/CMakeLists.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/flib/CMakeLists.txt b/src/flib/CMakeLists.txt index 58c30caca1a..42b70edb789 100644 --- a/src/flib/CMakeLists.txt +++ b/src/flib/CMakeLists.txt @@ -39,8 +39,7 @@ endif () # Include flib source and binary directories (for Fortran modules) target_include_directories (piof PUBLIC ${CMAKE_CURRENT_SOURCE_DIR} - PUBLIC ${CMAKE_CURRENT_BINARY_DIR} - ) + PUBLIC ${CMAKE_CURRENT_BINARY_DIR}) # System and compiler CPP directives target_compile_definitions (piof From b4882a338526f3ec9b274c6f0d1d88548fbe5933 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Fri, 12 Jul 2019 09:40:07 -0600 Subject: [PATCH 17/17] commenting out --- src/flib/pio.F90 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/flib/pio.F90 b/src/flib/pio.F90 index 3c3c007ed13..46daa9604e3 100644 --- a/src/flib/pio.F90 +++ b/src/flib/pio.F90 @@ -22,10 +22,10 @@ module pio pio_freedecomp, pio_syncfile, & pio_finalize, pio_set_hint, pio_getnumiotasks, pio_file_is_open, & PIO_deletefile, PIO_get_numiotasks, PIO_iotype_available, & -#ifdef NETCDF_INTEGRATION - nf_init_intracom, & -#endif pio_set_rearr_opts +! #ifdef NETCDF_INTEGRATION +! nf_init_intracom, & +! #endif use pio_types, only : io_desc_t, file_desc_t, var_desc_t, iosystem_desc_t, & pio_rearr_opt_t, pio_rearr_comm_fc_opt_t, pio_rearr_comm_fc_2d_enable,&