From 853d5c94eb30965600e4a95b2fbc5c287994c553 Mon Sep 17 00:00:00 2001 From: Jayesh Krishna Date: Fri, 30 Oct 2015 19:32:29 +0000 Subject: [PATCH 1/3] Updating timing events and associated logic in pio Porting pre-CIME change #1d59b4e to CIME. Performance experiments have identified the utility of some additional timers in the PIO library. --- cime/externals/pio/pio/box_rearrange.F90.in | 45 +++++++++++++++++++++ cime/externals/pio/pio/piodarray.F90.in | 14 +++++++ 2 files changed, 59 insertions(+) diff --git a/cime/externals/pio/pio/box_rearrange.F90.in b/cime/externals/pio/pio/box_rearrange.F90.in index e55df93e325a..a8546c72426f 100644 --- a/cime/externals/pio/pio/box_rearrange.F90.in +++ b/cime/externals/pio/pio/box_rearrange.F90.in @@ -81,6 +81,9 @@ module box_rearrange #else use pio_support, only : piodie, Debug, DebugIO, CheckMPIReturn, pio_fc_gather_offset +#endif +#ifdef TIMING + use perf_mod, only : t_startf, t_stopf !_EXTERNAL #endif use alloc_mod, only : alloc_check, dealloc_check use pio_spmd_utils, only : pio_swapm @@ -296,16 +299,30 @@ subroutine box_rearrange_comp2io_{TYPE} (IOsystem, ioDesc, s1, src, niodof, & #ifdef _USE_ALLTOALLW if (pio_option == COLLECTIVE) then +#ifdef TIMING + call t_startf("a2a_box_rear_comp2io_{TYPE}") +#endif call MPI_ALLTOALLW(src, a2a_sendcounts, a2a_displs, a2a_sendtypes, & dest, a2a_recvcounts, a2a_displs, a2a_recvtypes, & IOsystem%union_comm, ierror ) +#ifdef TIMING + call t_stopf("a2a_box_rear_comp2io_{TYPE}") +#endif call CheckMPIReturn('box_rearrange', ierror) else #endif + +#ifdef TIMING + call t_startf("swapm_box_rear_comp2io_{TYPE}") +#endif call pio_swapm( nprocs, myrank, & src, ndof, a2a_sendcounts, a2a_displs, a2a_sendtypes, & dest, niodof, a2a_recvcounts, a2a_displs, a2a_recvtypes, & IOsystem%union_comm, pio_hs, pio_isend, pio_maxreq ) +#ifdef TIMING + call t_stopf("swapm_box_rear_comp2io_{TYPE}") +#endif + #ifdef _USE_ALLTOALLW endif #endif @@ -324,6 +341,9 @@ subroutine box_rearrange_comp2io_{TYPE} (IOsystem, ioDesc, s1, src, niodof, & endif #endif +#ifdef TIMING + call t_startf("p2p_box_rear_comp2io_{TYPE}") +#endif ! ! send data from comp procs ! @@ -379,6 +399,10 @@ subroutine box_rearrange_comp2io_{TYPE} (IOsystem, ioDesc, s1, src, niodof, & call dealloc_check(sreq, 'send requests') +#ifdef TIMING + call t_stopf("p2p_box_rear_comp2io_{TYPE}") +#endif + #if DEBUG_BARRIER call MPI_BARRIER(IOsystem%union_comm,ierror) call CheckMPIReturn(subName,ierror) @@ -562,16 +586,31 @@ subroutine box_rearrange_io2comp_{TYPE} (IOsystem,ioDesc,s1, iobuf,s2, compbuf, #ifdef _USE_ALLTOALLW if (pio_option == COLLECTIVE) then + +#ifdef TIMING + call t_startf("a2a_box_rear_io2comp_{TYPE}") +#endif call MPI_ALLTOALLW(iobuf, a2a_sendcounts, a2a_displs, a2a_sendtypes, & compbuf, a2a_recvcounts, a2a_displs, a2a_recvtypes, & IOsystem%union_comm, ierror ) +#ifdef TIMING + call t_stopf("a2a_box_rear_io2comp_{TYPE}") +#endif call CheckMPIReturn(subName, ierror) else #endif + +#ifdef TIMING + call t_startf("swapm_box_rear_io2comp_{TYPE}") +#endif call pio_swapm( nprocs, myrank, & iobuf, niodof, a2a_sendcounts, a2a_displs, a2a_sendtypes, & compbuf, ndof, a2a_recvcounts, a2a_displs, a2a_recvtypes, & IOsystem%union_comm, pio_hs, pio_isend, pio_maxreq ) +#ifdef TIMING + call t_stopf("swapm_box_rear_io2comp_{TYPE}") +#endif + #ifdef _USE_ALLTOALLW endif #endif @@ -583,6 +622,9 @@ subroutine box_rearrange_io2comp_{TYPE} (IOsystem,ioDesc,s1, iobuf,s2, compbuf, else +#ifdef TIMING + call t_startf("p2p_box_rear_io2comp_{TYPE}") +#endif call alloc_check(rreq, num_iotasks, 'recv requests') ! @@ -642,6 +684,9 @@ subroutine box_rearrange_io2comp_{TYPE} (IOsystem,ioDesc,s1, iobuf,s2, compbuf, call dealloc_check(sreq,'send requests') endif +#ifdef TIMING + call t_stopf("p2p_box_rear_io2comp_{TYPE}") +#endif endif ! POINT_TO_POINT #endif /* not _MPISERIAL */ diff --git a/cime/externals/pio/pio/piodarray.F90.in b/cime/externals/pio/pio/piodarray.F90.in index b9517be8855d..e808455ac0d1 100644 --- a/cime/externals/pio/pio/piodarray.F90.in +++ b/cime/externals/pio/pio/piodarray.F90.in @@ -503,6 +503,7 @@ contains File%iosystem%comp_rank,' UseRearranger: ',UseRearranger,iodesc%glen, iodesc%iomap%start, len #ifdef TIMING call t_startf("pio_rearrange_write") + call t_startf("pre_pio_write_nf") #endif if(UseRearranger) then if (IOproc) then @@ -638,6 +639,7 @@ contains endif #ifdef TIMING + call t_stopf("pre_pio_write_nf") call t_startf("pio_write_nf") #endif ierr = write_nf(File,IOBUF,varDesc,iodesc,start,count, request) @@ -648,11 +650,17 @@ contains call dealloc_check(count) if(IOPROC) then +#ifdef TIMING + call t_startf("post_pio_write_nf") +#endif if(file%iotype==pio_iotype_pnetcdf) then call add_data_to_buffer(File, IOBUF, request) else if(Userearranger) then deallocate(iobuf) end if +#ifdef TIMING + call t_stopf("post_pio_write_nf") +#endif end if ! call MPI_Barrier(File%iosystem%comp_comm,ierr) @@ -1249,7 +1257,13 @@ contains this_buffsize = size(iobuf)*c_sizeof(iobuf(1)) file%buffsize=file%buffsize+this_buffsize total_buffsize = total_buffsize+this_buffsize +#ifdef TIMING + call t_startf("allred_add_data_to_buf") +#endif call MPI_ALLREDUCE(total_buffsize,maxbuffsize,1,MPI_INTEGER,MPI_MAX,file%iosystem%io_comm, mpierr) +#ifdef TIMING + call t_stopf("allred_add_data_to_buf") +#endif if(maxbuffsize > pio_buffer_size_limit) then call darray_write_complete(File) From fa0254e45568fb5bbd1b284882c1f41415bcdeaa Mon Sep 17 00:00:00 2001 From: Jayesh Krishna Date: Fri, 30 Oct 2015 19:37:14 +0000 Subject: [PATCH 2/3] Allow adding PIO cmake config options in config scripts Porting pre-CIME change #c28f7f80 to CIME This change allows users to add PIO cmake configuration options using the PIO_CONFIG_OPTS tag in the xml configuration files. Without the fix there is no easy way to add cmake configuration options for PIO from configuration files like config_machines.xml [B4B] --- cime/machines-acme/Makefile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cime/machines-acme/Makefile b/cime/machines-acme/Makefile index 6bcd6f3e2472..9207ef194be7 100644 --- a/cime/machines-acme/Makefile +++ b/cime/machines-acme/Makefile @@ -582,6 +582,9 @@ endif ifdef PIO_FILESYSTEM_HINTS CMAKE_OPTS += -D PIO_FILESYSTEM_HINTS:STRING="$(PIO_FILESYSTEM_HINTS)" endif +ifdef PIO_CONFIG_OPTS + CMAKE_OPTS += $(PIO_CONFIG_OPTS) +endif # This captures the many cism-specific options to cmake CMAKE_OPTS += $(USER_CMAKE_OPTS) From beb25bcd7c4502c049b81a5d748b05e5fd5910ca Mon Sep 17 00:00:00 2001 From: Jayesh Krishna Date: Fri, 30 Oct 2015 20:51:26 +0000 Subject: [PATCH 3/3] Adding CIME hooks for the PIO config options Modifying the CIME build scripts to recognize the PIO config options added in #fa0254e . Without this change the PIO config options in env_build.xml is not used when building cesm. --- cime/scripts/Tools/cesm_build.pl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cime/scripts/Tools/cesm_build.pl b/cime/scripts/Tools/cesm_build.pl index 6275282b5935..5516cfced3b3 100755 --- a/cime/scripts/Tools/cesm_build.pl +++ b/cime/scripts/Tools/cesm_build.pl @@ -50,6 +50,7 @@ my $SHAREDPATH; my $CLM_CONFIG_OPTS; my $CAM_CONFIG_OPTS; +my $PIO_CONFIG_OPTS; my $sysmod; # Stash the build log paths here.. @@ -106,6 +107,7 @@ sub main { $MPASLI_USE_ALBANY = `./xmlquery MPASLI_USE_ALBANY -value`; $CLM_CONFIG_OPTS = `./xmlquery CLM_CONFIG_OPTS -value`; $CAM_CONFIG_OPTS = `./xmlquery CAM_CONFIG_OPTS -value`; + $PIO_CONFIG_OPTS = `./xmlquery PIO_CONFIG_OPTS -value`; my $NINST_VALUE = `./xmlquery NINST_VALUE -value `; my $MACH = `./xmlquery MACH -value `; @@ -138,6 +140,7 @@ sub main { $ENV{COMP_ROF} = $COMP_ROF ; $ENV{CLM_CONFIG_OPTS} = $CLM_CONFIG_OPTS ; $ENV{CAM_CONFIG_OPTS} = $CAM_CONFIG_OPTS ; + $ENV{PIO_CONFIG_OPTS} = $PIO_CONFIG_OPTS ; $ENV{OCN_SUBMODEL} = `./xmlquery OCN_SUBMODEL -value `; $ENV{PROFILE_PAPI_ENABLE} = `./xmlquery PROFILE_PAPI_ENABLE -value `;