Skip to content

Commit

Permalink
Merge branch 'jayeshkrishna/cime/pio/pio_timers_port' into next (PR #411
Browse files Browse the repository at this point in the history 
)

Porting PIO timers to CIME

The latest version of PIO adopted via CIME does not
have some PIO timers that were added to PIO in
ACME. So porting these timers to CIME.

This change adds some additional timers in the PIO
library that were identified during performance
experiments.

Also including the necessary hooks to recognize
user-specified PIO configure options.

[BFB]

SEG-193

* jayeshkrishna/cime/pio/pio_timers_port:
  Adding CIME hooks for the PIO config options
  Allow adding PIO cmake config options in config scripts
  Updating timing events and associated logic in pio
  • Loading branch information
Patrick Worley committed Dec 16, 2015
2 parents c505fc1 + beb25bc commit 0e2471c
Show file tree
Hide file tree
Showing 4 changed files with 61 additions and 0 deletions.
41 changes: 41 additions & 0 deletions cime/externals/pio/pio/box_rearrange.F90.in
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,9 @@ module box_rearrange
#else
use pio_support, only : piodie, Debug, DebugIO, CheckMPIReturn, pio_fc_gather_offset

#endif
#ifdef TIMING
use perf_mod, only : t_startf, t_stopf !_EXTERNAL
#endif
use alloc_mod, only : alloc_check, dealloc_check
use pio_spmd_utils, only : pio_swapm
Expand Down Expand Up @@ -282,15 +285,28 @@ subroutine box_rearrange_comp2io_{TYPE} (IOsystem, ioDesc, s1, src, niodof, &

if (pio_option == COLLECTIVE) then

#ifdef TIMING
call t_startf("a2a_box_rear_comp2io_{TYPE}")
#endif
call MPI_ALLTOALLW(src, a2a_sendcounts, a2a_displs, a2a_sendtypes, &
dest, a2a_recvcounts, a2a_displs, a2a_recvtypes, &
IOsystem%union_comm, ierror )
#ifdef TIMING
call t_stopf("a2a_box_rear_comp2io_{TYPE}")
#endif
call CheckMPIReturn('box_rearrange', ierror)
else

#ifdef TIMING
call t_startf("swapm_box_rear_comp2io_{TYPE}")
#endif
call pio_swapm( nprocs, myrank, &
src, ndof, a2a_sendcounts, a2a_displs, a2a_sendtypes, &
dest, niodof, a2a_recvcounts, a2a_displs, a2a_recvtypes, &
IOsystem%union_comm, pio_hs, pio_isend, pio_maxreq )
#ifdef TIMING
call t_stopf("swapm_box_rear_comp2io_{TYPE}")
#endif
endif
call dealloc_check(a2a_sendcounts)
call dealloc_check(a2a_displs)
Expand All @@ -307,6 +323,9 @@ subroutine box_rearrange_comp2io_{TYPE} (IOsystem, ioDesc, s1, src, niodof, &
endif
#endif

#ifdef TIMING
call t_startf("p2p_box_rear_comp2io_{TYPE}")
#endif
!
! send data from comp procs
!
Expand Down Expand Up @@ -362,6 +381,10 @@ subroutine box_rearrange_comp2io_{TYPE} (IOsystem, ioDesc, s1, src, niodof, &

call dealloc_check(sreq, 'send requests')

#ifdef TIMING
call t_stopf("p2p_box_rear_comp2io_{TYPE}")
#endif

#if DEBUG_BARRIER
call MPI_BARRIER(IOsystem%union_comm,ierror)
call CheckMPIReturn(subName,ierror)
Expand Down Expand Up @@ -564,15 +587,27 @@ subroutine box_rearrange_io2comp_{TYPE} (IOsystem,ioDesc,s1, iobuf,s2, compbuf,
endif

if (pio_option == COLLECTIVE) then
#ifdef TIMING
call t_startf("a2a_box_rear_io2comp_{TYPE}")
#endif
call MPI_ALLTOALLW(iobuf, a2a_sendcounts, a2a_displs, a2a_sendtypes, &
compbuf, a2a_recvcounts, a2a_displs, a2a_recvtypes, &
IOsystem%union_comm, ierror )
#ifdef TIMING
call t_stopf("a2a_box_rear_io2comp_{TYPE}")
#endif
call CheckMPIReturn(subName, ierror)
else
#ifdef TIMING
call t_startf("swapm_box_rear_io2comp_{TYPE}")
#endif
call pio_swapm( nprocs, myrank, &
iobuf, niodof, a2a_sendcounts, a2a_displs, a2a_sendtypes, &
compbuf, ndof, a2a_recvcounts, a2a_displs, a2a_recvtypes, &
IOsystem%union_comm, pio_hs, pio_isend, pio_maxreq )
#ifdef TIMING
call t_stopf("swapm_box_rear_io2comp_{TYPE}")
#endif
endif
call dealloc_check(a2a_sendcounts)
call dealloc_check(a2a_displs)
Expand All @@ -582,6 +617,9 @@ subroutine box_rearrange_io2comp_{TYPE} (IOsystem,ioDesc,s1, iobuf,s2, compbuf,

else

#ifdef TIMING
call t_startf("p2p_box_rear_io2comp_{TYPE}")
#endif
call alloc_check(rreq, num_iotasks, 'recv requests')

!
Expand Down Expand Up @@ -641,6 +679,9 @@ subroutine box_rearrange_io2comp_{TYPE} (IOsystem,ioDesc,s1, iobuf,s2, compbuf,

call dealloc_check(sreq,'send requests')
endif
#ifdef TIMING
call t_stopf("p2p_box_rear_io2comp_{TYPE}")
#endif

endif ! POINT_TO_POINT
#endif /* not _MPISERIAL */
Expand Down
14 changes: 14 additions & 0 deletions cime/externals/pio/pio/piodarray.F90.in
View file
Original file line number Diff line number Diff line change
Expand Up @@ -503,6 +503,7 @@ contains
File%iosystem%comp_rank,' UseRearranger: ',UseRearranger,iodesc%glen, iodesc%iomap%start, len
#ifdef TIMING
call t_startf("pio_rearrange_write")
call t_startf("pre_pio_write_nf")
#endif
if(UseRearranger) then
if (IOproc) then
Expand Down Expand Up @@ -638,6 +639,7 @@ contains
endif

#ifdef TIMING
call t_stopf("pre_pio_write_nf")
call t_startf("pio_write_nf")
#endif
ierr = write_nf(File,IOBUF,varDesc,iodesc,start,count, request)
Expand All @@ -648,11 +650,17 @@ contains
call dealloc_check(count)

if(IOPROC) then
#ifdef TIMING
call t_startf("post_pio_write_nf")
#endif
if(file%iotype==pio_iotype_pnetcdf) then
call add_data_to_buffer(File, IOBUF, request)
else if(Userearranger) then
deallocate(iobuf)
end if
#ifdef TIMING
call t_stopf("post_pio_write_nf")
#endif
end if

! call MPI_Barrier(File%iosystem%comp_comm,ierr)
Expand Down Expand Up @@ -1249,7 +1257,13 @@ contains
this_buffsize = size(iobuf)*c_sizeof(iobuf(1))
file%buffsize=file%buffsize+this_buffsize
total_buffsize = total_buffsize+this_buffsize
#ifdef TIMING
call t_startf("allred_add_data_to_buf")
#endif
call MPI_ALLREDUCE(total_buffsize,maxbuffsize,1,MPI_INTEGER,MPI_MAX,file%iosystem%io_comm, mpierr)
#ifdef TIMING
call t_stopf("allred_add_data_to_buf")
#endif

if(maxbuffsize > pio_buffer_size_limit) then
call darray_write_complete(File)
Expand Down
3 changes: 3 additions & 0 deletions cime/machines-acme/Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -588,6 +588,9 @@ endif
ifdef PIO_FILESYSTEM_HINTS
CMAKE_OPTS += -D PIO_FILESYSTEM_HINTS:STRING="$(PIO_FILESYSTEM_HINTS)"
endif
ifdef PIO_CONFIG_OPTS
CMAKE_OPTS += $(PIO_CONFIG_OPTS)
endif

# This captures the many cism-specific options to cmake
CMAKE_OPTS += $(USER_CMAKE_OPTS)
Expand Down
3 changes: 3 additions & 0 deletions cime/scripts/Tools/cesm_build.pl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@
my $SHAREDPATH;
my $CLM_CONFIG_OPTS;
my $CAM_CONFIG_OPTS;
my $PIO_CONFIG_OPTS;
my $sysmod;

# Stash the build log paths here..
Expand Down Expand Up @@ -106,6 +107,7 @@ sub main {
$MPASLI_USE_ALBANY = `./xmlquery MPASLI_USE_ALBANY -value`;
$CLM_CONFIG_OPTS = `./xmlquery CLM_CONFIG_OPTS -value`;
$CAM_CONFIG_OPTS = `./xmlquery CAM_CONFIG_OPTS -value`;
$PIO_CONFIG_OPTS = `./xmlquery PIO_CONFIG_OPTS -value`;

my $NINST_VALUE = `./xmlquery NINST_VALUE -value `;
my $MACH = `./xmlquery MACH -value `;
Expand Down Expand Up @@ -138,6 +140,7 @@ sub main {
$ENV{COMP_ROF} = $COMP_ROF ;
$ENV{CLM_CONFIG_OPTS} = $CLM_CONFIG_OPTS ;
$ENV{CAM_CONFIG_OPTS} = $CAM_CONFIG_OPTS ;
$ENV{PIO_CONFIG_OPTS} = $PIO_CONFIG_OPTS ;

$ENV{OCN_SUBMODEL} = `./xmlquery OCN_SUBMODEL -value `;
$ENV{PROFILE_PAPI_ENABLE} = `./xmlquery PROFILE_PAPI_ENABLE -value `;
Expand Down

0 comments on commit 0e2471c

Please sign in to comment.