Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Porting PIO timers to CIME #411

Merged
merged 3 commits into from
Dec 18, 2015
Merged

Porting PIO timers to CIME #411

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
853d5c9
Updating timing events and associated logic in pio
jayeshkrishna Oct 30, 2015
fa0254e
Allow adding PIO cmake config options in config scripts
jayeshkrishna Oct 30, 2015
beb25bc
Adding CIME hooks for the PIO config options
jayeshkrishna Oct 30, 2015
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 45 additions & 0 deletions cime/externals/pio/pio/box_rearrange.F90.in
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,9 @@ module box_rearrange
#else
use pio_support, only : piodie, Debug, DebugIO, CheckMPIReturn, pio_fc_gather_offset

#endif
#ifdef TIMING
use perf_mod, only : t_startf, t_stopf !_EXTERNAL
#endif
use alloc_mod, only : alloc_check, dealloc_check
use pio_spmd_utils, only : pio_swapm
Expand Down Expand Up @@ -296,16 +299,30 @@ subroutine box_rearrange_comp2io_{TYPE} (IOsystem, ioDesc, s1, src, niodof, &
#ifdef _USE_ALLTOALLW
if (pio_option == COLLECTIVE) then

#ifdef TIMING
call t_startf("a2a_box_rear_comp2io_{TYPE}")
#endif
call MPI_ALLTOALLW(src, a2a_sendcounts, a2a_displs, a2a_sendtypes, &
dest, a2a_recvcounts, a2a_displs, a2a_recvtypes, &
IOsystem%union_comm, ierror )
#ifdef TIMING
call t_stopf("a2a_box_rear_comp2io_{TYPE}")
#endif
call CheckMPIReturn('box_rearrange', ierror)
else
#endif

#ifdef TIMING
call t_startf("swapm_box_rear_comp2io_{TYPE}")
#endif
call pio_swapm( nprocs, myrank, &
src, ndof, a2a_sendcounts, a2a_displs, a2a_sendtypes, &
dest, niodof, a2a_recvcounts, a2a_displs, a2a_recvtypes, &
IOsystem%union_comm, pio_hs, pio_isend, pio_maxreq )
#ifdef TIMING
call t_stopf("swapm_box_rear_comp2io_{TYPE}")
#endif

#ifdef _USE_ALLTOALLW
endif
#endif
Expand All @@ -324,6 +341,9 @@ subroutine box_rearrange_comp2io_{TYPE} (IOsystem, ioDesc, s1, src, niodof, &
endif
#endif

#ifdef TIMING
call t_startf("p2p_box_rear_comp2io_{TYPE}")
#endif
!
! send data from comp procs
!
Expand Down Expand Up @@ -379,6 +399,10 @@ subroutine box_rearrange_comp2io_{TYPE} (IOsystem, ioDesc, s1, src, niodof, &

call dealloc_check(sreq, 'send requests')

#ifdef TIMING
call t_stopf("p2p_box_rear_comp2io_{TYPE}")
#endif

#if DEBUG_BARRIER
call MPI_BARRIER(IOsystem%union_comm,ierror)
call CheckMPIReturn(subName,ierror)
Expand Down Expand Up @@ -562,16 +586,31 @@ subroutine box_rearrange_io2comp_{TYPE} (IOsystem,ioDesc,s1, iobuf,s2, compbuf,

#ifdef _USE_ALLTOALLW
if (pio_option == COLLECTIVE) then

#ifdef TIMING
call t_startf("a2a_box_rear_io2comp_{TYPE}")
#endif
call MPI_ALLTOALLW(iobuf, a2a_sendcounts, a2a_displs, a2a_sendtypes, &
compbuf, a2a_recvcounts, a2a_displs, a2a_recvtypes, &
IOsystem%union_comm, ierror )
#ifdef TIMING
call t_stopf("a2a_box_rear_io2comp_{TYPE}")
#endif
call CheckMPIReturn(subName, ierror)
else
#endif

#ifdef TIMING
call t_startf("swapm_box_rear_io2comp_{TYPE}")
#endif
call pio_swapm( nprocs, myrank, &
iobuf, niodof, a2a_sendcounts, a2a_displs, a2a_sendtypes, &
compbuf, ndof, a2a_recvcounts, a2a_displs, a2a_recvtypes, &
IOsystem%union_comm, pio_hs, pio_isend, pio_maxreq )
#ifdef TIMING
call t_stopf("swapm_box_rear_io2comp_{TYPE}")
#endif

#ifdef _USE_ALLTOALLW
endif
#endif
Expand All @@ -583,6 +622,9 @@ subroutine box_rearrange_io2comp_{TYPE} (IOsystem,ioDesc,s1, iobuf,s2, compbuf,

else

#ifdef TIMING
call t_startf("p2p_box_rear_io2comp_{TYPE}")
#endif
call alloc_check(rreq, num_iotasks, 'recv requests')

!
Expand Down Expand Up @@ -642,6 +684,9 @@ subroutine box_rearrange_io2comp_{TYPE} (IOsystem,ioDesc,s1, iobuf,s2, compbuf,

call dealloc_check(sreq,'send requests')
endif
#ifdef TIMING
call t_stopf("p2p_box_rear_io2comp_{TYPE}")
#endif

endif ! POINT_TO_POINT
#endif /* not _MPISERIAL */
Expand Down
14 changes: 14 additions & 0 deletions cime/externals/pio/pio/piodarray.F90.in
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -503,6 +503,7 @@ contains
File%iosystem%comp_rank,' UseRearranger: ',UseRearranger,iodesc%glen, iodesc%iomap%start, len
#ifdef TIMING
call t_startf("pio_rearrange_write")
call t_startf("pre_pio_write_nf")
#endif
if(UseRearranger) then
if (IOproc) then
Expand Down Expand Up @@ -638,6 +639,7 @@ contains
endif

#ifdef TIMING
call t_stopf("pre_pio_write_nf")
call t_startf("pio_write_nf")
#endif
ierr = write_nf(File,IOBUF,varDesc,iodesc,start,count, request)
Expand All @@ -648,11 +650,17 @@ contains
call dealloc_check(count)

if(IOPROC) then
#ifdef TIMING
call t_startf("post_pio_write_nf")
#endif
if(file%iotype==pio_iotype_pnetcdf) then
call add_data_to_buffer(File, IOBUF, request)
else if(Userearranger) then
deallocate(iobuf)
end if
#ifdef TIMING
call t_stopf("post_pio_write_nf")
#endif
end if

! call MPI_Barrier(File%iosystem%comp_comm,ierr)
Expand Down Expand Up @@ -1249,7 +1257,13 @@ contains
this_buffsize = size(iobuf)*c_sizeof(iobuf(1))
file%buffsize=file%buffsize+this_buffsize
total_buffsize = total_buffsize+this_buffsize
#ifdef TIMING
call t_startf("allred_add_data_to_buf")
#endif
call MPI_ALLREDUCE(total_buffsize,maxbuffsize,1,MPI_INTEGER,MPI_MAX,file%iosystem%io_comm, mpierr)
#ifdef TIMING
call t_stopf("allred_add_data_to_buf")
#endif

if(maxbuffsize > pio_buffer_size_limit) then
call darray_write_complete(File)
Expand Down
3 changes: 3 additions & 0 deletions cime/machines-acme/Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -582,6 +582,9 @@ endif
ifdef PIO_FILESYSTEM_HINTS
CMAKE_OPTS += -D PIO_FILESYSTEM_HINTS:STRING="$(PIO_FILESYSTEM_HINTS)"
endif
ifdef PIO_CONFIG_OPTS
CMAKE_OPTS += $(PIO_CONFIG_OPTS)
endif

# This captures the many cism-specific options to cmake
CMAKE_OPTS += $(USER_CMAKE_OPTS)
Expand Down
3 changes: 3 additions & 0 deletions cime/scripts/Tools/cesm_build.pl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@
my $SHAREDPATH;
my $CLM_CONFIG_OPTS;
my $CAM_CONFIG_OPTS;
my $PIO_CONFIG_OPTS;
my $sysmod;

# Stash the build log paths here..
Expand Down Expand Up @@ -106,6 +107,7 @@ sub main {
$MPASLI_USE_ALBANY = `./xmlquery MPASLI_USE_ALBANY -value`;
$CLM_CONFIG_OPTS = `./xmlquery CLM_CONFIG_OPTS -value`;
$CAM_CONFIG_OPTS = `./xmlquery CAM_CONFIG_OPTS -value`;
$PIO_CONFIG_OPTS = `./xmlquery PIO_CONFIG_OPTS -value`;

my $NINST_VALUE = `./xmlquery NINST_VALUE -value `;
my $MACH = `./xmlquery MACH -value `;
Expand Down Expand Up @@ -138,6 +140,7 @@ sub main {
$ENV{COMP_ROF} = $COMP_ROF ;
$ENV{CLM_CONFIG_OPTS} = $CLM_CONFIG_OPTS ;
$ENV{CAM_CONFIG_OPTS} = $CAM_CONFIG_OPTS ;
$ENV{PIO_CONFIG_OPTS} = $PIO_CONFIG_OPTS ;

$ENV{OCN_SUBMODEL} = `./xmlquery OCN_SUBMODEL -value `;
$ENV{PROFILE_PAPI_ENABLE} = `./xmlquery PROFILE_PAPI_ENABLE -value `;
Expand Down