Skip to content

Commit

Permalink
Cime hooks for more PIO1 rearranger options
Browse files Browse the repository at this point in the history 
This commit adds the cime hooks required for using the new
runtime options added in PIO1.

* This commit allows user to control the flow control options
  for comp2io and io2comp directions (separately) via env_run.xml

* The old flow control options are suffixed with COMP2IO and IO2COMP
  as needed.

Note: Due to changes in the pio rearranger opt structure this
commit requires the corresponding changes in PIO1.
  • Loading branch information
@jayeshkrishna
jayeshkrishna committed Jul 26, 2016
1 parent 8ff5f19 commit 2e9be53
Show file tree
Hide file tree
Showing 5 changed files with 173 additions and 56 deletions.
9 changes: 6 additions & 3 deletions driver_cpl/bld/build-namelist
View file
Original file line number Diff line number Diff line change
Expand Up @@ -753,9 +753,12 @@ add_default($nl, 'pio_buffer_size_limit');
if($opts{pio_version} == 1){
add_default($nl, 'pio_rearr_comm_type', 'PIO_REARR_COMM_TYPE'=>"$xmlvars{'PIO_REARR_COMM_TYPE'}");
add_default($nl, 'pio_rearr_comm_fcd', 'PIO_REARR_COMM_FCD'=>"$xmlvars{'PIO_REARR_COMM_FCD'}");
add_default($nl, 'pio_rearr_comm_max_pend_req', 'PIO_REARR_COMM_MAX_PEND_REQ'=>"$xmlvars{'PIO_REARR_COMM_MAX_PEND_REQ'}");
add_default($nl, 'pio_rearr_comm_enable_hs', 'PIO_REARR_COMM_ENABLE_HS'=>"$xmlvars{'PIO_REARR_COMM_ENABLE_HS'}");
add_default($nl, 'pio_rearr_comm_enable_isend', 'PIO_REARR_COMM_ENABLE_ISEND'=>"$xmlvars{'PIO_REARR_COMM_ENABLE_ISEND'}");
add_default($nl, 'pio_rearr_comm_max_pend_req_comp2io', 'PIO_REARR_COMM_MAX_PEND_REQ_COMP2IO'=>"$xmlvars{'PIO_REARR_COMM_MAX_PEND_REQ_COMP2IO'}");
add_default($nl, 'pio_rearr_comm_enable_hs_comp2io', 'PIO_REARR_COMM_ENABLE_HS_COMP2IO'=>"$xmlvars{'PIO_REARR_COMM_ENABLE_HS_COMP2IO'}");
add_default($nl, 'pio_rearr_comm_enable_isend_comp2io', 'PIO_REARR_COMM_ENABLE_ISEND_COMP2IO'=>"$xmlvars{'PIO_REARR_COMM_ENABLE_ISEND_COMP2IO'}");
add_default($nl, 'pio_rearr_comm_max_pend_req_io2comp', 'PIO_REARR_COMM_MAX_PEND_REQ_IO2COMP'=>"$xmlvars{'PIO_REARR_COMM_MAX_PEND_REQ_IO2COMP'}");
add_default($nl, 'pio_rearr_comm_enable_hs_io2comp', 'PIO_REARR_COMM_ENABLE_HS_IO2COMP'=>"$xmlvars{'PIO_REARR_COMM_ENABLE_HS_IO2COMP'}");
add_default($nl, 'pio_rearr_comm_enable_isend_io2comp', 'PIO_REARR_COMM_ENABLE_ISEND_IO2COMP'=>"$xmlvars{'PIO_REARR_COMM_ENABLE_ISEND_IO2COMP'}");
}
# Note: pio_async_interface=.true. is not yet supported
# If pio_async_interface is .true. or {component}_PIO_* variable is not set or set to -99
Expand Down
15 changes: 10 additions & 5 deletions driver_cpl/bld/namelist_files/namelist_defaults_drv.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -180,11 +180,16 @@
<pio_async_interface PIO_ASYNC_INTERFACE="FALSE">.false.</pio_async_interface>
<pio_rearr_comm_type >$PIO_REARR_COMM_TYPE</pio_rearr_comm_type>
<pio_rearr_comm_fcd >$PIO_REARR_COMM_FCD</pio_rearr_comm_fcd>
<pio_rearr_comm_max_pend_req>$PIO_REARR_COMM_MAX_PEND_REQ</pio_rearr_comm_max_pend_req>
<pio_rearr_comm_enable_hs PIO_REARR_COMM_ENABLE_HS="TRUE">.true.</pio_rearr_comm_enable_hs>
<pio_rearr_comm_enable_hs PIO_REARR_COMM_ENABLE_HS="FALSE">.false.</pio_rearr_comm_enable_hs>
<pio_rearr_comm_enable_isend PIO_REARR_COMM_ENABLE_ISEND="TRUE">.true.</pio_rearr_comm_enable_isend>
<pio_rearr_comm_enable_isend PIO_REARR_COMM_ENABLE_ISEND="FALSE">.false.</pio_rearr_comm_enable_isend>
<pio_rearr_comm_max_pend_req_comp2io>$PIO_REARR_COMM_MAX_PEND_REQ_COMP2IO</pio_rearr_comm_max_pend_req_comp2io>
<pio_rearr_comm_enable_hs_comp2io PIO_REARR_COMM_ENABLE_HS_COMP2IO="TRUE">.true.</pio_rearr_comm_enable_hs_comp2io>
<pio_rearr_comm_enable_hs_comp2io PIO_REARR_COMM_ENABLE_HS_COMP2IO="FALSE">.false.</pio_rearr_comm_enable_hs_comp2io>
<pio_rearr_comm_enable_isend_comp2io PIO_REARR_COMM_ENABLE_ISEND_COMP2IO="TRUE">.true.</pio_rearr_comm_enable_isend_comp2io>
<pio_rearr_comm_enable_isend_comp2io PIO_REARR_COMM_ENABLE_ISEND_COMP2IO="FALSE">.false.</pio_rearr_comm_enable_isend_comp2io>
<pio_rearr_comm_max_pend_req_io2comp>$PIO_REARR_COMM_MAX_PEND_REQ_IO2COMP</pio_rearr_comm_max_pend_req_io2comp>
<pio_rearr_comm_enable_hs_io2comp PIO_REARR_COMM_ENABLE_HS_IO2COMP="TRUE">.true.</pio_rearr_comm_enable_hs_io2comp>
<pio_rearr_comm_enable_hs_io2comp PIO_REARR_COMM_ENABLE_HS_IO2COMP="FALSE">.false.</pio_rearr_comm_enable_hs_io2comp>
<pio_rearr_comm_enable_isend_io2comp PIO_REARR_COMM_ENABLE_ISEND_IO2COMP="TRUE">.true.</pio_rearr_comm_enable_isend_io2comp>
<pio_rearr_comm_enable_isend_io2comp PIO_REARR_COMM_ENABLE_ISEND_IO2COMP="FALSE">.false.</pio_rearr_comm_enable_isend_io2comp>

<!-- ========================================= -->
<!--- seq_maps.rc -->
Expand Down
38 changes: 32 additions & 6 deletions driver_cpl/bld/namelist_files/namelist_definition_drv.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2116,28 +2116,54 @@ valid values: 2denable,io2comp,comp2io,disable,default
</entry>

<entry
id="pio_rearr_comm_max_pend_req"
id="pio_rearr_comm_max_pend_req_comp2io"
type="integer"
category="pio"
group="pio_default_inparm">
pio rearranger communication max pending req
pio rearranger communication max pending req (comp2io)
</entry>

<entry
id="pio_rearr_comm_enable_hs"
id="pio_rearr_comm_enable_hs_comp2io"
type="logical"
category="pio"
group="pio_default_inparm">
pio rearranger communication option: Enable handshake
pio rearranger communication option: Enable handshake (comp2io)
default: .true.
</entry>

<entry
id="pio_rearr_comm_enable_isend"
id="pio_rearr_comm_enable_isend_comp2io"
type="logical"
category="pio"
group="pio_default_inparm">
pio rearranger communication option: Enable isends
pio rearranger communication option: Enable isends (comp2io)
default: .false.
</entry>

<entry
id="pio_rearr_comm_max_pend_req_io2comp"
type="integer"
category="pio"
group="pio_default_inparm">
pio rearranger communication max pending req (io2comp)
</entry>

<entry
id="pio_rearr_comm_enable_hs_io2comp"
type="logical"
category="pio"
group="pio_default_inparm">
pio rearranger communication option: Enable handshake (io2comp)
default: .true.
</entry>

<entry
id="pio_rearr_comm_enable_isend_io2comp"
type="logical"
category="pio"
group="pio_default_inparm">
pio rearranger communication option: Enable isends (io2comp)
default: .false.
</entry>

Expand Down
39 changes: 33 additions & 6 deletions driver_cpl/cime_config/config_component.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2417,31 +2417,58 @@
<desc>pio rearranger communication flow control direction</desc>
</entry>

<entry id="PIO_REARR_COMM_MAX_PEND_REQ">
<entry id="PIO_REARR_COMM_MAX_PEND_REQ_COMP2IO">
<type>integer</type>
<valid_values></valid_values>
<default_value>64</default_value>
<group>run_pio</group>
<file>env_run.xml</file>
<desc>pio rearranger communication max pending requests</desc>
<desc>pio rearranger communication max pending requests (comp2io) </desc>
</entry>

<entry id="PIO_REARR_COMM_ENABLE_HS">
<entry id="PIO_REARR_COMM_ENABLE_HS_COMP2IO">
<type>logical</type>
<valid_values>TRUE,FALSE</valid_values>
<default_value>TRUE</default_value>
<group>run_pio</group>
<file>env_run.xml</file>
<desc>pio rearranger communiation options: TRUE implies enable handshake</desc>
<desc>pio rearranger communiation options (comp2io) : TRUE implies enable handshake </desc>
</entry>

<entry id="PIO_REARR_COMM_ENABLE_ISEND">
<entry id="PIO_REARR_COMM_ENABLE_ISEND_COMP2IO">
<type>logical</type>
<valid_values>TRUE,FALSE</valid_values>
<default_value>FALSE</default_value>
<group>run_pio</group>
<file>env_run.xml</file>
<desc>pio rearranger communiation options: TRUE implies enable isend</desc>
<desc>pio rearranger communiation options (comp2io) : TRUE implies enable isend</desc>
</entry>

<entry id="PIO_REARR_COMM_MAX_PEND_REQ_IO2COMP">
<type>integer</type>
<valid_values></valid_values>
<default_value>64</default_value>
<group>run_pio</group>
<file>env_run.xml</file>
<desc>pio rearranger communication max pending requests (io2comp)</desc>
</entry>

<entry id="PIO_REARR_COMM_ENABLE_HS_IO2COMP">
<type>logical</type>
<valid_values>TRUE,FALSE</valid_values>
<default_value>TRUE</default_value>
<group>run_pio</group>
<file>env_run.xml</file>
<desc>pio rearranger communiation options (io2comp) : TRUE implies enable handshake</desc>
</entry>

<entry id="PIO_REARR_COMM_ENABLE_ISEND_IO2COMP">
<type>logical</type>
<valid_values>TRUE,FALSE</valid_values>
<default_value>FALSE</default_value>
<group>run_pio</group>
<file>env_run.xml</file>
<desc>pio rearranger communiation options (io2comp) : TRUE implies enable isend</desc>
</entry>

<entry id="PIO_TYPENAME">
Expand Down
128 changes: 92 additions & 36 deletions share/csm_share/shr/shr_pio_mod.F90
View file
Original file line number Diff line number Diff line change
Expand Up @@ -375,8 +375,10 @@ subroutine shr_pio_read_default_namelist(nlfilename, Comm, pio_stride, pio_root,

character(len=shr_kind_cs) :: pio_typename
character(len=shr_kind_cs) :: pio_rearr_comm_type, pio_rearr_comm_fcd
integer :: pio_rearr_comm_max_pend_req
logical :: pio_rearr_comm_enable_hs, pio_rearr_comm_enable_isend
integer :: pio_rearr_comm_max_pend_req_comp2io
logical :: pio_rearr_comm_enable_hs_comp2io, pio_rearr_comm_enable_isend_comp2io
integer :: pio_rearr_comm_max_pend_req_io2comp
logical :: pio_rearr_comm_enable_hs_io2comp, pio_rearr_comm_enable_isend_io2comp
character(*),parameter :: subName = '(shr_pio_read_default_namelist) '

integer :: iam, ierr, npes, unitn
Expand All @@ -386,8 +388,10 @@ subroutine shr_pio_read_default_namelist(nlfilename, Comm, pio_stride, pio_root,
pio_typename, pio_async_interface, pio_debug_level, pio_blocksize, &
pio_buffer_size_limit, pio_rearranger, &
pio_rearr_comm_type, pio_rearr_comm_fcd, &
pio_rearr_comm_max_pend_req, pio_rearr_comm_enable_hs, &
pio_rearr_comm_enable_isend
pio_rearr_comm_max_pend_req_comp2io, pio_rearr_comm_enable_hs_comp2io, &
pio_rearr_comm_enable_isend_comp2io, &
pio_rearr_comm_max_pend_req_io2comp, pio_rearr_comm_enable_hs_io2comp, &
pio_rearr_comm_enable_isend_io2comp
#else
namelist /pio_default_inparm/ pio_stride, pio_root, pio_numiotasks, &
pio_typename, pio_async_interface, pio_debug_level, pio_blocksize, &
Expand Down Expand Up @@ -465,8 +469,10 @@ subroutine shr_pio_read_default_namelist(nlfilename, Comm, pio_stride, pio_root,

#ifdef PIO1
call shr_pio_rearr_opts_set(Comm, pio_rearr_comm_type, pio_rearr_comm_fcd, &
pio_rearr_comm_max_pend_req, pio_rearr_comm_enable_hs, &
pio_rearr_comm_enable_isend)
pio_rearr_comm_max_pend_req_comp2io, pio_rearr_comm_enable_hs_comp2io, &
pio_rearr_comm_enable_isend_comp2io, &
pio_rearr_comm_max_pend_req_io2comp, pio_rearr_comm_enable_hs_io2comp, &
pio_rearr_comm_enable_isend_io2comp)
#endif

end subroutine shr_pio_read_default_namelist
Expand Down Expand Up @@ -680,15 +686,21 @@ end subroutine shr_pio_namelist_set
! on the root proc of comm
! The rearranger options are passed to PIO_Init() in shr_pio_init2()
subroutine shr_pio_rearr_opts_set(comm, pio_rearr_comm_type, pio_rearr_comm_fcd, &
pio_rearr_comm_max_pend_req, pio_rearr_comm_enable_hs, &
pio_rearr_comm_enable_isend)
pio_rearr_comm_max_pend_req_comp2io, pio_rearr_comm_enable_hs_comp2io, &
pio_rearr_comm_enable_isend_comp2io, &
pio_rearr_comm_max_pend_req_io2comp, pio_rearr_comm_enable_hs_io2comp, &
pio_rearr_comm_enable_isend_io2comp)
integer(SHR_KIND_IN), intent(in) :: comm
character(len=shr_kind_cs), intent(in) :: pio_rearr_comm_type, pio_rearr_comm_fcd
integer, intent(in) :: pio_rearr_comm_max_pend_req
logical, intent(in) :: pio_rearr_comm_enable_hs, pio_rearr_comm_enable_isend
integer, intent(in) :: pio_rearr_comm_max_pend_req_comp2io
logical, intent(in) :: pio_rearr_comm_enable_hs_comp2io
logical, intent(in) :: pio_rearr_comm_enable_isend_comp2io
integer, intent(in) :: pio_rearr_comm_max_pend_req_io2comp
logical, intent(in) :: pio_rearr_comm_enable_hs_io2comp
logical, intent(in) :: pio_rearr_comm_enable_isend_io2comp

character(*), parameter :: subname = '(shr_pio_rearr_opts_set) '
integer, parameter :: NUM_REARR_COMM_OPTS = 5
integer, parameter :: NUM_REARR_COMM_OPTS = 8
integer, parameter :: PIO_REARR_COMM_DEF_MAX_PEND_REQ = 64
integer(SHR_KIND_IN), dimension(NUM_REARR_COMM_OPTS) :: buf
integer :: rank, ierr
Expand All @@ -699,9 +711,12 @@ subroutine shr_pio_rearr_opts_set(comm, pio_rearr_comm_type, pio_rearr_comm_fcd,
buf = 0
! buf(1) = comm_type
! buf(2) = comm_fcd
! buf(3) = max_pend_req
! buf(4) = enable_hs
! buf(5) = enable_isend
! buf(3) = max_pend_req_comp2io
! buf(4) = enable_hs_comp2io
! buf(5) = enable_isend_comp2io
! buf(6) = max_pend_req_io2comp
! buf(7) = enable_hs_io2comp
! buf(8) = enable_isend_io2comp
if(rank == 0) then
! buf(1) = comm_type
select case(pio_rearr_comm_type)
Expand Down Expand Up @@ -733,58 +748,99 @@ subroutine shr_pio_rearr_opts_set(comm, pio_rearr_comm_type, pio_rearr_comm_fcd,
buf(2) = pio_rearr_comm_fc_2d_enable
end select

! buf(3) = max_pend_req
if((pio_rearr_comm_max_pend_req < 0) .and. &
(pio_rearr_comm_max_pend_req /= PIO_REARR_COMM_UNLIMITED_PEND_REQ)) then
write(shr_log_unit, *) "Invalid PIO rearranger comm max pend req, ", pio_rearr_comm_max_pend_req
write(shr_log_unit, *) "Resetting PIO rearranger comm max pend req to ", PIO_REARR_COMM_DEF_MAX_PEND_REQ
! buf(3) = max_pend_req_comp2io
if((pio_rearr_comm_max_pend_req_comp2io < 0) .and. &
(pio_rearr_comm_max_pend_req_comp2io /= PIO_REARR_COMM_UNLIMITED_PEND_REQ)) then
write(shr_log_unit, *) "Invalid PIO rearranger comm max pend req (comp2io), ", pio_rearr_comm_max_pend_req_comp2io
write(shr_log_unit, *) "Resetting PIO rearranger comm max pend req (comp2io) to ", PIO_REARR_COMM_DEF_MAX_PEND_REQ
buf(3) = PIO_REARR_COMM_DEF_MAX_PEND_REQ
else
buf(3) = pio_rearr_comm_max_pend_req
buf(3) = pio_rearr_comm_max_pend_req_comp2io
end if

! buf(4) = enable_hs
if(pio_rearr_comm_enable_hs) then
! buf(4) = enable_hs_comp2io
if(pio_rearr_comm_enable_hs_comp2io) then
buf(4) = 1
else
buf(4) = 0
end if

! buf(5) = enable_isend
if(pio_rearr_comm_enable_isend) then
! buf(5) = enable_isend_comp2io
if(pio_rearr_comm_enable_isend_comp2io) then
buf(5) = 1
else
buf(5) = 0
end if

! buf(6) = max_pend_req_io2comp
if((pio_rearr_comm_max_pend_req_io2comp < 0) .and. &
(pio_rearr_comm_max_pend_req_io2comp /= PIO_REARR_COMM_UNLIMITED_PEND_REQ)) then
write(shr_log_unit, *) "Invalid PIO rearranger comm max pend req (io2comp), ", pio_rearr_comm_max_pend_req_io2comp
write(shr_log_unit, *) "Resetting PIO rearranger comm max pend req (io2comp) to ", PIO_REARR_COMM_DEF_MAX_PEND_REQ
buf(6) = PIO_REARR_COMM_DEF_MAX_PEND_REQ
else
buf(6) = pio_rearr_comm_max_pend_req_io2comp
end if

! buf(7) = enable_hs_io2comp
if(pio_rearr_comm_enable_hs_io2comp) then
buf(7) = 1
else
buf(7) = 0
end if

! buf(8) = enable_isend_io2comp
if(pio_rearr_comm_enable_isend_io2comp) then
buf(8) = 1
else
buf(8) = 0
end if

! Log the rearranger options
write(shr_log_unit, *) "PIO rearranger options:"
write(shr_log_unit, *) " comm type =", pio_rearr_comm_type
write(shr_log_unit, *) " comm fcd =", pio_rearr_comm_fcd
write(shr_log_unit, *) " max pend req =", pio_rearr_comm_max_pend_req
write(shr_log_unit, *) " enable_hs =", pio_rearr_comm_enable_hs
write(shr_log_unit, *) " enable_isend =", pio_rearr_comm_enable_isend
write(shr_log_unit, *) " max pend req (comp2io) =", pio_rearr_comm_max_pend_req_comp2io
write(shr_log_unit, *) " enable_hs (comp2io) =", pio_rearr_comm_enable_hs_comp2io
write(shr_log_unit, *) " enable_isend (comp2io) =", pio_rearr_comm_enable_isend_comp2io
write(shr_log_unit, *) " max pend req (io2comp) =", pio_rearr_comm_max_pend_req_io2comp
write(shr_log_unit, *) " enable_hs (io2comp) =", pio_rearr_comm_enable_hs_io2comp
write(shr_log_unit, *) " enable_isend (io2comp) =", pio_rearr_comm_enable_isend_io2comp
end if

call shr_mpi_bcast(buf, comm)

! buf(1) = comm_type
! buf(2) = comm_fcd
! buf(3) = max_pend_req
! buf(4) = enable_hs
! buf(5) = enable_isend
! buf(3) = max_pend_req_comp2io
! buf(4) = enable_hs_comp2io
! buf(5) = enable_isend_comp2io
! buf(6) = max_pend_req_io2comp
! buf(7) = enable_hs_io2comp
! buf(8) = enable_isend_io2comp
pio_rearr_opts%comm_type = buf(1)
pio_rearr_opts%comm_fc_opts%fcd = buf(2)
pio_rearr_opts%comm_fc_opts%max_pend_req = buf(3)
pio_rearr_opts%fcd = buf(2)
pio_rearr_opts%comm_fc_opts_comp2io%max_pend_req = buf(3)
if(buf(4) == 0) then
pio_rearr_opts%comm_fc_opts%enable_hs = .false.
pio_rearr_opts%comm_fc_opts_comp2io%enable_hs = .false.
else
pio_rearr_opts%comm_fc_opts%enable_hs = .true.
pio_rearr_opts%comm_fc_opts_comp2io%enable_hs = .true.
end if
if(buf(5) == 0) then
pio_rearr_opts%comm_fc_opts%enable_isend = .false.
pio_rearr_opts%comm_fc_opts_comp2io%enable_isend = .false.
else
pio_rearr_opts%comm_fc_opts_comp2io%enable_isend = .true.
end if
pio_rearr_opts%comm_fc_opts_io2comp%max_pend_req = buf(6)
if(buf(7) == 0) then
pio_rearr_opts%comm_fc_opts_io2comp%enable_hs = .false.
else
pio_rearr_opts%comm_fc_opts_io2comp%enable_hs = .true.
end if
if(buf(8) == 0) then
pio_rearr_opts%comm_fc_opts_io2comp%enable_isend = .false.
else
pio_rearr_opts%comm_fc_opts%enable_isend = .true.
pio_rearr_opts%comm_fc_opts_io2comp%enable_isend = .true.
end if
end subroutine
#endif
Expand Down

0 comments on commit 2e9be53

Please sign in to comment.