Changing PIO1 flow control logic for io2comp and comp2io

Changing the defaults of some PIO1 flow control runtime
parameters. These defaults were identified during high
res performance runs (by Pat Worley).

For io2comp, the handshaking protocol is very expensive (and
otherwise unneeded) for large process counts. The handshaking
protocol is however very important for comp2io. This
modification disables handshaking for io2comp when FLOW_CONTROL
is specified, to prevent this bad choice from being enforced.

Specifying no limit on the number of preposted receive requests for
comp2io is also potentially a problem. (It is less of an issue for
io2comp, so is not addressed here.) While this can be avoided in
env_run.xml, this modification also adds logic to change the
'no limit' setting for comp2io to

      pio_maxreq = max(2*IOsystem%num_iotasks,DEF_P2P_MAXREQ)

as a small multiple of IOsystem%num_iotasks has proven to perform
well empirically. We can improve this further by communicating
between iotasks first, and then non-iotasks to iotasks, but
that is a future task).

driver_cpl/cime_config/config_component.xml

@@ -2420,10 +2420,10 @@
   <entry id="PIO_REARR_COMM_MAX_PEND_REQ_COMP2IO">
     <type>integer</type>
     <valid_values></valid_values>
-    <default_value>64</default_value>
+    <default_value>0</default_value>
     <group>run_pio</group>
     <file>env_run.xml</file>
-    <desc>pio rearranger communication max pending requests (comp2io) </desc>
+    <desc>pio rearranger communication max pending requests (comp2io) : 0 implies that CIME internally calculates the value ( = max(64, 2 * PIO_NUMTASKS) ), -1 implies no bound on max pending requests </desc>
   </entry>
 
   <entry id="PIO_REARR_COMM_ENABLE_HS_COMP2IO">
@@ -2450,13 +2450,15 @@
     <default_value>64</default_value>
     <group>run_pio</group>
     <file>env_run.xml</file>
-    <desc>pio rearranger communication max pending requests (io2comp)</desc>
+    <desc>pio rearranger communication max pending requests (io2comp) : -1 implies no bound on max pending requests </desc>
   </entry>
 
+  <!-- Handshaking is a very bad idea for large process counts and for -->
+  <!-- io2comp (but important for comp2io -->
   <entry id="PIO_REARR_COMM_ENABLE_HS_IO2COMP">
     <type>logical</type>
     <valid_values>TRUE,FALSE</valid_values>
-    <default_value>TRUE</default_value>
+    <default_value>FALSE</default_value>
     <group>run_pio</group>
     <file>env_run.xml</file>
     <desc>pio rearranger communiation options (io2comp) : TRUE implies enable handshake</desc>
@@ -2465,7 +2467,7 @@
   <entry id="PIO_REARR_COMM_ENABLE_ISEND_IO2COMP">
     <type>logical</type>
     <valid_values>TRUE,FALSE</valid_values>
-    <default_value>FALSE</default_value>
+    <default_value>TRUE</default_value>
     <group>run_pio</group>
     <file>env_run.xml</file>
     <desc>pio rearranger communiation options (io2comp) : TRUE implies enable isend</desc>

share/csm_share/shr/shr_pio_mod.F90

@@ -472,7 +472,7 @@ subroutine shr_pio_read_default_namelist(nlfilename, Comm, pio_stride, pio_root,
            pio_rearr_comm_max_pend_req_comp2io, pio_rearr_comm_enable_hs_comp2io, &
            pio_rearr_comm_enable_isend_comp2io, &
            pio_rearr_comm_max_pend_req_io2comp, pio_rearr_comm_enable_hs_io2comp, &
-           pio_rearr_comm_enable_isend_io2comp)
+           pio_rearr_comm_enable_isend_io2comp, pio_numiotasks)
 #endif
 
   end subroutine shr_pio_read_default_namelist
@@ -689,7 +689,8 @@ subroutine shr_pio_rearr_opts_set(comm, pio_rearr_comm_type, pio_rearr_comm_fcd,
           pio_rearr_comm_max_pend_req_comp2io, pio_rearr_comm_enable_hs_comp2io, &
           pio_rearr_comm_enable_isend_comp2io, &
           pio_rearr_comm_max_pend_req_io2comp, pio_rearr_comm_enable_hs_io2comp, &
-          pio_rearr_comm_enable_isend_io2comp)
+          pio_rearr_comm_enable_isend_io2comp, &
+          pio_numiotasks)
     integer(SHR_KIND_IN), intent(in) :: comm
     character(len=shr_kind_cs), intent(in) :: pio_rearr_comm_type, pio_rearr_comm_fcd
     integer, intent(in) :: pio_rearr_comm_max_pend_req_comp2io
@@ -698,6 +699,7 @@ subroutine shr_pio_rearr_opts_set(comm, pio_rearr_comm_type, pio_rearr_comm_fcd,
     integer, intent(in) :: pio_rearr_comm_max_pend_req_io2comp
     logical, intent(in) :: pio_rearr_comm_enable_hs_io2comp
     logical, intent(in) :: pio_rearr_comm_enable_isend_io2comp
+    integer, intent(in) :: pio_numiotasks
 
     character(*), parameter :: subname = '(shr_pio_rearr_opts_set) '
     integer, parameter :: NUM_REARR_COMM_OPTS = 8
@@ -749,11 +751,17 @@ subroutine shr_pio_rearr_opts_set(comm, pio_rearr_comm_type, pio_rearr_comm_fcd,
       end select
 
       ! buf(3) = max_pend_req_comp2io
-      if((pio_rearr_comm_max_pend_req_comp2io < 0) .and. &
+      if((pio_rearr_comm_max_pend_req_comp2io <= 0) .and. &
           (pio_rearr_comm_max_pend_req_comp2io /= PIO_REARR_COMM_UNLIMITED_PEND_REQ)) then
+
+        ! Small multiple of pio_numiotasks has proven to perform
+        ! well empirically, and we do not want to allow maximum for
+        ! very large process count runs. Can improve this by
+        ! communicating between iotasks first, and then non-iotasks
+        ! to iotasks (TO DO)
         write(shr_log_unit, *) "Invalid PIO rearranger comm max pend req (comp2io), ", pio_rearr_comm_max_pend_req_comp2io
-        write(shr_log_unit, *) "Resetting PIO rearranger comm max pend req (comp2io) to ", PIO_REARR_COMM_DEF_MAX_PEND_REQ
-        buf(3) = PIO_REARR_COMM_DEF_MAX_PEND_REQ
+        write(shr_log_unit, *) "Resetting PIO rearranger comm max pend req (comp2io) to ", max(PIO_REARR_COMM_DEF_MAX_PEND_REQ, 2 * pio_numiotasks)
+        buf(3) = max(PIO_REARR_COMM_DEF_MAX_PEND_REQ, 2 * pio_numiotasks)
       else
         buf(3) = pio_rearr_comm_max_pend_req_comp2io
       end if
@@ -773,7 +781,7 @@ subroutine shr_pio_rearr_opts_set(comm, pio_rearr_comm_type, pio_rearr_comm_fcd,
       end if
 
       ! buf(6) = max_pend_req_io2comp
-      if((pio_rearr_comm_max_pend_req_io2comp < 0) .and. &
+      if((pio_rearr_comm_max_pend_req_io2comp <= 0) .and. &
           (pio_rearr_comm_max_pend_req_io2comp /= PIO_REARR_COMM_UNLIMITED_PEND_REQ)) then
         write(shr_log_unit, *) "Invalid PIO rearranger comm max pend req (io2comp), ", pio_rearr_comm_max_pend_req_io2comp
         write(shr_log_unit, *) "Resetting PIO rearranger comm max pend req (io2comp) to ", PIO_REARR_COMM_DEF_MAX_PEND_REQ