Adding more runtime rearranger options

This commit allows the user to separately control the flow
control options for comp2io and io2comp directions.

* The flow control options are now separately defined for
  comp2io direction and io2comp direction

* The rearranger options structure (passed during PIO_init())
  is also updated accordingly

Note: This change requires corresponding changes in the
share code (in CIME) due to changes in the rearranger opt
structure

pio/box_rearrange.F90.in

@@ -189,8 +189,8 @@ subroutine box_rearrange_comp2io_{TYPE} (IOsystem, ioDesc, s1, src, niodof, &
 
   ! The rearranger options in IODESC overrides the defaults
   if(IOsystem%rearr_opts%comm_type == PIO_rearr_comm_p2p) then
-    if( (IOsystem%rearr_opts%comm_fc_opts%fcd == PIO_rearr_comm_fc_2d_disable) .or.&
-        (IOsystem%rearr_opts%comm_fc_opts%fcd == PIO_rearr_comm_fc_1d_io2comp) ) then
+    if( (IOsystem%rearr_opts%fcd == PIO_rearr_comm_fc_2d_disable) .or.&
+        (IOsystem%rearr_opts%fcd == PIO_rearr_comm_fc_1d_io2comp) ) then
       pio_option = POINT_TO_POINT
     else
       pio_option = FLOW_CONTROL
@@ -210,9 +210,9 @@ subroutine box_rearrange_comp2io_{TYPE} (IOsystem, ioDesc, s1, src, niodof, &
   endif
 
   if (pio_option == FLOW_CONTROL) then
-    pio_hs     = IOsystem%rearr_opts%comm_fc_opts%enable_hs
-    pio_isend  = IOsystem%rearr_opts%comm_fc_opts%enable_isend
-    pio_maxreq = IOsystem%rearr_opts%comm_fc_opts%max_pend_req
+    pio_hs     = IOsystem%rearr_opts%comm_fc_opts_comp2io%enable_hs
+    pio_isend  = IOsystem%rearr_opts%comm_fc_opts_comp2io%enable_isend
+    pio_maxreq = IOsystem%rearr_opts%comm_fc_opts_comp2io%max_pend_req
 
     ! The rearranger options passed to this function overrides
     ! both the default and the options in IODESC
@@ -488,8 +488,8 @@ subroutine box_rearrange_io2comp_{TYPE} (IOsystem,ioDesc,s1, iobuf,s2, compbuf,
 
   ! The rearranger options in IODESC overrides the defaults
   if(IOsystem%rearr_opts%comm_type == PIO_rearr_comm_p2p) then
-    if( (IOsystem%rearr_opts%comm_fc_opts%fcd == PIO_rearr_comm_fc_2d_disable) .or.&
-        (IOsystem%rearr_opts%comm_fc_opts%fcd == PIO_rearr_comm_fc_1d_comp2io) ) then
+    if( (IOsystem%rearr_opts%fcd == PIO_rearr_comm_fc_2d_disable) .or.&
+        (IOsystem%rearr_opts%fcd == PIO_rearr_comm_fc_1d_comp2io) ) then
       pio_option = POINT_TO_POINT
     else
       pio_option = FLOW_CONTROL
@@ -509,9 +509,9 @@ subroutine box_rearrange_io2comp_{TYPE} (IOsystem,ioDesc,s1, iobuf,s2, compbuf,
   endif
 
   if (pio_option == FLOW_CONTROL) then
-    pio_hs     = IOsystem%rearr_opts%comm_fc_opts%enable_hs
-    pio_isend  = IOsystem%rearr_opts%comm_fc_opts%enable_isend
-    pio_maxreq = IOsystem%rearr_opts%comm_fc_opts%max_pend_req
+    pio_hs     = IOsystem%rearr_opts%comm_fc_opts_io2comp%enable_hs
+    pio_isend  = IOsystem%rearr_opts%comm_fc_opts_io2comp%enable_isend
+    pio_maxreq = IOsystem%rearr_opts%comm_fc_opts_io2comp%max_pend_req
 
     ! The rearranger options passed to this function overrides
     ! both the default and the options in IODESC

pio/pio_types.F90

@@ -79,14 +79,12 @@ module pio_types
 !! @defgroup PIO_rearr_comm_fc_options PIO_rearr_comm_fc_options
 !! @brief Type that defines the PIO rearranger options
 !! @details
-!!  - fcd : @copydoc PIO_rearr_comm_dir
 !!  - enable_hs : Enable handshake (true/false) 
 !!  - enable_isend : Enable Isends (true/false)
 !!  - max_pend_req : Maximum pending requests (To indicated unlimited
 !!                    number of requests use PIO_REARR_COMM_UNLIMITED_PEND_REQ)
 !>
     type, public :: PIO_rearr_comm_fc_opt_t
-      integer :: fcd                  ! Flow control direction
       logical :: enable_hs            ! Enable handshake?
       logical :: enable_isend         ! Enable isends?
       integer :: max_pend_req         ! Maximum pending requests
@@ -98,11 +96,14 @@ module pio_types
 !! @brief Type that defines the PIO rearranger options
 !! @details
 !!  - comm_type : @copydoc PIO_rearr_comm_t
+!!  - fcd : @copydoc PIO_rearr_comm_dir
 !!  - comm_fc_opts : @copydoc PIO_rearr_comm_fc_options
 !>
     type, public :: PIO_rearr_opt_t
       integer                         :: comm_type
-      type(PIO_rearr_comm_fc_opt_t)   :: comm_fc_opts
+      integer                         :: fcd       ! Flow control direction
+      type(PIO_rearr_comm_fc_opt_t)   :: comm_fc_opts_comp2io
+      type(PIO_rearr_comm_fc_opt_t)   :: comm_fc_opts_io2comp
     end type PIO_rearr_opt_t
 
     public :: PIO_rearr_comm_p2p, PIO_rearr_comm_coll,&

pio/piolib_mod.F90

@@ -1600,21 +1600,25 @@ subroutine init_iosystem_rearr_options(iosystem)
 #endif
 
 #ifdef _NO_FLOW_CONTROL
-    iosystem%rearr_opts%comm_fc_opts%fcd = PIO_rearr_comm_fc_2d_disable
+    iosystem%rearr_opts%fcd = PIO_rearr_comm_fc_2d_disable
 #else
     ! We ignore the following flags 
     ! 1) _MPISERIAL : The flow control code is never used when _MPISERIAL is set
     ! 2) _USE_COMP2IO_FC/_USE_IO2COMP_FC : These flags are not currently used
     !  (These were experimental flags). The user can explicitly control
     !  these options (comp2io and io2comp flow control) via rearranger
     !  options passed to pio_init()
-    iosystem%rearr_opts%comm_fc_opts%fcd = PIO_rearr_comm_fc_2d_enable
+    iosystem%rearr_opts%fcd = PIO_rearr_comm_fc_2d_enable
 #endif
 
     ! the following will be ignored if not p2p with flow control
-    iosystem%rearr_opts%comm_fc_opts%enable_hs = DEF_P2P_HANDSHAKE
-    iosystem%rearr_opts%comm_fc_opts%enable_isend = DEF_P2P_ISEND
-    iosystem%rearr_opts%comm_fc_opts%max_pend_req = DEF_P2P_MAXREQ
+    iosystem%rearr_opts%comm_fc_opts_comp2io%enable_hs = DEF_P2P_HANDSHAKE
+    iosystem%rearr_opts%comm_fc_opts_comp2io%enable_isend = DEF_P2P_ISEND
+    iosystem%rearr_opts%comm_fc_opts_comp2io%max_pend_req = DEF_P2P_MAXREQ
+
+    iosystem%rearr_opts%comm_fc_opts_io2comp%enable_hs = DEF_P2P_HANDSHAKE
+    iosystem%rearr_opts%comm_fc_opts_io2comp%enable_isend = DEF_P2P_ISEND
+    iosystem%rearr_opts%comm_fc_opts_io2comp%max_pend_req = DEF_P2P_MAXREQ
 
   end subroutine init_iosystem_rearr_options