Skip to content

Commit

Permalink
Merge branch 'jayeshkrishna/cime/pio/pio_rearranger_opts_port' into n…
Browse files Browse the repository at this point in the history 
…ext (PR #410)

Port PIO rearranger options to CIME

Porting PIO rearranger options from pre-CIME to CIME.
These changes that were added to PIO in ACME were
not available in the version of PIO we adopted in CIME.
So porting the code to CIME.

This change adds run-time options to control the PIO
rearranger communication options. The user can now control
the PIO rearranger options using the newly added options in
env_run.xml. The defaults are set based on the compile-time
settings of these parameters.

The user can control the rearranger communication type (point
to point, collective), rearranger flow control direction (flow control
between io and compute processes), maximum pending requests
in the flow control algorithm, whether to enable handshake in the
flow control algorithm and whether to enable isends in the flow
control algorithm. The defaults for these values are the same as
before.

Also see PR #159 and SEG-57

[BFB]
[NML]

SEG-180

* jayeshkrishna/cime/pio/pio_rearranger_opts_port:
  Removing some comments - No code change
  Simplifying the initialization of PIO rearranger options
  Allow rearranging data with a collective without any compile-time flags
  Removing some FIXMEs - No code change
  Setting the PIO rearranger options at runtime
  Moving PIO rearranger options to runtime
  • Loading branch information
Patrick Worley committed Dec 14, 2015
2 parents be091d6 + c3f245b commit 77616d2
Show file tree
Hide file tree
Showing 7 changed files with 359 additions and 90 deletions.
10 changes: 5 additions & 5 deletions cime/driver_cpl/bld/build-namelist
View file
Original file line number Diff line number Diff line change
Expand Up @@ -707,11 +707,11 @@ add_default($nl, 'pio_typename');
add_default($nl, 'pio_debug_level');
add_default($nl, 'pio_blocksize');
add_default($nl, 'pio_buffer_size_limit');
#pw add_default($nl, 'pio_rearr_comm_type', 'PIO_REARR_COMM_TYPE'=>"$xmlvars{'PIO_REARR_COMM_TYPE'}");
#pw add_default($nl, 'pio_rearr_comm_fcd', 'PIO_REARR_COMM_FCD'=>"$xmlvars{'PIO_REARR_COMM_FCD'}");
#pw add_default($nl, 'pio_rearr_comm_max_pend_req', 'PIO_REARR_COMM_MAX_PEND_REQ'=>"$xmlvars{'PIO_REARR_COMM_MAX_PEND_REQ'}");
#pw add_default($nl, 'pio_rearr_comm_enable_hs', 'PIO_REARR_COMM_ENABLE_HS'=>"$xmlvars{'PIO_REARR_COMM_ENABLE_HS'}");
#pw add_default($nl, 'pio_rearr_comm_enable_isend', 'PIO_REARR_COMM_ENABLE_ISEND'=>"$xmlvars{'PIO_REARR_COMM_ENABLE_ISEND'}");
add_default($nl, 'pio_rearr_comm_type', 'PIO_REARR_COMM_TYPE'=>"$xmlvars{'PIO_REARR_COMM_TYPE'}");
add_default($nl, 'pio_rearr_comm_fcd', 'PIO_REARR_COMM_FCD'=>"$xmlvars{'PIO_REARR_COMM_FCD'}");
add_default($nl, 'pio_rearr_comm_max_pend_req', 'PIO_REARR_COMM_MAX_PEND_REQ'=>"$xmlvars{'PIO_REARR_COMM_MAX_PEND_REQ'}");
add_default($nl, 'pio_rearr_comm_enable_hs', 'PIO_REARR_COMM_ENABLE_HS'=>"$xmlvars{'PIO_REARR_COMM_ENABLE_HS'}");
add_default($nl, 'pio_rearr_comm_enable_isend', 'PIO_REARR_COMM_ENABLE_ISEND'=>"$xmlvars{'PIO_REARR_COMM_ENABLE_ISEND'}");

# Note: pio_async_interface=.true. is not yet supported
# If pio_async_interface is .true. or {component}_PIO_* variable is not set or set to -99
Expand Down
111 changes: 55 additions & 56 deletions cime/externals/pio/pio/box_rearrange.F90.in
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,8 @@
!>
!!
!! @file
!! $Revision$
!! $LastChangedDate$
!! $Revision: 819 $
!! $LastChangedDate: 2013-05-31 13:32:27 -0500 (Fri, 31 May 2013) $
!! @brief
!! Perform data rearrangement with each io processor
!! owning a rectangular box in the output domain
Expand All @@ -29,39 +29,7 @@
! cache communication pattern for rearranger in the ioDesc
#define BOX_CACHE 1


! communication algorithm options
#define COLLECTIVE 0
#define POINT_TO_POINT 1
#define FLOW_CONTROL 2
#define DEF_P2P_HANDSHAKE .true.
#define DEF_P2P_ISEND .false.
#define DEF_P2P_MAXREQ 64

#ifndef _MPISERIAL
#ifndef _NO_FLOW_CONTROL
#define _USE_COMP2IO_FC 1
#define _USE_IO2COMP_FC 1
#endif
#endif
!
! The completely unreadable nature of the following lines is required by some compilers
!
#ifdef _USE_ALLTOALLW
#define DEF_COMP2IO_OPTION 0
#define DEF_IO2COMP_OPTION 0
#else
#ifdef _USE_COMP2IO_FC
#define DEF_COMP2IO_OPTION 2
#else
#define DEF_COMP2IO_OPTION 1
#endif
#ifdef _USE_IO2COMP_FC
#define DEF_IO2COMP_OPTION 2
#else
#define DEF_IO2COMP_OPTION 1
#endif
#endif
#include "rearr_options.h"

!>
!! \def TAG
Expand All @@ -74,7 +42,7 @@
module box_rearrange

use pio_kinds, only : pio_offset, r4, r8, i4, i8
use pio_types, only : io_desc_t, iosystem_desc_t
use pio_types
#ifdef NO_MPI2
use pio_support, only : MPI_TYPE_CREATE_INDEXED_BLOCK, piodie, &
Debug, DebugIO, CheckMPIReturn, pio_fc_gather_offset
Expand Down Expand Up @@ -212,20 +180,39 @@ subroutine box_rearrange_comp2io_{TYPE} (IOsystem, ioDesc, s1, src, niodof, &
#else


! begin
! FIXME: Ideally the iodesc should contain the rearr and
! the rearr_opts - however the current code sets the
! rearr and rearr_opts on the IOsystem

! The rearranger options in IODESC overrides the defaults
if(IOsystem%rearr_opts%comm_type == PIO_rearr_comm_p2p) then
if( (IOsystem%rearr_opts%comm_fc_opts%fcd == PIO_rearr_comm_fc_2d_disable) .or.&
(IOsystem%rearr_opts%comm_fc_opts%fcd == PIO_rearr_comm_fc_1d_io2comp) ) then
pio_option = POINT_TO_POINT
else
pio_option = FLOW_CONTROL
end if
else
pio_option = COLLECTIVE
end if

! The rearranger options passed in to this function overrides
! both the defaults and the options in IODESC
if ( present( comm_option ) ) then
if ((comm_option == COLLECTIVE) &
.or. (comm_option == POINT_TO_POINT) &
.or. (comm_option == FLOW_CONTROL)) then
pio_option = comm_option
endif
else
pio_option = DEF_COMP2IO_OPTION
endif

if (pio_option == FLOW_CONTROL) then
pio_hs = DEF_P2P_HANDSHAKE
pio_isend = DEF_P2P_ISEND
pio_maxreq = DEF_P2P_MAXREQ
pio_hs = IOsystem%rearr_opts%comm_fc_opts%enable_hs
pio_isend = IOsystem%rearr_opts%comm_fc_opts%enable_isend
pio_maxreq = IOsystem%rearr_opts%comm_fc_opts%max_pend_req

! The rearranger options passed to this function overrides
! both the default and the options in IODESC
if ( present(fc_options) ) then
if (fc_options(1) == 0) then
pio_hs = .false.
Expand Down Expand Up @@ -293,22 +280,18 @@ subroutine box_rearrange_comp2io_{TYPE} (IOsystem, ioDesc, s1, src, niodof, &
endif
end do

#ifdef _USE_ALLTOALLW
if (pio_option == COLLECTIVE) then

call MPI_ALLTOALLW(src, a2a_sendcounts, a2a_displs, a2a_sendtypes, &
dest, a2a_recvcounts, a2a_displs, a2a_recvtypes, &
IOsystem%union_comm, ierror )
call CheckMPIReturn('box_rearrange', ierror)
else
#endif
call pio_swapm( nprocs, myrank, &
src, ndof, a2a_sendcounts, a2a_displs, a2a_sendtypes, &
dest, niodof, a2a_recvcounts, a2a_displs, a2a_recvtypes, &
IOsystem%union_comm, pio_hs, pio_isend, pio_maxreq )
#ifdef _USE_ALLTOALLW
endif
#endif
call dealloc_check(a2a_sendcounts)
call dealloc_check(a2a_displs)
call dealloc_check(a2a_sendtypes)
Expand Down Expand Up @@ -477,19 +460,39 @@ subroutine box_rearrange_io2comp_{TYPE} (IOsystem,ioDesc,s1, iobuf,s2, compbuf,
end do

#else
! FIXME: Ideally the iodesc should contain the rearr and
! the rearr_opts - however the current code sets the
! rearr and rearr_opts on the IOsystem

! The rearranger options in IODESC overrides the defaults
if(IOsystem%rearr_opts%comm_type == PIO_rearr_comm_p2p) then
if( (IOsystem%rearr_opts%comm_fc_opts%fcd == PIO_rearr_comm_fc_2d_disable) .or.&
(IOsystem%rearr_opts%comm_fc_opts%fcd == PIO_rearr_comm_fc_1d_comp2io) ) then
pio_option = POINT_TO_POINT
else
pio_option = FLOW_CONTROL
end if
else
pio_option = COLLECTIVE
end if

! The rearranger options passed in to this function overrides
! both the defaults and the options in IODESC
if ( present( comm_option ) ) then
if ((comm_option == COLLECTIVE) &
.or. (comm_option == POINT_TO_POINT) &
.or. (comm_option == FLOW_CONTROL)) then
pio_option = comm_option
endif
else
pio_option = DEF_IO2COMP_OPTION
endif

if (pio_option == FLOW_CONTROL) then
pio_hs = DEF_P2P_HANDSHAKE
pio_isend = DEF_P2P_ISEND
pio_maxreq = DEF_P2P_MAXREQ
pio_hs = IOsystem%rearr_opts%comm_fc_opts%enable_hs
pio_isend = IOsystem%rearr_opts%comm_fc_opts%enable_isend
pio_maxreq = IOsystem%rearr_opts%comm_fc_opts%max_pend_req

! The rearranger options passed to this function overrides
! both the default and the options in IODESC
if ( present(fc_options) ) then
if (fc_options(1) == 0) then
pio_hs = .false.
Expand Down Expand Up @@ -560,21 +563,17 @@ subroutine box_rearrange_io2comp_{TYPE} (IOsystem,ioDesc,s1, iobuf,s2, compbuf,
end do
endif

#ifdef _USE_ALLTOALLW
if (pio_option == COLLECTIVE) then
call MPI_ALLTOALLW(iobuf, a2a_sendcounts, a2a_displs, a2a_sendtypes, &
compbuf, a2a_recvcounts, a2a_displs, a2a_recvtypes, &
IOsystem%union_comm, ierror )
call CheckMPIReturn(subName, ierror)
else
#endif
call pio_swapm( nprocs, myrank, &
iobuf, niodof, a2a_sendcounts, a2a_displs, a2a_sendtypes, &
compbuf, ndof, a2a_recvcounts, a2a_displs, a2a_recvtypes, &
IOsystem%union_comm, pio_hs, pio_isend, pio_maxreq )
#ifdef _USE_ALLTOALLW
endif
#endif
call dealloc_check(a2a_sendcounts)
call dealloc_check(a2a_displs)
call dealloc_check(a2a_sendtypes)
Expand Down Expand Up @@ -1336,7 +1335,7 @@ end subroutine box_rearrange_io2comp_{TYPE}
do i=1,nrecvs
recv_counts(rfrom(i)+1) = rcount(i)
enddo

rbuf_size = sum(recv_counts)
call alloc_check(rindex, rbuf_size, 'rindex buffer')
rindex = 0
Expand Down
14 changes: 10 additions & 4 deletions cime/externals/pio/pio/pio.F90
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,8 @@
!! @file
!! @brief User interface Module for PIO, this is the only file a user program should 'use'
!!
!! $Revision$
!! $LastChangedDate$
!! $Revision: 856 $
!! $LastChangedDate: 2013-11-19 15:48:54 -0600 (Tue, 19 Nov 2013) $
!<

module pio
Expand All @@ -19,7 +19,11 @@ module pio
pio_dupiodesc, pio_finalize, pio_set_hint, pio_getnumiotasks, pio_file_is_open, &
pio_setnum_OST, pio_getnum_OST

use pio_types, only : io_desc_t, file_desc_t, var_desc_t, iosystem_desc_t, &
use pio_types, only : io_desc_t, file_desc_t, var_desc_t, iosystem_desc_t,&
pio_rearr_opt_t, pio_rearr_comm_fc_opt_t, pio_rearr_comm_fc_2d_enable,&
pio_rearr_comm_fc_1d_comp2io, pio_rearr_comm_fc_1d_io2comp,&
pio_rearr_comm_fc_2d_disable, pio_rearr_comm_unlimited_pend_req,&
pio_rearr_comm_p2p, pio_rearr_comm_coll,&
pio_int, pio_real, pio_double, pio_noerr, iotype_netcdf, &
iotype_pnetcdf, iotype_binary, iotype_direct_pbinary, iotype_pbinary, &
PIO_iotype_binary, PIO_iotype_direct_pbinary, PIO_iotype_pbinary, &
Expand Down Expand Up @@ -63,12 +67,14 @@ module pio
use pionfget_mod, only : PIO_get_var => get_var

use calcdecomp, only : pio_set_blocksize



implicit none
public
! Added for pio2 compatability
integer, parameter :: pio_offset_kind = pio_offset
integer, parameter :: pio_rearr_subset = pio_rearr_box

contains
function pio_iam_iotask(iosystem) result(task)
type(iosystem_desc_t), intent(in) :: iosystem
Expand Down
90 changes: 78 additions & 12 deletions cime/externals/pio/pio/pio_types.F90
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,8 @@
!! @file
!! @brief Derived datatypes and constants for PIO
!!
!! $Revision$
!! $LastChangedDate$
!! $Revision: 943 $
!! $LastChangedDate: 2014-02-14 10:20:17 -0600 (Fri, 14 Feb 2014) $
!<
module pio_types
use pio_kinds
Expand Down Expand Up @@ -34,6 +34,80 @@ module pio_types
integer(i4) :: length
end type

!>
!! @defgroup PIO_rearr_method PIO_rearr_method
!! @public
!! @brief The three choices to control rearrangement are:
!! @details
!! - PIO_rearr_none : Do not use any form of rearrangement
!! - PIO_rearr_box : Use a PIO internal box rearrangement
!>
integer(i4), public, parameter :: PIO_rearr_none = 0
integer(i4), public, parameter :: PIO_rearr_box = 1

!>
!! @defgroup PIO_rearr_comm_t PIO_rearr_comm_t
!! @public
!! @brief The two choices for rearranger communication
!! @details
!! - PIO_rearr_comm_p2p : Point to point
!! - PIO_rearr_comm_coll : Collective
!>
enum, bind(c)
enumerator :: PIO_rearr_comm_p2p = 0
enumerator :: PIO_rearr_comm_coll
end enum

!>
!! @defgroup PIO_rearr_comm_dir PIO_rearr_comm_dir
!! @public
!! @brief The four choices for rearranger communication direction
!! @details
!! - PIO_rearr_comm_fc_2d_enable : COMM procs to IO procs and vice versa
!! - PIO_rearr_comm_fc_1d_comp2io: COMM procs to IO procs only
!! - PIO_rearr_comm_fc_1d_io2comp: IO procs to COMM procs only
!! - PIO_rearr_comm_fc_2d_disable: Disable flow control
!>
enum, bind(c)
enumerator :: PIO_rearr_comm_fc_2d_enable = 0
enumerator :: PIO_rearr_comm_fc_1d_comp2io
enumerator :: PIO_rearr_comm_fc_1d_io2comp
enumerator :: PIO_rearr_comm_fc_2d_disable
end enum

!>
!! @defgroup PIO_rearr_comm_fc_options PIO_rearr_comm_fc_options
!! @brief Type that defines the PIO rearranger options
!! @details
!! - fcd : @copydoc PIO_rearr_comm_dir
!! - enable_hs : Enable handshake (true/false)
!! - enable_isend : Enable Isends (true/false)
!! - max_pend_req : Maximum pending requests (To indicated unlimited
!! number of requests use PIO_REARR_COMM_UNLIMITED_PEND_REQ)
!>
type, public :: PIO_rearr_comm_fc_opt_t
integer :: fcd ! Flow control direction
logical :: enable_hs ! Enable handshake?
logical :: enable_isend ! Enable isends?
integer :: max_pend_req ! Maximum pending requests
end type PIO_rearr_comm_fc_opt_t

integer, public, parameter :: PIO_REARR_COMM_UNLIMITED_PEND_REQ = -1
!>
!! @defgroup PIO_rearr_options PIO_rearr_options
!! @brief Type that defines the PIO rearranger options
!! @details
!! - comm_type : @copydoc PIO_rearr_comm_t
!! - comm_fc_opts : @copydoc PIO_rearr_comm_fc_options
!>
type, public :: PIO_rearr_opt_t
integer :: comm_type
type(PIO_rearr_comm_fc_opt_t) :: comm_fc_opts
end type PIO_rearr_opt_t

public :: PIO_rearr_comm_p2p, PIO_rearr_comm_coll,&
PIO_rearr_comm_fc_2d_enable, PIO_rearr_comm_fc_1d_comp2io,&
PIO_rearr_comm_fc_1d_io2comp, PIO_rearr_comm_fc_2d_disable

!------------------------------------
! a file descriptor data structure
Expand Down Expand Up @@ -79,6 +153,8 @@ module pio_types
logical(log_kind) :: async_interface=.false. ! .true. if using the async interface model
integer(i4) :: rearr ! type of rearranger
! e.g. rearr_{none,box}
!integer(i4), dimension(IOSYS_REARR_OPT_MAX) :: rearr_opts ! Rearranger options - see PIO_rearr_opt_t for details
type(PIO_rearr_opt_t) :: rearr_opts ! Rearranger options
integer(i4) :: error_handling ! how pio handles errors
integer(i4),pointer :: ioranks(:) => null() ! the computational ranks for the IO tasks

Expand Down Expand Up @@ -242,16 +318,6 @@ module pio_types
iotype_netcdf = PIO_iotype_netcdf


!>
!! @defgroup PIO_rearr_method PIO_rearr_method
!! @public
!! @brief The three choices to control rearrangement are:
!! @details
!! - PIO_rearr_none : Do not use any form of rearrangement
!! - PIO_rearr_box : Use a PIO internal box rearrangement
!>
integer(i4), public, parameter :: PIO_rearr_none = 0
integer(i4), public, parameter :: PIO_rearr_box = 1

!>
!! @public
Expand Down
Loading

0 comments on commit 77616d2

Please sign in to comment.