Skip to content

Commit

Permalink
ESMF managed threading (#1018)
Browse files Browse the repository at this point in the history 
Add code updates to enable the ESMF managed threading in UFS.


Co-authored-by: Gerhard Theurich <gerhard.j.theurich@noaa.gov >
  • Loading branch information
@junwang-noaa @theurich
junwang-noaa and theurich authored Jan 27, 2022
1 parent 4604d70 commit bce0ff3
Show file tree
Hide file tree
Showing 37 changed files with 3,788 additions and 3,986 deletions.
2 changes: 1 addition & 1 deletion FV3
Submodule FV3 updated 1 files
+26 −1 fv3_cap.F90
2 changes: 1 addition & 1 deletion WW3
Submodule WW3 updated 1 files
+29 −1 model/src/wmesmfmd.F90
105 changes: 96 additions & 9 deletions driver/EARTH_GRID_COMP.F90
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,18 +54,21 @@ MODULE module_EARTH_GRID_COMP
use FRONT_HYCOM, only: HYCOM_SS => SetServices
#endif
#ifdef FRONT_MOM6
use FRONT_MOM6, only: MOM6_SS => SetServices
use FRONT_MOM6, only: MOM6_SS => SetServices, &
MOM6_SV => SetVM
#endif
#ifdef FRONT_CDEPS_DOCN
use FRONT_CDEPS_DOCN, only: DOCN_SS => SetServices
#endif
! - Handle build time ICE options:
#ifdef FRONT_CICE6
use FRONT_CICE6, only: CICE6_SS => SetServices
use FRONT_CICE6, only: CICE6_SS => SetServices, &
CICE6_SV => SetVM
#endif
! - Handle build time WAV options:
#ifdef FRONT_WW3
use FRONT_WW3, only: WW3_SS => SetServices
use FRONT_WW3, only: WW3_SS => SetServices, &
WW3_SV => SetVM
#endif
! - Handle build time LND options:
#ifdef FRONT_NOAH
Expand All @@ -84,7 +87,8 @@ MODULE module_EARTH_GRID_COMP
#endif
! - Mediator
#ifdef FRONT_CMEPS
use MED, only: MED_SS => SetServices
use MED, only: MED_SS => SetServices, &
MED_SV => SetVM
#endif
!
!-----------------------------------------------------------------------
Expand Down Expand Up @@ -226,12 +230,14 @@ subroutine SetModelServices(driver, rc)
character(ESMF_MAXSTR) :: name
type(ESMF_GridComp) :: comp
type(ESMF_Config) :: config
type(ESMF_Info) :: info
character(len=32), allocatable :: compLabels(:)
integer, allocatable :: petList(:)
character(len=10) :: value
character(len=20) :: model, prefix
character(len=160) :: msg
integer :: petListBounds(2)
integer :: ompNumThreads
integer :: componentCount
type(NUOPC_FreeFormat) :: attrFF, fdFF
logical :: found_comp
Expand Down Expand Up @@ -332,17 +338,42 @@ subroutine SetModelServices(driver, rc)
petList(j-petListBounds(1)+1) = j ! PETs are 0 based
enddo

! *** read in number of OpenMP threads for this component
call ESMF_ConfigGetAttribute(config, ompNumThreads, &
label=trim(prefix)//"_omp_num_threads:", default=-1, rc=rc)
if (ChkErr(rc,__LINE__,u_FILE_u)) return

! *** create info object
info = ESMF_InfoCreate(rc=rc)
if (ChkErr(rc,__LINE__,u_FILE_u)) return

! *** set up PePerPet NUOPC hints
if (ompNumThreads /= -1) then
call ESMF_InfoSet(info, key="/NUOPC/Hint/PePerPet/MaxCount", &
value=ompNumThreads, rc=rc)
if (ChkErr(rc,__LINE__,u_FILE_u)) return
endif

found_comp = .false.
#ifdef FRONT_FV3
if (trim(model) == "fv3") then
call NUOPC_DriverAddComp(driver, trim(prefix), FV3_SS, &
petList=petList, comp=comp, rc=rc)
info=info, petList=petList, comp=comp, rc=rc)
if (ChkErr(rc,__LINE__,u_FILE_u)) return
found_comp = .true.
end if
#endif
#if defined FRONT_CDEPS_DATM
if (trim(model) == "datm" ) then
!TODO: Remove bail code and pass info and SetVM to DriverAddComp
!TODO: once component supports threading.
if (ompNumThreads > 1) then
write (msg, *) "ESMF-aware threading NOT implemented for model: "//&
trim(model)
call ESMF_LogSetError(ESMF_RC_NOT_VALID, msg=msg, line=__LINE__, &
file=__FILE__, rcToReturn=rc)
return ! bail out
endif
call NUOPC_DriverAddComp(driver, trim(prefix), DATM_SS, &
petList=petList, comp=comp, rc=rc)
if (ChkErr(rc,__LINE__,u_FILE_u)) return
Expand All @@ -351,6 +382,15 @@ subroutine SetModelServices(driver, rc)
#endif
#ifdef FRONT_HYCOM
if (trim(model) == "hycom") then
!TODO: Remove bail code and pass info and SetVM to DriverAddComp
!TODO: once component supports threading.
if (ompNumThreads > 1) then
write (msg, *) "ESMF-aware threading NOT implemented for model: "//&
trim(model)
call ESMF_LogSetError(ESMF_RC_NOT_VALID, msg=msg, line=__LINE__, &
file=__FILE__, rcToReturn=rc)
return ! bail out
endif
call NUOPC_DriverAddComp(driver, trim(prefix), HYCOM_SS, &
petList=petList, comp=comp, rc=rc)
if (ChkErr(rc,__LINE__,u_FILE_u)) return
Expand All @@ -360,13 +400,22 @@ subroutine SetModelServices(driver, rc)
#ifdef FRONT_MOM6
if (trim(model) == "mom6") then
call NUOPC_DriverAddComp(driver, trim(prefix), MOM6_SS, &
petList=petList, comp=comp, rc=rc)
MOM6_SV, info=info, petList=petList, comp=comp, rc=rc)
if (ChkErr(rc,__LINE__,u_FILE_u)) return
found_comp = .true.
end if
#endif
#ifdef FRONT_CDEPS_DOCN
if (trim(model) == "docn") then
!TODO: Remove bail code and pass info and SetVM to DriverAddComp
!TODO: once component supports threading.
if (ompNumThreads > 1) then
write (msg, *) "ESMF-aware threading NOT implemented for model: "//&
trim(model)
call ESMF_LogSetError(ESMF_RC_NOT_VALID, msg=msg, line=__LINE__, &
file=__FILE__, rcToReturn=rc)
return ! bail out
endif
call NUOPC_DriverAddComp(driver, trim(prefix), DOCN_SS, &
petList=petList, comp=comp, rc=rc)
if (ChkErr(rc,__LINE__,u_FILE_u)) return
Expand All @@ -376,21 +425,30 @@ subroutine SetModelServices(driver, rc)
#ifdef FRONT_CICE6
if (trim(model) == "cice6") then
call NUOPC_DriverAddComp(driver, trim(prefix), CICE6_SS, &
petList=petList, comp=comp, rc=rc)
CICE6_SV, info=info, petList=petList, comp=comp, rc=rc)
if (ChkErr(rc,__LINE__,u_FILE_u)) return
found_comp = .true.
end if
#endif
#ifdef FRONT_WW3
if (trim(model) == "ww3") then
call NUOPC_DriverAddComp(driver, trim(prefix), WW3_SS, &
petList=petList, comp=comp, rc=rc)
WW3_SV, info=info, petList=petList, comp=comp, rc=rc)
if (ChkErr(rc,__LINE__,u_FILE_u)) return
found_comp = .true.
end if
#endif
#ifdef FRONT_NOAH
if (trim(model) == "noah") then
!TODO: Remove bail code and pass info and SetVM to DriverAddComp
!TODO: once component supports threading.
if (ompNumThreads > 1) then
write (msg, *) "ESMF-aware threading NOT implemented for model: "//&
trim(model)
call ESMF_LogSetError(ESMF_RC_NOT_VALID, msg=msg, line=__LINE__, &
file=__FILE__, rcToReturn=rc)
return ! bail out
endif
call NUOPC_DriverAddComp(driver, trim(prefix), NOAH_SS, &
petList=petList, comp=comp, rc=rc)
if (ChkErr(rc,__LINE__,u_FILE_u)) return
Expand All @@ -399,6 +457,15 @@ subroutine SetModelServices(driver, rc)
#endif
#ifdef FRONT_LIS
if (trim(model) == "lis") then
!TODO: Remove bail code and pass info and SetVM to DriverAddComp
!TODO: once component supports threading.
if (ompNumThreads > 1) then
write (msg, *) "ESMF-aware threading NOT implemented for model: "//&
trim(model)
call ESMF_LogSetError(ESMF_RC_NOT_VALID, msg=msg, line=__LINE__, &
file=__FILE__, rcToReturn=rc)
return ! bail out
endif
call NUOPC_DriverAddComp(driver, trim(prefix), LIS_SS, &
petList=petList, comp=comp, rc=rc)
if (ChkErr(rc,__LINE__,u_FILE_u)) return
Expand All @@ -407,6 +474,15 @@ subroutine SetModelServices(driver, rc)
#endif
#ifdef FRONT_IPE
if (trim(model) == "ipe") then
!TODO: Remove bail code and pass info and SetVM to DriverAddComp
!TODO: once component supports threading.
if (ompNumThreads > 1) then
write (msg, *) "ESMF-aware threading NOT implemented for model: "//&
trim(model)
call ESMF_LogSetError(ESMF_RC_NOT_VALID, msg=msg, line=__LINE__, &
file=__FILE__, rcToReturn=rc)
return ! bail out
endif
call NUOPC_DriverAddComp(driver, trim(prefix), IPE_SS, &
petList=petList, comp=comp, rc=rc)
if (ChkErr(rc,__LINE__,u_FILE_u)) return
Expand All @@ -415,6 +491,15 @@ subroutine SetModelServices(driver, rc)
#endif
#ifdef FRONT_GOCART
if (trim(model) == "gocart") then
!TODO: Remove bail code and pass info and SetVM to DriverAddComp
!TODO: once component supports threading.
if (ompNumThreads > 1) then
write (msg, *) "ESMF-aware threading NOT implemented for model: "//&
trim(model)
call ESMF_LogSetError(ESMF_RC_NOT_VALID, msg=msg, line=__LINE__, &
file=__FILE__, rcToReturn=rc)
return ! bail out
endif
call NUOPC_DriverAddComp(driver, trim(prefix), GOCART_SS, &
petList=petList, comp=comp, rc=rc)
if (ChkErr(rc,__LINE__,u_FILE_u)) return
Expand All @@ -424,7 +509,7 @@ subroutine SetModelServices(driver, rc)
#ifdef FRONT_CMEPS
if (trim(model) == "cmeps") then
call NUOPC_DriverAddComp(driver, trim(prefix), MED_SS, &
petList=petList, comp=comp, rc=rc)
MED_SV, info=info, petList=petList, comp=comp, rc=rc)
if (ChkErr(rc,__LINE__,u_FILE_u)) return
found_comp = .true.
end if
Expand All @@ -441,6 +526,8 @@ subroutine SetModelServices(driver, rc)

! clean-up
deallocate(petList)
call ESMF_InfoDestroy(info, rc=rc)
if (ChkErr(rc,__LINE__,u_FILE_u)) return
enddo

! clean-up
Expand Down
75 changes: 15 additions & 60 deletions driver/UFS.F90
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,8 +76,6 @@ PROGRAM UFS
TYPE(ESMF_Clock) :: CLOCK_MAIN !<-- The ESMF time management clock
!
TYPE(ESMF_Config) :: CF_MAIN !<-- The Configure object
!
LOGICAL :: PRINT_ESMF !<-- Flag for ESMF PET files
!
CHARACTER(ESMF_MAXSTR) :: MESSAGE_CHECK
!
Expand All @@ -89,28 +87,20 @@ PROGRAM UFS
!-----------------------------------------------------------------------
!
!-----------------------------------------------------------------------
!*** Check if we want ESMF PET files or not
!-----------------------------------------------------------------------
!
CALL CHECK_ESMF_PET(PRINT_ESMF)
!
!-----------------------------------------------------------------------
!*** Initialize the ESMF framework.
!-----------------------------------------------------------------------
!
IF(PRINT_ESMF) THEN
CALL ESMF_Initialize(VM =VM & !<-- The ESMF Virtual Machine
,defaultCalKind =ESMF_CALKIND_GREGORIAN & !<-- Set up the default calendar.
,logkindflag =ESMF_LOGKIND_MULTI & !<-- Define multiple log error output files;
,rc =RC)
ESMF_ERR_ABORT(RC)
ELSE
CALL ESMF_Initialize(VM =VM & !<-- The ESMF Virtual Machine
,defaultCalKind =ESMF_CALKIND_GREGORIAN & !<-- Set up the default calendar.
,logkindflag =ESMF_LOGKIND_NONE & !<-- Define no log error output files;
,rc =RC)
ESMF_ERR_ABORT(RC)
ENDIF
CALL ESMF_Initialize(configFileName="nems.configure" & !<-- top level configuration
,defaultCalKind =ESMF_CALKIND_GREGORIAN & !<-- Set up the default calendar.
,VM =VM & !<-- The ESMF Virtual Machine
,rc =RC)
ESMF_ERR_ABORT(RC)

#if 0
!*** Useful when debugging ESMF-managed threading and resource control
call ESMF_VMLog(vm, rc=rc)
ESMF_ERR_ABORT(RC)
#endif
!
!-----------------------------------------------------------------------
!*** Extract the MPI task ID.
Expand Down Expand Up @@ -142,12 +132,10 @@ PROGRAM UFS
! CALL ESMF_LogWrite(MESSAGE_CHECK,ESMF_LOGMSG_INFO,rc=RC)
! ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~
!
IF(PRINT_ESMF) THEN
CALL ESMF_LogSet(flush =.false. &
,trace =.false. &
,rc =RC)
ESMF_ERR_ABORT(RC)
ENDIF
CALL ESMF_LogSet(flush =.false. &
,trace =.false. &
,rc =RC)
ESMF_ERR_ABORT(RC)

! ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~
!
Expand Down Expand Up @@ -479,39 +467,6 @@ PROGRAM UFS
!
!-----------------------------------------------------------------------
!
contains
subroutine check_esmf_pet(print_esmf)

implicit none
integer :: i,n
character *256 :: c1,c2
logical :: opened,print_esmf

do n=101,201
inquire(n,opened=opened)
if(.not.opened)then
open(n,file='model_configure',status='old') !<-- Open configure file
exit
endif
enddo

print_esmf=.false.

do i=1,10000
read(n,*,end=22)c1,c2
if(c1(1:10) == 'print_esmf') then !<-- Search for print_esmf flag
if( c2 == 'true' .or. & !<-- Check if print_esmf is true or false
c2 == '.true.' .or. &
c2 == 'TRUE' .or. &
c2 == '.TRUE.' ) print_esmf=.true.
exit
endif
enddo
22 close(n)
return

end subroutine check_esmf_pet

END PROGRAM UFS
!
!-----------------------------------------------------------------------
1 change: 1 addition & 0 deletions modulefiles/ufs_hera.intel
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ module load hpc/1.1.0

module load hpc-intel/18.0.5.274
module load hpc-impi/2018.0.4
#module swap impi/2021.3.0

module load ufs_common

Expand Down
1 change: 1 addition & 0 deletions modulefiles/ufs_hera.intel_debug
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ module load hpc/1.1.0

module load hpc-intel/18.0.5.274
module load hpc-impi/2018.0.4
#module swap impi/2021.3.0

module load ufs_common_debug

Expand Down
Loading

0 comments on commit bce0ff3

Please sign in to comment.