Merge pull request #1655 from GEOS-ESM/feature/aoloso/hybrid-openmp

Feature/aoloso/hybrid openmp

Apps/MAPL_GridCompSpecs_ACG.py

@@ -94,7 +94,7 @@ def emit_declare_pointers(self):
         text = text + type
         if kind:
             text = text + '(kind=' + str(kind) + ')'
-        text = text +', pointer, ' + dimension + ' :: ' + MAPL_DataSpec.internal_name(self.args['short_name']) + ' => null()'
+        text = text +', pointer, ' + dimension + ' :: ' + MAPL_DataSpec.internal_name(self.args['short_name'])
         return text
 
     def emit_get_pointers(self):

CHANGELOG.md

@@ -7,6 +7,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Fixed
+- Removed ESMFL_UnitsRadians from ESMFL_Mod.F90 and replaced it with MAPL_UnitsRadians in InternalConstants.F90. Did a global search/replace. This avoid the circular dependency build error when ESMF_UnitsRadians is used in MaplGrid.F90
+
+### Added
+- Several changes to support component level hybrid MPI/OpenMP.
+  - See files OpenMP_Support.F90, EntryPointVector.F90, RunEntryPoint.F90, MaplGenericComponent.F90, MAPL_Generic.F90, and BaseProfiler.F90. The first three are new and the rest  are modified.
+
+
+## [hybrid-openmp] - 2022-09-14
 ### Fixed
 
 ### Added

Tests/ExtDataDriverGridComp.F90

@@ -536,7 +536,7 @@ subroutine run(this, rc)
     integer :: status
     integer :: userRc
 
-    call ESMF_GridCompRun(this%gc, userRC=userRC,__RC__)
+    call ESMF_GridCompRun(this%gc, userRC=userRC,_RC)
     _VERIFY(userRC)
     _RETURN(ESMF_SUCCESS)
 

Tests/ExtDataRoot_GridComp.F90

@@ -339,13 +339,13 @@ SUBROUTINE Run_ ( GC, IMPORT, EXPORT, CLOCK, rc )
          call ESMF_UserCompGetInternalState(gc,wrap_name,synthWrap,status)
          _VERIFY(status)
          synth => synthWrap%ptr
-         call ESMF_GridCompGet(GC,grid=grid,__RC__)
-         call MAPL_GetPointer(internal,ptrR4,'lons',__RC__)
+         call ESMF_GridCompGet(GC,grid=grid,_RC)
+         call MAPL_GetPointer(internal,ptrR4,'lons',_RC)
          call ESMF_GridGetCoord (Grid, coordDim=1, localDE=0, &
                            staggerloc=ESMF_STAGGERLOC_CENTER, &
                            farrayPtr=ptrR8, rc=status)
          ptrR4=ptrR8
-         call MAPL_GetPointer(internal,ptrR4,'lats',__RC__)
+         call MAPL_GetPointer(internal,ptrR4,'lats',_RC)
          call ESMF_GridGetCoord (Grid, coordDim=2, localDE=0, &
                            staggerloc=ESMF_STAGGERLOC_CENTER, &
                            farrayPtr=ptrR8, rc=status) 
@@ -355,17 +355,17 @@ SUBROUTINE Run_ ( GC, IMPORT, EXPORT, CLOCK, rc )
 
          case(RunModeGenerateExports)
 
-            call FillState(internal,export,currTime,grid,synth,__RC__) 
+            call FillState(internal,export,currTime,grid,synth,_RC) 
 
          case(runModecompareImports)
-            call FillState(internal,export,currTime,grid,synth,__RC__)
-            call CompareState(import,export,0.001,__RC__) 
+            call FillState(internal,export,currTime,grid,synth,_RC)
+            call CompareState(import,export,0.001,_RC) 
 
          case(runModeFillImport)
 ! Nothing to do, we are just letting ExtData run
 
          case(runModeFillExportFromImport)
-            call CopyState(import,export,__RC__)
+            call CopyState(import,export,_RC)
 
          end select
 
@@ -516,29 +516,29 @@ subroutine CopyState(inState,outState,rc)
       character(len=ESMF_MAXSTR), allocatable :: outNameList(:)
       type(ESMF_Field) :: expf,impf
 
-      call ESMF_StateGet(inState,itemcount=itemCountIn,__RC__)
+      call ESMF_StateGet(inState,itemcount=itemCountIn,_RC)
       allocate(InNameList(itemCountIn),stat=status)
       _VERIFY(status)
-      call ESMF_StateGet(inState,itemNameList=InNameList,__RC__)
+      call ESMF_StateGet(inState,itemNameList=InNameList,_RC)
 
-      call ESMF_StateGet(outState,itemcount=ItemCountOut,__RC__)
+      call ESMF_StateGet(outState,itemcount=ItemCountOut,_RC)
       allocate(outNameList(ItemCountOut),stat=status)
       _VERIFY(status)
-      call ESMF_StateGet(outState,itemNameList=outNameList,__RC__)
+      call ESMF_StateGet(outState,itemNameList=outNameList,_RC)
 
       _ASSERT(itemCountIn == itemCountOut,'needs informative message')
-      call ESMF_StateGet(inState,itemNameList=inNameList,__RC__)
+      call ESMF_StateGet(inState,itemNameList=inNameList,_RC)
       do i=1,itemCountIn
-         call ESMF_StateGet(inState,trim(inNameList(i)),impf,__RC__)
-         call ESMF_StateGet(outState,trim(outNameList(i)),expf,__RC__)
-         call ESMF_FieldGet(impf,rank=rank,__RC__)
+         call ESMF_StateGet(inState,trim(inNameList(i)),impf,_RC)
+         call ESMF_StateGet(outState,trim(outNameList(i)),expf,_RC)
+         call ESMF_FieldGet(impf,rank=rank,_RC)
          if (rank==2) then
-            call MAPL_GetPointer(inState,IMptr2,inNameList(i),__RC__)
-            call MAPL_GetPointer(outState,Exptr2,inNameList(i),alloc=.true.,__RC__)
+            call MAPL_GetPointer(inState,IMptr2,inNameList(i),_RC)
+            call MAPL_GetPointer(outState,Exptr2,inNameList(i),alloc=.true.,_RC)
             EXptr2=IMptr2
          else if (rank==3) then
-            call MAPL_GetPointer(inState,IMptr3,inNameList(i),__RC__)
-            call MAPL_GetPointer(outState,EXptr3,inNameList(i),alloc=.true.,__RC__)
+            call MAPL_GetPointer(inState,IMptr3,inNameList(i),_RC)
+            call MAPL_GetPointer(outState,EXptr3,inNameList(i),alloc=.true.,_RC)
             EXptr3=IMptr3
          end if
       end do
@@ -566,25 +566,25 @@ subroutine FillState(inState,outState,time,grid,Synth,rc)
       character(len=:), pointer :: fexpr
       integer :: i1,in,j1,jn,ldims(3),i,j
 
-      call MAPL_GridGet(grid,localcellcountperdim=ldims,__RC__)
+      call MAPL_GridGet(grid,localcellcountperdim=ldims,_RC)
       call MAPL_Grid_Interior(grid,i1,in,j1,jn)
-      call ESMF_StateGet(outState,itemcount=itemCount,__RC__)
+      call ESMF_StateGet(outState,itemcount=itemCount,_RC)
       allocate(outNameList(itemCount),stat=status)
       _VERIFY(status)
-      call ESMF_StateGet(outState,itemNameList=outNameList,__RC__)
+      call ESMF_StateGet(outState,itemNameList=outNameList,_RC)
 
       call MAPL_GetPointer(inState,exPtr2,'time',rc=status)
       _VERIFY(status)
       exPtr2=synth%tFunc%evaluateTime(Time,rc=status)
       _VERIFY(status)
 
-      call MAPL_GetPointer(inState,exPtr2,'i_index',__RC__)
+      call MAPL_GetPointer(inState,exPtr2,'i_index',_RC)
       do j = 1,ldims(2)
          do i=1,ldims(1)
             exPtr2(i,j)=i1+i-1
          enddo
       enddo
-      call MAPL_GetPointer(inState,exPtr2,'j_index',__RC__)
+      call MAPL_GetPointer(inState,exPtr2,'j_index',_RC)
       do i = 1,ldims(1)
          do j=1,ldims(2)
             exPtr2(i,j)=j1+j-1
@@ -636,22 +636,22 @@ subroutine CompareState(State1,State2,tol,rc)
       logical, allocatable :: foundDiff(:)
       type(ESMF_Field) :: Field1,Field2
 
-      call ESMF_StateGet(State1,itemcount=itemCount,__RC__)
+      call ESMF_StateGet(State1,itemcount=itemCount,_RC)
          allocate(NameList(itemCount),stat=status)
          _VERIFY(status)
          allocate(foundDiff(itemCount),stat=status)
          _VERIFY(status)
-         call ESMF_StateGet(State1,itemNameList=NameList,__RC__)
+         call ESMF_StateGet(State1,itemNameList=NameList,_RC)
          do ii=1,itemCount
-            call ESMF_StateGet(State1,trim(nameList(ii)),field1,__RC__)
-            call ESMF_StateGet(State2,trim(nameList(ii)),field2,__RC__)
-            call ESMF_FieldGet(field1,rank=rank1,__RC__)
-            call ESMF_FieldGet(field1,rank=rank2,__RC__)
+            call ESMF_StateGet(State1,trim(nameList(ii)),field1,_RC)
+            call ESMF_StateGet(State2,trim(nameList(ii)),field2,_RC)
+            call ESMF_FieldGet(field1,rank=rank1,_RC)
+            call ESMF_FieldGet(field1,rank=rank2,_RC)
             _ASSERT(rank1==rank2,'needs informative message')
             foundDiff(ii)=.false.
             if (rank1==2) then
-               call MAPL_GetPointer(state1,ptr2_1,trim(nameList(ii)),__RC__)
-               call MAPL_GetPointer(state2,ptr2_2,trim(nameList(ii)),__RC__)
+               call MAPL_GetPointer(state1,ptr2_1,trim(nameList(ii)),_RC)
+               call MAPL_GetPointer(state2,ptr2_2,trim(nameList(ii)),_RC)
                do i=1,size(ptr2_1,1) 
                   do j=1,size(ptr2_1,2)
                      if (abs(ptr2_1(i,j)-ptr2_2(i,j)) .gt. tol) then
@@ -661,8 +661,8 @@ subroutine CompareState(State1,State2,tol,rc)
                   enddo
                enddo
             else if (rank1==3) then
-               call MAPL_GetPointer(state1,ptr3_1,trim(nameList(ii)),__RC__)
-               call MAPL_GetPointer(state2,ptr3_2,trim(nameList(ii)),__RC__)
+               call MAPL_GetPointer(state1,ptr3_1,trim(nameList(ii)),_RC)
+               call MAPL_GetPointer(state2,ptr3_2,trim(nameList(ii)),_RC)
                lb=lbound(ptr3_1)
                ub=ubound(ptr3_1) 
                do i=1,size(ptr3_1,1) 
@@ -700,23 +700,23 @@ subroutine ForceAllocation(state,rc)
          type (ESMF_StateItem_Flag), allocatable :: itemTypeList(:)
          type(ESMF_Field) :: Field
 
-         call ESMF_StateGet(State,itemcount=itemCount,__RC__)
+         call ESMF_StateGet(State,itemcount=itemCount,_RC)
          allocate(NameList(itemCount),stat=status)
          _VERIFY(status)
          allocate(itemTypeList(itemCount),stat=status)
          _VERIFY(status)
-         call ESMF_StateGet(State,itemNameList=NameList,itemTypeList=itemTypeList,__RC__)
+         call ESMF_StateGet(State,itemNameList=NameList,itemTypeList=itemTypeList,_RC)
          if (itemCount == 0) then
             _RETURN(ESMF_SUCCESS)
          end if
          do ii=1,itemCount
             if (itemTypeList(ii)==ESMF_STATEITEM_FIELD) then
-               call ESMF_StateGet(State,trim(nameList(ii)),field,__RC__)
-               call ESMF_AttributeGet(field,name='DIMS',value=dims,__RC__)
+               call ESMF_StateGet(State,trim(nameList(ii)),field,_RC)
+               call ESMF_AttributeGet(field,name='DIMS',value=dims,_RC)
                if (dims==MAPL_DimsHorzOnly) then
-                  call MAPL_GetPointer(state,ptr2d,trim(nameList(ii)),alloc=.true.,__RC__)
+                  call MAPL_GetPointer(state,ptr2d,trim(nameList(ii)),alloc=.true.,_RC)
                else if (dims==MAPL_DimsHorzVert) then
-                  call MAPL_GetPointer(state,ptr3d,trim(nameList(ii)),alloc=.true.,__RC__)
+                  call MAPL_GetPointer(state,ptr3d,trim(nameList(ii)),alloc=.true.,_RC)
                end if
             end if
          enddo

base/Base/Base_Base_implementation.F90

@@ -2760,7 +2760,6 @@ module subroutine MAPL_StateAddField(State, Field, RC)
     type(ESMF_Field)                        :: Fields(1)
     logical                                 :: haveAttr
 
-
     fields(1) = field
     call ESMF_StateAdd(state, fields, RC=status)
     _VERIFY(STATUS)
@@ -3021,6 +3020,7 @@ module subroutine MAPL_GetHorzIJIndex(npts,II,JJ,lon,lat,lonR8,latR8,Grid, rc)
     real(ESMF_KIND_R8), allocatable :: target_lons(:),target_lats(:)
     real(ESMF_KIND_R8), allocatable :: corner_lons(:,:),corner_lats(:,:),center_lats(:,:),center_lons(:,:)
     type(ESMF_CoordSys_Flag) :: coordSys
+    character(len=ESMF_MAXSTR) :: grid_type
 
     ! if the grid is present then we can just get the prestored edges and the dimensions of the grid
     ! this also means we are running on a distributed grid
@@ -3049,7 +3049,9 @@ module subroutine MAPL_GetHorzIJIndex(npts,II,JJ,lon,lat,lonR8,latR8,Grid, rc)
 
     _ASSERT(localSearch,"Global Search for IJ not implemented")   
 
-    if (im_world*6==jm_world) then
+!AOO change tusing GridType atribute    if (im_world*6==jm_world) then
+    call ESMF_AttributeGet(grid, name='GridType', value=grid_type, _RC)
+    if(trim(grid_type) == "Cubed-Sphere") then
        call ESMF_GridGetCoord(grid,coordDim=1, localDe=0, &
             staggerloc=ESMF_STAGGERLOC_CENTER, fArrayPtr = lons, rc=status)
        _VERIFY(STATUS)

base/ESMFL_Mod.F90

@@ -31,9 +31,6 @@ module ESMFL_MOD
 !ALT These need to be changed!!! values here are just to compile
 !
 
-  !DEFINED PARAMETERS:
-  integer, parameter, public :: ESMFL_UnitsRadians = 99
-
   !PUBLIC MEMBER FUNCTIONS:
   public ESMFL_StateGetField
   public ESMFL_StateGetFieldArray
@@ -1070,14 +1067,14 @@ subroutine ESMFL_FieldRegrid(src, dst, RC)
    call ESMFL_GridCoordGet(   DSTGRID, LATS                   , &
                               Name     = "Latitude"              , &
                               Location = ESMF_CELL_CENTER        , &
-                              Units    = ESMFL_UnitsRadians      , &
+                              Units    = MAPL_UnitsRadians      , &
                               RC       = STATUS                    )
    _VERIFY(STATUS)
 
    call ESMFL_GridCoordGet(   DSTGRID, LONS                   , &
                               Name     = "Longitude"             , &
                               Location = ESMF_CELL_CENTER        , &
-                              Units    = ESMFL_UnitsRadians      , &
+                              Units    = MAPL_UnitsRadians      , &
                               RC       = STATUS                    )
    _VERIFY(STATUS)
 
@@ -2131,7 +2128,7 @@ subroutine Bundle_Prep_ (srcBUN, dstBUN, only_vars)
    call ESMFL_GridCoordGet(   srcGrid, llats                     , &
                               Name     = "Latitude"              , &
                               Location = ESMF_STAGGERLOC_CENTER  , &
-                              Units    = ESMFL_UnitsRadians      , &
+                              Units    = MAPL_UnitsRadians      , &
                               RC       = STATUS                    )
    if (status /= ESMF_SUCCESS) call ESMFL_FailedRC(mype,Iam)
    allocate(gLats(ims_world,jms_world), stat=status)
@@ -2148,7 +2145,7 @@ subroutine Bundle_Prep_ (srcBUN, dstBUN, only_vars)
    call ESMFL_GridCoordGet(   srcGrid, llons                     , &
                               Name     = "Longitude"             , &
                               Location = ESMF_STAGGERLOC_CENTER  , &
-                              Units    = ESMFL_UnitsRadians      , &
+                              Units    = MAPL_UnitsRadians      , &
                               RC       = STATUS                    )
    if (status /= ESMF_SUCCESS) call ESMFL_FailedRC(mype,Iam)
    allocate(gLons(ims_world,jms_world), stat=status)

base/GetPointer.H

@@ -83,25 +83,28 @@
          _VERIFY(STATUS)
       endif
    endif
-
+   
 !ALT I dont think the next lines are needed anymore
 #if 0
    block
      integer :: DIMS
      call ESMF_AttributeGet(field, name='VLOCATION', value=DIMS, rc=status)
      if (STATUS==ESMF_SUCCESS) then
         if (DIMS == MAPL_VLocationEdge .and. associated(ptr)) then
+#if RANK_ == 3
            call AdjustPtrBounds(ptr, ptr, 1, size(ptr,1), 1, size(ptr,2), 0, size(ptr,3)-1)
+#elif RANK_ == 1
+           call AdjustPtrBounds(ptr, ptr, 0, size(ptr,1)-1)
+#endif
         end if
      end if
    end block
 #endif
-   
+
    _RETURN(ESMF_SUCCESS)
 
  end subroutine SUB_
 
 #undef DIMENSIONS_
 #undef RANK_
 #undef RANKSTR_
-

base/HorizontalFluxRegridder.F90

@@ -50,8 +50,8 @@ logical function supports(spec, unusable, rc)
       supports = (spec%regrid_method == REGRID_METHOD_CONSERVE_HFLUX)
       if (.not. supports) return
 
-      call MAPL_GridGet(spec%grid_in, localCellCountPerDim=counts_in, __RC__)
-      call MAPL_GridGet(spec%grid_out, localCellCountPerDim=counts_out, __RC__)
+      call MAPL_GridGet(spec%grid_in, localCellCountPerDim=counts_in, _RC)
+      call MAPL_GridGet(spec%grid_out, localCellCountPerDim=counts_out, _RC)
 
       supports = all(mod(counts_in(1:2), counts_out(1:2)) == 0) .or. all(mod(counts_out, counts_in) == 0)
 
@@ -78,11 +78,11 @@ subroutine initialize_subclass(this, unusable, rc)
 
        associate (IM_in => this%IM_in, JM_in => this%JM_in, IM_out => this%IM_out, JM_out => this%JM_out)
 
-         call MAPL_GridGet(grid_in, localCellCountPerDim=counts, __RC__)
+         call MAPL_GridGet(grid_in, localCellCountPerDim=counts, _RC)
          IM_in = counts(1)
          JM_in = counts(2)
 
-         call MAPL_GridGet(grid_out, localCellCountPerDim=counts, __RC__)
+         call MAPL_GridGet(grid_out, localCellCountPerDim=counts, _RC)
          IM_out = counts(1)
          JM_out = counts(2)