Merge pull request #2419 from GEOS-ESM/feature/bmauer/fixes-#2418

Feature/bmauer/fixes #2418

CHANGELOG.md

@@ -23,6 +23,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Fixed
 
+- Updated write_by_oserver logic so that the decision to write by the oserver is based on whether the output server client is passed in
 - Updated CI GEOSadas build to use special branch (as stock ADAS at the moment is too far behind GEOSgcm main)
 - Fix incorrect History print during runtime
 

base/NCIO.F90

@@ -312,6 +312,7 @@ subroutine MAPL_FieldWriteNCPar(formatter, name, FIELD, ARRDES, HomePE, oClients
     type (ESMF_DistGrid)               :: distGrid
     type (LocalMemReference) :: lMemRef
     integer :: size_1d
+    logical :: have_oclients
 
     call ESMF_FieldGet(field, grid=grid, rc=status)
     _VERIFY(STATUS)
@@ -320,9 +321,7 @@ subroutine MAPL_FieldWriteNCPar(formatter, name, FIELD, ARRDES, HomePE, oClients
     call ESMF_DistGridGet(distGrid, delayout=layout, rc=STATUS)
     _VERIFY(STATUS)
 
-    if( arrdes%write_restart_by_oserver) then
-      _ASSERT(present(oClients), "output server is needed")
-    endif
+    have_oclients = present(oClients) 
 
     call ESMF_AttributeGet(field, name='DIMS', value=DIMS, rc=status)
     _VERIFY(STATUS)
@@ -351,7 +350,7 @@ subroutine MAPL_FieldWriteNCPar(formatter, name, FIELD, ARRDES, HomePE, oClients
                 size_1d = size(var_1d,1)
              endif
 
-             if (arrdes%write_restart_by_oserver) then
+             if (have_oclients) then
                 if( MAPL_AM_I_ROOT())  then
                    lMemRef = LocalMemReference(pFIO_REAL32,[size_1d])
                    call c_f_pointer(lMemRef%base_address, gvar_1d, shape=[size_1d])
@@ -390,7 +389,7 @@ subroutine MAPL_FieldWriteNCPar(formatter, name, FIELD, ARRDES, HomePE, oClients
                 size_1d = size(vr8_1d,1)
              endif
 
-             if (arrdes%write_restart_by_oserver) then
+             if (have_oclients) then
                 if(MAPL_AM_I_ROOT()) then
                    lMemRef = LocalMemReference(pFIO_REAL64,[size_1d])
                    call c_f_pointer(lMemRef%base_address, gvr8_1d, shape=[size_1d])
@@ -428,7 +427,7 @@ subroutine MAPL_FieldWriteNCPar(formatter, name, FIELD, ARRDES, HomePE, oClients
           if (associated(var_2d)) then !ALT: temp kludge
              if (DIMS == MAPL_DimsTileOnly .or. DIMS == MAPL_DimsTileTile) then
 
-                if (arrdes%write_restart_by_oserver) then
+                if (have_oclients) then
                    if(MAPL_AM_I_ROOT()) then
                       lMemRef = LocalMemReference(pFIO_REAL32,[arrdes%im_world, size(var_2d,2)])
                       call c_f_pointer(lMemRef%base_address, gvar_2d, shape=[arrdes%im_world, size(var_2d,2)])
@@ -462,7 +461,7 @@ subroutine MAPL_FieldWriteNCPar(formatter, name, FIELD, ARRDES, HomePE, oClients
           if (associated(vr8_2d)) then !ALT: temp kludge
              if (DIMS == MAPL_DimsTileOnly .or. DIMS == MAPL_DimsTileTile) then
 
-                if (arrdes%write_restart_by_oserver) then
+                if (have_oclients) then
                    if( MAPL_AM_I_ROOT() ) then
                       lMemRef = LocalMemReference(pFIO_REAL64,[arrdes%im_world,size(vr8_2d,2)])
                       call c_f_pointer(lMemRef%base_address, gvr8_2d, shape=[arrdes%im_world,size(vr8_2d,2)])
@@ -497,7 +496,7 @@ subroutine MAPL_FieldWriteNCPar(formatter, name, FIELD, ARRDES, HomePE, oClients
           if (associated(var_3d)) then !ALT: temp kludge
              if (DIMS == MAPL_DimsTileOnly) then
 
-                if (arrdes%write_restart_by_oserver) then
+                if (have_oclients) then
                    if( MAPL_Am_I_Root() ) then
                       lMemRef = LocalMemReference(pFIO_REAL32,[arrdes%im_world, size(var_3d,2), size(var_3d,3)])
                       call c_f_pointer(lMemRef%base_address, gvar_3d, shape=[arrdes%im_world, size(var_3d,2), size(var_3d,3)])
@@ -536,7 +535,7 @@ subroutine MAPL_FieldWriteNCPar(formatter, name, FIELD, ARRDES, HomePE, oClients
           if (associated(vr8_3d)) then !ALT: temp kludge
              if (DIMS == MAPL_DimsTileOnly) then
 
-                if (arrdes%write_restart_by_oserver) then
+                if (have_oclients) then
                    if( MAPL_Am_I_Root() ) then
                       lMemRef = LocalMemReference(pFIO_REAL64,[arrdes%im_world,size(vr8_3d,2), size(vr8_3d,3)])
                       call c_f_pointer(lMemRef%base_address, gvr8_3d, shape=[arrdes%im_world,size(vr8_3d,2), size(vr8_3d,3)])
@@ -619,8 +618,7 @@ subroutine MAPL_VarWriteNCpar_R4_4d(formatter, name, A, ARRDES, oClients, RC)
     type(ArrayReference)     :: ref
 
     if (present(arrdes)) then
-       if (arrdes%write_restart_by_oserver) then
-          _ASSERT(present(oClients), "output server is needed")
+       if (present(oClients)) then
           call MAPL_GridGet(arrdes%grid,globalCellCountPerDim=global_dim,rc=status)
           _VERIFY(status)
           call MAPL_Grid_interior(arrdes%grid,i1,in,j1,jn)
@@ -670,8 +668,7 @@ subroutine MAPL_VarWriteNCpar_R8_4d(formatter, name, A, ARRDES, oClients, RC)
     integer ::  i1, j1, in, jn,  global_dim(3)
     type(ArrayReference)     :: ref
 
-    if (arrdes%write_restart_by_oserver) then
-       _ASSERT(present(oClients), "output server is needed")
+    if (present(oClients)) then
        call MAPL_GridGet(arrdes%grid,globalCellCountPerDim=global_dim,rc=status)
        _VERIFY(status)
        call MAPL_Grid_interior(arrdes%grid,i1,in,j1,jn)
@@ -712,8 +709,7 @@ subroutine MAPL_VarWriteNCpar_R4_3d(formatter, name, A, ARRDES, oClients, RC)
     type(ArrayReference)     :: ref
 
     if (present(arrdes)) then
-       if (arrdes%write_restart_by_oserver) then
-          _ASSERT(present(oClients), "output server is needed")
+       if (present(oclients)) then
           call MAPL_GridGet(arrdes%grid,globalCellCountPerDim=global_dim,rc=status)
           _VERIFY(status)
           call MAPL_Grid_interior(arrdes%grid,i1,in,j1,jn)
@@ -780,8 +776,7 @@ subroutine MAPL_VarWriteNCpar_R8_3d(formatter, name, A, ARRDES, oClients, RC)
     type(ArrayReference)     :: ref
 
 
-    if (arrdes%write_restart_by_oserver) then
-       _ASSERT(present(oClients), "outpur server is needed")
+    if (present(oclients)) then
        call MAPL_GridGet(arrdes%grid,globalCellCountPerDim=global_dim,rc=status)
         _VERIFY(status)
        call MAPL_Grid_interior(arrdes%grid,i1,in,j1,jn)
@@ -852,8 +847,7 @@ subroutine MAPL_VarWriteNCpar_R4_2d(formatter, name, A, ARRDES, lev, offset2, oC
     integer ::  i1, j1, in, jn,  global_dim(3)
 
     if (present(arrdes)) then
-       if(arrdes%write_restart_by_oserver) then
-          _ASSERT(present(oClients), "output server is needed")
+       if(present(oClients)) then
           call MAPL_GridGet(arrdes%grid,globalCellCountPerDim=global_dim,rc=status)
            _VERIFY(status)
           call MAPL_Grid_interior(arrdes%grid,i1,in,j1,jn)
@@ -2342,8 +2336,7 @@ subroutine MAPL_VarWriteNCpar_R8_2d(formatter, name, A, ARRDES, lev, offset2, oC
     integer ::  i1, j1, in, jn,  global_dim(3)
 
     if (present(arrdes)) then
-       if( arrdes%write_restart_by_oserver) then
-          _ASSERT(present(oClients), "output server is needed")
+       if(present(oClients)) then
           call MAPL_GridGet(arrdes%grid,globalCellCountPerDim=global_dim,rc=status)
            _VERIFY(status)
           call MAPL_Grid_interior(arrdes%grid,i1,in,j1,jn)
@@ -3274,8 +3267,9 @@ subroutine MAPL_BundleWriteNCPar(Bundle, arrdes, CLOCK, filename, oClients, rc)
     character(len=ESMF_MAXSTR) :: positive
     type(StringVector) :: flip_vars
     type(ESMF_Field) :: lons_field, lats_field
-    logical :: isGridCapture
+    logical :: isGridCapture, have_oclients
     real(kind=ESMF_KIND_R8), pointer :: grid_lons(:,:), grid_lats(:,:), lons_field_ptr(:,:), lats_field_ptr(:,:)
+    have_oclients = present(oClients)
 
     call ESMF_FieldBundleGet(Bundle,FieldCount=nVars, name=BundleName, rc=STATUS)
     _VERIFY(STATUS)
@@ -3471,7 +3465,7 @@ subroutine MAPL_BundleWriteNCPar(Bundle, arrdes, CLOCK, filename, oClients, rc)
     ndims = ndims + 1
 
     !WJ note: if arrdes%write_restart_by_oserver is true, all processors will participate
-    if (arrdes%writers_comm/=MPI_COMM_NULL .or. arrdes%write_restart_by_oserver) then
+    if (arrdes%writers_comm/=MPI_COMM_NULL .or. have_oclients ) then
 
        ! Create dimensions as needed
        if (Have_HorzVert .or. Have_HorzOnly) then
@@ -3832,8 +3826,7 @@ subroutine MAPL_BundleWriteNCPar(Bundle, arrdes, CLOCK, filename, oClients, rc)
        _VERIFY(STATUS)
 
 
-       if (arrdes%write_restart_by_oserver) then
-          _ASSERT(present(oClients), 'output server is needed')
+       if (have_oclients) then
           call oClients%set_optimal_server(1)
           iter = RstCollections%find(trim(BundleName))
           if (iter == RstCollections%end()) then
@@ -3935,7 +3928,7 @@ subroutine MAPL_BundleWriteNCPar(Bundle, arrdes, CLOCK, filename, oClients, rc)
        call MAPL_FieldWriteNCPar(formatter, 'lats', lats_field, arrdes, HomePE=mask, oClients=oClients, rc=status)
     end if
 
-    if (arrdes%write_restart_by_oserver) then
+    if (have_oclients) then
        call oClients%done_collective_stage(_RC)
        call oClients%post_wait()
        call MPI_Info_free(info, status)

generic/MAPL_Generic.F90

@@ -2019,14 +2019,12 @@ subroutine capture(POS, PHASE, GC, IMPORT, EXPORT, CLOCK, RC)
         write(phase_, '(i1)') phase
 
         call MAPL_ESMFStateWriteToFile(import, CLOCK, trim(FILENAME)//"import_"//trim(POS)//"_runPhase"//phase_, &
-             FILETYPE, STATE, .false., _RC)
-
+             FILETYPE, STATE, .false., state%grid%write_restart_by_oserver, _RC)
         call MAPL_ESMFStateWriteToFile(export, CLOCK, trim(FILENAME)//"export_"//trim(POS)//"_runPhase"//phase_, &
-             FILETYPE, STATE, .false., oClients = o_Clients, _RC)
-
+             FILETYPE, STATE, .false., state%grid%write_restart_by_oserver, _RC)
         call MAPL_GetResource(STATE, hdr, default=0, LABEL="INTERNAL_HEADER:", _RC)
         call MAPL_ESMFStateWriteToFile(internal, CLOCK, trim(FILENAME)//"internal_"//trim(POS)//"_runPhase"//phase_, &
-             FILETYPE, STATE, hdr/=0, oClients = o_Clients, _RC)
+             FILETYPE, STATE, hdr/=0, state%grid%write_restart_by_oserver, _RC)
      end if
      _RETURN(_SUCCESS)
    end subroutine capture
@@ -2407,7 +2405,7 @@ recursive subroutine MAPL_GenericFinalize ( GC, IMPORT, EXPORT, CLOCK, RC )
             _VERIFY(status)
             internal_state => state%get_internal_state()
             call MAPL_ESMFStateWriteToFile(internal_state,CLOCK,FILENAME, &
-                 FILETYPE, STATE, hdr/=0, oClients = o_Clients, RC=status)
+                 FILETYPE, STATE, hdr/=0, state%grid%write_restart_by_oserver, RC=status)
             _VERIFY(status)
          endif
 
@@ -2431,7 +2429,7 @@ recursive subroutine MAPL_GenericFinalize ( GC, IMPORT, EXPORT, CLOCK, RC )
             endif
 #endif
             call MAPL_ESMFStateWriteToFile(IMPORT,CLOCK,FILENAME, &
-                 FILETYPE, STATE, .FALSE., oClients = o_Clients, RC=status)
+                 FILETYPE, STATE, .FALSE., state%grid%write_restart_by_oserver, RC=status)
             _VERIFY(status)
          endif
 
@@ -2486,7 +2484,7 @@ subroutine checkpoint_export_state(rc)
             endif
 #endif
             call MAPL_ESMFStateWriteToFile(EXPORT,CLOCK,FILENAME, &
-                 FILETYPE, STATE, .FALSE., oClients = o_Clients, RC=status)
+                 FILETYPE, STATE, .FALSE., state%grid%write_restart_by_oserver, RC=status)
             _VERIFY(status)
          endif
          _RETURN(_SUCCESS)
@@ -2772,7 +2770,7 @@ subroutine MAPL_StateRecord( GC, IMPORT, EXPORT, CLOCK, RC )
          end if
          call MAPL_ESMFStateWriteToFile(IMPORT, CLOCK, &
               STATE%RECORD%IMP_FNAME, &
-              FILETYPE, STATE, .FALSE., oClients = o_Clients, &
+              FILETYPE, STATE, .FALSE., state%grid%write_restart_by_oserver, &
               RC=status)
          _VERIFY(status)
       end if
@@ -2789,7 +2787,7 @@ subroutine MAPL_StateRecord( GC, IMPORT, EXPORT, CLOCK, RC )
          internal_state => STATE%get_internal_state()
          call MAPL_ESMFStateWriteToFile(internal_state, CLOCK, &
               STATE%RECORD%INT_FNAME, &
-              FILETYPE, STATE, hdr/=0, oClients = o_Clients, &
+              FILETYPE, STATE, hdr/=0, state%grid%write_restart_by_oserver, &
               RC=status)
          _VERIFY(status)
       end if
@@ -5737,14 +5735,14 @@ end subroutine MAPL_GenericStateClockAdd
    !=============================================================================
    !=============================================================================
 
-   subroutine MAPL_ESMFStateWriteToFile(STATE,CLOCK,FILENAME,FILETYPE,MPL,HDR, oClients,RC)
+   subroutine MAPL_ESMFStateWriteToFile(STATE,CLOCK,FILENAME,FILETYPE,MPL,HDR, write_with_oserver,RC)
       type(ESMF_State),                 intent(INOUT) :: STATE
       type(ESMF_Clock),                 intent(IN   ) :: CLOCK
       character(len=*),                 intent(IN   ) :: FILENAME
       character(LEN=*),                 intent(INout) :: FILETYPE
       type(MAPL_MetaComp),              intent(INOUT) :: MPL
       logical,                          intent(IN   ) :: HDR
-      type (ClientManager), optional,   intent(inout) :: oClients
+      logical, optional,                intent(in   ) :: write_with_oserver
       integer, optional,                intent(  OUT) :: RC
 
       character(len=ESMF_MAXSTR), parameter :: IAm="MAPL_ESMFStateWriteToFile"
@@ -5774,7 +5772,10 @@ subroutine MAPL_ESMFStateWriteToFile(STATE,CLOCK,FILENAME,FILETYPE,MPL,HDR, oCli
       !real(kind=ESMF_KIND_R8),save ::  total_time = 0.0d0
       !logical                               :: amIRoot
       !type (ESMF_VM)                        :: vm
-      logical :: empty
+      logical :: empty, local_write_with_oserver
+
+      local_write_with_oserver=.false.
+      if (present(write_with_oserver)) local_write_with_oserver = write_with_oserver
 
       ! Check if state is empty. If "yes", simply return
       empty = MAPL_IsStateEmpty(state, _RC)
@@ -5991,8 +5992,11 @@ subroutine MAPL_ESMFStateWriteToFile(STATE,CLOCK,FILENAME,FILETYPE,MPL,HDR, oCli
          !itime_beg = MPI_Wtime()
          !_VERIFY(status)
 
-         call MAPL_VarWriteNCPar(filename,STATE,ArrDes,CLOCK, oClients=oClients, RC=status)
-         _VERIFY(status)
+         if (local_write_with_oserver) then
+            call MAPL_VarWriteNCPar(filename,STATE,ArrDes,CLOCK, oClients=o_clients, _RC)
+         else
+            call MAPL_VarWriteNCPar(filename,STATE,ArrDes,CLOCK, _RC)
+         end if
 
          !call MPI_Barrier(mpl%grid%comm, status)
          !_VERIFY(status)
@@ -10496,7 +10500,7 @@ recursive subroutine MAPL_GenericStateSave( GC, IMPORT, EXPORT, CLOCK, RC )
          end if
          call MAPL_ESMFStateWriteToFile(IMPORT, CLOCK, &
               STATE%initial_state%IMP_FNAME, &
-              CFILETYPE, STATE, .FALSE., oClients = o_Clients, &
+              CFILETYPE, STATE, .FALSE.,  write_with_oserver = state%grid%write_restart_by_oserver, &
               RC=status)
          _VERIFY(status)
       end if
@@ -10512,7 +10516,7 @@ recursive subroutine MAPL_GenericStateSave( GC, IMPORT, EXPORT, CLOCK, RC )
          internal_state => STATE%get_internal_state()
          call MAPL_ESMFStateWriteToFile(internal_state, CLOCK, &
               STATE%initial_state%INT_FNAME, &
-              CFILETYPE, STATE, hdr/=0, oClients = o_Clients, &
+              CFILETYPE, STATE, hdr/=0, write_with_oserver = state%grid%write_restart_by_oserver, &
               RC=status)
          _VERIFY(status)
       end if