Update CMEPS to its ESCOMP/master branch 20220606 commit

.github/workflows/extbuild.yml

@@ -19,12 +19,10 @@ jobs:
       CXX: mpicxx
       CPPFLAGS: "-I/usr/include -I/usr/local/include"
       # Versions of all dependencies can be updated here
-      ESMF_VERSION: ESMF_8_2_0_beta_snapshot_14
-      PNETCDF_VERSION: pnetcdf-1.12.2
+      ESMF_VERSION: v8.3.0b13
+      PNETCDF_VERSION: pnetcdf-1.12.3
       NETCDF_FORTRAN_VERSION: v4.5.2
-      # PIO version is awkward
-      PIO_VERSION_DIR: pio2_5_3
-      PIO_VERSION: pio-2.5.3
+      PIO_VERSION: pio2_5_7
     steps:
       - uses: actions/checkout@v2
         # Build the ESMF library, if the cache contains a previous build
@@ -38,12 +36,19 @@ jobs:
         run: |
           sudo apt-get update
           sudo apt-get install gfortran wget openmpi-bin netcdf-bin libopenmpi-dev libnetcdf-dev
+      - id: checkout-ESMF
+        uses: actions/checkout@v3
+        with:
+          repository: esmf-org/esmf
+          path: esmf-src
+          ref: ${{ env.ESMF_VERSION }}
       - id: build-ESMF
         if: steps.cache-esmf.outputs.cache-hit != 'true'
         run: |
-          wget https://github.com/esmf-org/esmf/archive/${{ env.ESMF_VERSION }}.tar.gz
-          tar -xzvf ${{ env.ESMF_VERSION }}.tar.gz
-          pushd esmf-${{ env.ESMF_VERSION }}
+          #wget https://github.com/esmf-org/esmf/archive/${{ env.ESMF_VERSION }}.tar.gz
+          #tar -xzvf ${{ env.ESMF_VERSION }}.tar.gz
+          #pushd esmf-${{ env.ESMF_VERSION }}
+          pushd esmf-src
           export ESMF_DIR=`pwd`
           export ESMF_COMM=openmpi
           export ESMF_YAMLCPP="internal"
@@ -95,14 +100,18 @@ jobs:
             ${{ runner.os }}-${{ env.NETCDF_FORTRAN_VERSION }}-netcdf-fortran
             ${{ runner.os }}-${{ env.PNETCDF_VERSION }}-pnetcdf
 
+      - id: checkout-PIO
+        uses: actions/checkout@v3
+        with:
+          repository: NCAR/ParallelIO
+          path: parallelio-src
+          ref: ${{ env.PIO_VERSION }}
       - name: Build PIO
         if: steps.cache-PIO.outputs.cache-hit != 'true'
         run: |
-          wget https://github.com/NCAR/ParallelIO/releases/download/${{ env.PIO_VERSION_DIR }}/${{ env.PIO_VERSION }}.tar.gz
-          tar -xzvf ${{ env.PIO_VERSION }}.tar.gz
           mkdir build-pio
           pushd build-pio
-          cmake -Wno-dev -DNetCDF_C_LIBRARY=/usr/lib/x86_64-linux-gnu/libnetcdf.so -DNetCDF_C_INCLUDE_DIR=/usr/include -DCMAKE_PREFIX_PATH=/usr -DCMAKE_INSTALL_PREFIX=$HOME/pio -DPIO_HDF5_LOGGING=On -DPIO_USE_MALLOC=On -DPIO_ENABLE_TESTS=Off -DPIO_ENABLE_LOGGING=On -DPIO_ENABLE_EXAMPLES=Off -DPIO_ENABLE_TIMING=Off -DNetCDF_Fortran_PATH=$HOME/netcdf-fortran -DPnetCDF_PATH=$HOME/pnetcdf ../${{ env.PIO_VERSION }}
+          cmake -Wno-dev -DNetCDF_C_LIBRARY=/usr/lib/x86_64-linux-gnu/libnetcdf.so -DNetCDF_C_INCLUDE_DIR=/usr/include -DCMAKE_PREFIX_PATH=/usr -DCMAKE_INSTALL_PREFIX=$HOME/pio -DPIO_HDF5_LOGGING=On -DPIO_USE_MALLOC=On -DPIO_ENABLE_TESTS=Off -DPIO_ENABLE_LOGGING=On -DPIO_ENABLE_EXAMPLES=Off -DPIO_ENABLE_TIMING=Off -DNetCDF_Fortran_PATH=$HOME/netcdf-fortran -DPnetCDF_PATH=$HOME/pnetcdf ../parallelio-src
           make VERBOSE=1
           make install
           popd

cesm/driver/ensemble_driver.F90

@@ -244,7 +244,7 @@ subroutine SetModelServices(ensemble_driver, rc)
           call ReadAttributes(driver, config, "DRIVER_attributes::", rc=rc)
           if (chkerr(rc,__LINE__,u_FILE_u)) return
 
-          call ReadAttributes(driver, config, "DRV_modelio"//trim(inst_suffix)//"::", rc=rc)
+          call ReadAttributes(driver, config, "DRV_modelio::", rc=rc)
           if (chkerr(rc,__LINE__,u_FILE_u)) return
 
           ! Set the driver log to the driver task 0 

cesm/driver/esm.F90

@@ -5,13 +5,12 @@ module ESM
   !-----------------------------------------------------------------------------
 
   use shr_kind_mod , only : r8=>shr_kind_r8, cl=>shr_kind_cl, cs=>shr_kind_cs
-  use shr_log_mod  , only : shrlogunit=> shr_log_unit
   use shr_sys_mod  , only : shr_sys_abort
   use shr_mpi_mod  , only : shr_mpi_bcast
   use shr_mem_mod  , only : shr_mem_init
   use shr_file_mod , only : shr_file_setLogunit
   use esm_utils_mod, only : logunit, mastertask, dbug_flag, chkerr
-  use perf_mod     , only : t_initf
+  use perf_mod     , only : t_initf, t_setLogUnit
 
   implicit none
   private
@@ -220,8 +219,7 @@ subroutine SetModelServices(driver, rc)
     !-------------------------------------------
     ! Timer initialization (has to be after pelayouts are determined)
     !-------------------------------------------
-
-    call t_initf('drv_in', LogPrint=.true., mpicom=global_comm, mastertask=mastertask, MaxThreads=maxthreads)
+    call t_initf('drv_in', LogPrint=.true., LogUnit=logunit, mpicom=global_comm, mastertask=mastertask, MaxThreads=maxthreads)
 
     call ESMF_LogWrite(trim(subname)//": done", ESMF_LOGMSG_INFO)
 
@@ -670,8 +668,11 @@ subroutine AddAttributes(gcomp, driver, config, compid, compname, inst_suffix, n
     if (chkerr(rc,__LINE__,u_FILE_u)) return
     call ReadAttributes(gcomp, config, "ALLCOMP_attributes::", rc=rc)
     if (chkerr(rc,__LINE__,u_FILE_u)) return
-    call ReadAttributes(gcomp, config, trim(compname)//"_modelio"//trim(inst_suffix)//"::", rc=rc)
-    if (chkerr(rc,__LINE__,u_FILE_u)) return
+    call ReadAttributes(gcomp, config, trim(compname)//"_modelio::", rc=rc)
+    if (chkerr(rc,__LINE__,u_FILE_u)) then
+       print *,__FILE__,__LINE__,"ERROR reading ",trim(compname)," modelio from runconfig"
+       return
+    endif
     call ReadAttributes(gcomp, config, "CLOCK_attributes::", rc=rc)
     if (chkerr(rc,__LINE__,u_FILE_u)) return
 
@@ -807,7 +808,7 @@ subroutine esm_init_pelayout(driver, maxthreads, rc)
     use mpi          , only : MPI_COMM_NULL, mpi_comm_size
 #endif
     use mct_mod      , only : mct_world_init
-    use shr_pio_mod  , only : shr_pio_init2
+    use shr_pio_mod  , only : shr_pio_init, shr_pio_component_init
 
 #ifdef MED_PRESENT
     use med_internalstate_mod , only : med_id
@@ -931,6 +932,11 @@ subroutine esm_init_pelayout(driver, maxthreads, rc)
        inst_suffix = ""
     endif
 
+    ! Initialize PIO
+    ! This reads in the pio parameters that are independent of component
+    call shr_pio_init(driver, rc=rc)
+    if (chkerr(rc,__LINE__,u_FILE_u)) return
+
     allocate(comms(componentCount+1), comps(componentCount+1))
     comps(1) = 1
     comms = MPI_COMM_NULL
@@ -1174,12 +1180,14 @@ subroutine esm_init_pelayout(driver, maxthreads, rc)
        if (chkerr(rc,__LINE__,u_FILE_u)) return
 
     enddo
+    ! Read in component dependent PIO parameters and initialize
+    ! IO systems
+    call shr_pio_component_init(driver, size(comps), rc)
+    if (chkerr(rc,__LINE__,u_FILE_u)) return
 
     ! Initialize MCT (this is needed for data models and cice prescribed capability)
     call mct_world_init(componentCount+1, GLOBAL_COMM, comms, comps)
 
-    ! Initialize PIO
-    call shr_pio_init2(comps(2:), compLabels, comp_iamin, comms(2:), comp_comm_iam)
 
     deallocate(petlist, comms, comps, comp_iamin, comp_comm_iam)
 
@@ -1195,6 +1203,8 @@ subroutine esm_set_single_column_attributes(compname, gcomp, rc)
     use netcdf, only : nf90_inq_dimid, nf90_inquire_dimension, nf90_inq_varid, nf90_get_var
     use NUOPC , only : NUOPC_CompAttributeGet, NUOPC_CompAttributeSet, NUOPC_CompAttributeAdd
     use ESMF  , only : ESMF_GridComp, ESMF_GridCompGet, ESMF_VM, ESMF_VMGet, ESMF_SUCCESS
+    use ESMF  , only : ESMF_Mesh, ESMF_MeshCreate, ESMF_FILEFORMAT_ESMFMESH, ESMF_MeshGet, ESMF_MESHLOC_ELEMENT
+    use ESMF  , only : ESMF_Field, ESMF_FieldCreate, ESMF_FieldGet, ESMF_FieldRegridGetArea, ESMF_TYPEKIND_r8
 
     ! input/output variables
     character(len=*)    , intent(in)    :: compname
@@ -1204,14 +1214,24 @@ subroutine esm_set_single_column_attributes(compname, gcomp, rc)
     ! local variables
     type(ESMF_VM)          :: vm
     character(len=CL)      :: single_column_lnd_domainfile
+    character(len=CL)      :: single_column_global_meshfile
     real(r8)               :: scol_lon
     real(r8)               :: scol_lat
     real(r8)               :: scol_area
     integer                :: scol_lndmask
     real(r8)               :: scol_lndfrac
     integer                :: scol_ocnmask
     real(r8)               :: scol_ocnfrac
-    integer                :: i,j,ni,nj
+    integer                :: scol_mesh_n
+    type(ESMF_Mesh)        :: mesh
+    type(ESMF_Field)       :: lfield
+    integer                :: lsize
+    integer                :: spatialdim
+    real(r8), pointer      :: ownedElemCoords(:)
+    real(r8), pointer      :: latMesh(:)
+    real(r8), pointer      :: lonMesh(:)
+    real(r8), pointer      :: dataptr(:)
+    integer                :: i,j,ni,nj,n
     integer                :: ncid
     integer                :: dimid
     integer                :: varid_xc
@@ -1235,7 +1255,6 @@ subroutine esm_set_single_column_attributes(compname, gcomp, rc)
     character(len=*), parameter :: subname= ' (esm_get_single_column_attributes) '
     !-------------------------------------------------------------------------------
 
-
     rc = ESMF_SUCCESS
 
     ! obtain the single column lon and lat
@@ -1247,6 +1266,8 @@ subroutine esm_set_single_column_attributes(compname, gcomp, rc)
     read(cvalue,*) scol_lat
     call NUOPC_CompAttributeGet(gcomp, name='single_column_lnd_domainfile', value=single_column_lnd_domainfile, rc=rc)
     if (ChkErr(rc,__LINE__,u_FILE_u)) return
+    call NUOPC_CompAttributeGet(gcomp, name='mesh_atm', value=single_column_global_meshfile, rc=rc)
+    if (ChkErr(rc,__LINE__,u_FILE_u)) return
     call NUOPC_CompAttributeAdd(gcomp, attrList=(/'scol_spval'/), rc=rc)
     if (chkerr(rc,__LINE__,u_FILE_u)) return
 
@@ -1341,6 +1362,7 @@ subroutine esm_set_single_column_attributes(compname, gcomp, rc)
           do j = 1,nj
              lats(j) = glob_grid(1,j)
           end do
+
           ! find nearest neighbor indices of scol_lon and scol_lat in single_column_lnd_domain file
           ! convert lons array and scol_lon to 0,360 and find index of value closest to 0
           ! and obtain single-column longitude/latitude indices to retrieve
@@ -1380,26 +1402,53 @@ subroutine esm_set_single_column_attributes(compname, gcomp, rc)
                   //' ocean and land mask cannot both be zero')
           end if
 
+          status = nf90_close(ncid)
+          if (status /= nf90_noerr) call shr_sys_abort (trim(subname) //': closing '//&
+               trim(single_column_lnd_domainfile))
+
+          ! Now read in mesh file to get exact values of scol_lon and scol_lat that will be used
+          ! by the models - assume that this occurs only on 1 processor
+          mesh = ESMF_MeshCreate(filename=trim(single_column_global_meshfile), fileformat=ESMF_FILEFORMAT_ESMFMESH, rc=rc)
+          if (chkerr(rc,__LINE__,u_FILE_u)) return
+          call ESMF_MeshGet(mesh, spatialDim=spatialDim, numOwnedElements=lsize, rc=rc)
+          if (ChkErr(rc,__LINE__,u_FILE_u)) return
+          allocate(ownedElemCoords(spatialDim*lsize))
+          allocate(lonMesh(lsize), latMesh(lsize))
+          call ESMF_MeshGet(mesh, ownedElemCoords=ownedElemCoords)
+          if (ChkErr(rc,__LINE__,u_FILE_u)) return
+          do n = 1,lsize
+             lonMesh(n) = ownedElemCoords(2*n-1)
+             latMesh(n) = ownedElemCoords(2*n)
+             if (abs(lonMesh(n) - scol_lon) < 1.e-4 .and. abs(latMesh(n) - scol_lat) < 1.e-4) then
+                scol_mesh_n = n
+                scol_mesh_n = n
+                exit
+             end if
+          end do
+          scol_lon  = lonMesh(scol_mesh_n)
+          scol_lat  = latMesh(scol_mesh_n)
+
+          ! Obtain mesh info areas
+          lfield = ESMF_FieldCreate(mesh, ESMF_TYPEKIND_r8, name='area', meshloc=ESMF_MESHLOC_ELEMENT, rc=rc)
+          if (ChkErr(rc,__LINE__,u_FILE_u)) return
+          call ESMF_FieldRegridGetArea(lfield, rc=rc)
+          if (chkerr(rc,__LINE__,u_FILE_u)) return
+          call ESMF_FieldGet(lfield, farrayPtr=dataptr, rc=rc)
+          if (chkerr(rc,__LINE__,u_FILE_u)) return
+          scol_area = dataptr(scol_mesh_n)
+
+          ! Set single column attribute values for all components
           write(cvalue,*) scol_lon
           call NUOPC_CompAttributeSet(gcomp, name='scol_lon', value=trim(cvalue), rc=rc)
           if (chkerr(rc,__LINE__,u_FILE_u)) return
-
           write(cvalue,*) scol_lat
           call NUOPC_CompAttributeSet(gcomp, name='scol_lat', value=trim(cvalue), rc=rc)
           if (chkerr(rc,__LINE__,u_FILE_u)) return
-
-          write(cvalue,*) ni
-          call NUOPC_CompAttributeSet(gcomp, name='scol_ni', value=trim(cvalue), rc=rc)
+          write(cvalue,*) scol_area
+          call NUOPC_CompAttributeSet(gcomp, name='scol_area', value=trim(cvalue), rc=rc)
           if (chkerr(rc,__LINE__,u_FILE_u)) return
 
-          write(cvalue,*) nj
-          call NUOPC_CompAttributeSet(gcomp, name='scol_nj', value=trim(cvalue), rc=rc)
-          if (chkerr(rc,__LINE__,u_FILE_u)) return
-
-          status = nf90_close(ncid)
-          if (status /= nf90_noerr) call shr_sys_abort (trim(subname) //': closing '//&
-               trim(single_column_lnd_domainfile))
-
+          ! Write out diagnostic info
           write(logunit,'(a,2(f13.5,2x))')trim(subname)//' nearest neighbor scol_lon and scol_lat in '&
                //trim(single_column_lnd_domainfile)//' are ',scol_lon,scol_lat
           if (trim(compname) == 'LND') then
@@ -1411,6 +1460,12 @@ subroutine esm_set_single_column_attributes(compname, gcomp, rc)
           else
              write(logunit,'(a)')trim(subname)//' atm point has unit mask and unit fraction '
           end if
+          write(cvalue,*) ni
+          call NUOPC_CompAttributeSet(gcomp, name='scol_ni', value=trim(cvalue), rc=rc)
+          if (chkerr(rc,__LINE__,u_FILE_u)) return
+          write(cvalue,*) nj
+          call NUOPC_CompAttributeSet(gcomp, name='scol_nj', value=trim(cvalue), rc=rc)
+          if (chkerr(rc,__LINE__,u_FILE_u)) return
 
        else
 
@@ -1423,12 +1478,11 @@ subroutine esm_set_single_column_attributes(compname, gcomp, rc)
           scol_ocnfrac = 1._r8
           scol_area = 1.e30
 
+          write(cvalue,*) 1
           call NUOPC_CompAttributeSet(gcomp, name='scol_ni', value=trim(cvalue), rc=rc)
           if (chkerr(rc,__LINE__,u_FILE_u)) return
-          write(cvalue,*) 1
           call NUOPC_CompAttributeSet(gcomp, name='scol_nj', value=trim(cvalue), rc=rc)
           if (chkerr(rc,__LINE__,u_FILE_u)) return
-          write(cvalue,*) 1
 
           write(logunit,'(a)')' single point mode is active'
           write(logunit,'(a,f13.5,a,f13.5,a)')' scol_lon is ',scol_lon,' and scol_lat is '

cesm/driver/esmApp.F90

@@ -15,7 +15,6 @@ program esmApp
   use mpi
   use NUOPC,           only : NUOPC_FieldDictionarySetup
   use ensemble_driver, only : SetServices
-  use shr_pio_mod,     only : shr_pio_init1
   use shr_sys_mod,     only : shr_sys_abort
 
   implicit none
@@ -44,17 +43,6 @@ program esmApp
 #endif
   COMP_COMM = MPI_COMM_WORLD
 
-  !-----------------------------------------------------------------------------
-  ! Initialize PIO
-  !-----------------------------------------------------------------------------
-
-  ! For planned future use of async io using pio2.  The IO tasks are seperated from the compute tasks here
-  ! and COMP_COMM will be MPI_COMM_NULL on the IO tasks which then call shr_pio_init2 and do not return until
-  ! the model completes.  All other tasks call ESMF_Initialize.  8 is the maximum number of component models
-  ! supported
-
-  call shr_pio_init1(8, "drv_in", COMP_COMM)
-
   !-----------------------------------------------------------------------------
   ! Initialize ESMF
   !-----------------------------------------------------------------------------

cesm/nuopc_cap_share/nuopc_shr_methods.F90

@@ -132,7 +132,7 @@ end subroutine get_component_instance
 !===============================================================================
 
   subroutine set_component_logging(gcomp, mastertask, logunit, shrlogunit, rc)
-
+    use shr_pio_mod, only : shr_pio_log_comp_settings
     ! input/output variables
     type(ESMF_GridComp)  :: gcomp
     logical, intent(in)  :: mastertask
@@ -143,6 +143,8 @@ subroutine set_component_logging(gcomp, mastertask, logunit, shrlogunit, rc)
     ! local variables
     character(len=CL) :: diro
     character(len=CL) :: logfile
+    character(len=CL) :: inst_suffix
+    integer :: inst_index  ! not used here
     !-----------------------------------------------------------------------
 
     rc = ESMF_SUCCESS
@@ -154,14 +156,23 @@ subroutine set_component_logging(gcomp, mastertask, logunit, shrlogunit, rc)
        if (chkerr(rc,__LINE__,u_FILE_u)) return
        call NUOPC_CompAttributeGet(gcomp, name="logfile", value=logfile, rc=rc)
        if (chkerr(rc,__LINE__,u_FILE_u)) return
+       call get_component_instance(gcomp, inst_suffix, inst_index, rc=rc)
+       if (chkerr(rc,__LINE__,u_FILE_u)) return
+       ! Multiinstance logfile name needs a correction
+       if(logfile(4:4) == '_') then
+          logfile = logfile(1:3)//trim(inst_suffix)//logfile(9:)
+       endif
 
        open(newunit=logunit,file=trim(diro)//"/"//trim(logfile))
+       ! Write the PIO settings to the beggining of each component log
+       call shr_pio_log_comp_settings(gcomp, logunit)
+
     else
        logUnit = 6
     endif
-
+    ! TODO: shr_file mod is deprecated and should be removed.
     call shr_file_setLogUnit (logunit)
-
+    
   end subroutine set_component_logging
 
 !===============================================================================

cesm/nuopc_cap_share/seq_drydep_mod.F90

@@ -893,6 +893,7 @@ subroutine seq_drydep_readnl(NLFilename, drydep_nflds)
     !-----------------------------------------------------------------------------
 
     rc = ESMF_SUCCESS
+    drydep_nflds = 0
 
     !--- Open and read namelist ---
     if ( len_trim(NLFilename) == 0  )then

cesm/nuopc_cap_share/shr_ndep_mod.F90

@@ -49,9 +49,9 @@ subroutine shr_ndep_readnl(NLFilename, ndep_nflds)
     character(len=32)  :: ndep_list(maxspc) = '' ! List of ndep species
     integer            :: localpet
     integer            :: mpicom
-    character(*),parameter :: F00   = "('(shr_ndep_read) ',8a)"
-    character(*),parameter :: FI1   = "('(shr_ndep_init) ',a,I2)"
-    character(*),parameter :: subName = '(shr_ndep_read) '
+
+    character(*),parameter :: subName = '(shr_ndep_readnl) '
+    character(*),parameter :: F00   = "('(shr_ndep_readnl) ',8a)"
     ! ------------------------------------------------------------------
 
     namelist /ndep_inparm/ ndep_list