Merge bogensch/atm/SCM_REPLAY_quasiB4B (PR #3051)

Improve accuracy of the SCM "Replay" mode

This PR improves the accuracy of the SCM Replay mode. Previous implementation allowed the user to "replicate the behavior" of an E3SM run in SCM, but with temperature errors on the order of ~0.1 K after a day. This PR will provide the user with capability to produce "quasi-bit-for-bit" results with Replay mode, with temperature errors on the order of ~1.e-5 K.

The reason why Replay mode is unable to provide fully B4B results is because the dynamics tendency computed for Replay is not identical to the way that the full model computes its dynamics tendencies (which involves sub-cycling in the SE dynamical core). Thus, there is issue with roundoff. Note that in the past, CAM Replay mode with Eulerian dy-core could produce "B4B" results but only because if the flag to generate Replay mode output was turned on, they updated their T and Q based on the tendency computed for Replay purposes. Thus the run to generate Replay forcing was NOT b4b with a normal production run. We are working on a truly B4B Replay mode for E3SM and a future PR will support this which will involve higher precision computation of the dynamical tendencies.

This PR also renames the old "-camiop" flag to "-e3smreplay" flag to be more current and consistent with E3SM. Other flags are renamed accordingly throughout the code.

Note that in this PR the e3sm_developer test "SMS_R_Ld5.ne4_ne4.FSCM5A97" is an expected fail when comparing results to baseline. This is because where the large-scale vertical velocity is read in and updated had to be moved so that CLUBB could see it at the consistent timestep. It was validated that results for selected tested cases (ARM97, GOAMAZON, DYCOMS-RF01, BOMEX) have very minimal impacts with this fix included.

All other e3sm_developer tests pass.

[BFB] except for SCM test.

cime/src/drivers/mct/main/cime_comp_mod.F90

@@ -1906,7 +1906,7 @@ subroutine cime_init()
     !----------------------------------------------------------
 
     areafact_samegrid = .false.
-#if (defined BFB_CAM_SCAM_IOP )
+#if (defined E3SM_SCM_REPLAY )
     if (.not.samegrid_alo) then
        call shr_sys_abort(subname//' ERROR: samegrid_alo is false - Must run with same atm/ocn/lnd grids when configured for scam iop')
     else

cime/src/drivers/moab/main/cime_comp_mod.F90

@@ -1697,7 +1697,7 @@ subroutine cime_init()
     !----------------------------------------------------------
 
     areafact_samegrid = .false.
-#if (defined BFB_CAM_SCAM_IOP )
+#if (defined E3SM_SCM_REPLAY )
     if (.not.samegrid_alo) then
        call shr_sys_abort(subname//' ERROR: samegrid_alo is false - Must run with same atm/ocn/lnd grids when configured for scam iop')
     else

components/cam/bld/build-namelist

@@ -3988,7 +3988,7 @@ if ($cfg->get('scam')) {
 }
 
 # CAM generates IOP file for SCAM
-if ($cfg->get('camiop')) {
+if ($cfg->get('e3smreplay')) {
     add_default($nl, 'inithist', 'val'=>'CAMIOP');
     add_default($nl, 'ndens', 'val'=>'1,1');
     add_default($nl, 'mfilt', 'val'=>'1,10');

components/cam/bld/config_files/definition.xml

@@ -142,9 +142,9 @@ Modifications that allow perturbation growth testing: 0=off, 1=on.
 Configure CAM for single column mode: 0=off, 1=on.  This option only
 supported for the Eulerian dycore.
 </entry>
-<entry id="camiop" valid_values="0,1" value="0">
-Configure CAM to generate an IOP file that can be used to drive SCAM: 0=no, 1=yes.
-This option only supported for the Eulerian dycore.
+<entry id="e3smreplay" valid_values="0,1" value="0">
+Configure E3SM to generate an IOP file that can be used to drive the SCM: 0=no, 1=yes.
+This option only supported for the Spectral Element dycore.
 </entry>
 <entry id="hgrid" value="">
 Horizontal grid specifier.  The recognized values depend on

components/cam/bld/configure

@@ -167,9 +167,9 @@ OPTIONS
 
   Options relevent to SCAM mode:
 
-     -camiop            Configure CAM to generate an IOP file that can be used to drive SCAM.
-                        This switch only works with the Eulerian dycore.
-     -scam              Compiles model in single column mode.  Only works with Eulerian dycore.
+     -e3smreplay        Configure E3SM to generate an IOP file that can be used to drive the SCM.
+                        This switch only works with the Spectral Element dycore.		
+     -scam              Compiles model in single column mode.  Only works with SE dycore.
 
   CAM parallelization:
 
@@ -309,7 +309,7 @@ GetOptions(
     "cam_bld=s"                 => \$opts{'cam_bld'},
     "cam_exe=s"                 => \$opts{'cam_exe'},
     "cam_exedir=s"              => \$opts{'cam_exedir'},
-    "camiop"                    => \$opts{'camiop'},
+    "e3smreplay"                => \$opts{'e3smreplay'},
     "cc=s"                      => \$opts{'cc'},
     "ccsm_seq"                  => \$opts{'ccsm_seq'},
     "cflags=s"                  => \$opts{'cflags'},
@@ -1223,20 +1223,21 @@ if ($print>=2) { print "CAM single column mode (SCAM): $scam$eol"; }
 
 #-----------------------------------------------------------------------------------------------
 # Generate IOP
-if (defined $opts{'camiop'}) {
-    $cfg_ref->set('camiop', 1);
+if (defined $opts{'e3smreplay'}) {
+    $cfg_ref->set('e3smreplay', 1);
 }
-my $camiop = $cfg_ref->get('camiop') ? "ON" : "OFF";
 
-# The only dycore supported in CAMIOP mode is Spectral Element
-if ($camiop eq 'ON' and $dyn_pkg ne 'se') {
+my $e3smreplay = $cfg_ref->get('e3smreplay') ? "ON" : "OFF";
+
+# The only dycore supported in REPLAY mode is Spectral Element
+if ($e3smreplay eq 'ON'  and $dyn_pkg ne 'se') {
     die <<"EOF";
-**  ERROR: CAMIOP mode only works with Spectral Element dycore.
+**  ERROR: REPLAY mode only works with Spectral Element dycore.
 **         Requested dycore is: $dyn_pkg
 EOF
 }
 
-if ($print>=2) { print "Produce IOP file for SCAM: $camiop$eol"; }
+if ($print>=2) { print "Produce IOP file for SCAM: $e3smreplay$eol"; }
 
 #-----------------------------------------------------------------------------------------------
 # Horizontal grid parameters
@@ -1789,7 +1790,7 @@ $cfg_cppdefs .= " -DMAXPATCH_PFT=numpft+1 -DLSMLAT=1 -DLSMLON=1";
 if ($pergro eq "ON") { $cfg_cppdefs .= " -DPERGRO"; }
 
 # Configure CAM to produce IOP files for SCAM
-if ($camiop eq 'ON') { $cfg_cppdefs .= " -DBFB_CAM_SCAM_IOP"; }
+if ($e3smreplay eq 'ON') { $cfg_cppdefs .= " -DE3SM_SCM_REPLAY"; }
 
 # Resolution parameters for rectangular lat/lon grids
 my $nlon = $cfg_ref->get('nlon');

components/cam/src/chemistry/mozart/mo_drydep.F90

@@ -2036,7 +2036,7 @@ subroutine interp_map( plon, plat, nlon_veg, nlat_veg, npft_veg, lat_veg, lat_ve
                          wetland, vegetation_map, soilw_map, do_soilw )
 
     use mo_constants, only : r2d
-    use scamMod, only : latiop,loniop,scmlat,scmlon,use_camiop
+    use scamMod, only : latiop,loniop,scmlat,scmlon,use_replay
     use shr_scam_mod  , only: shr_scam_getCloseLatLon  ! Standardized system subroutines
     use filenames, only: ncdata
     use dycore, only : dycore_is
@@ -2101,7 +2101,7 @@ subroutine interp_map( plon, plat, nlon_veg, nlat_veg, npft_veg, lat_veg, lat_ve
     ju = plon
 
     if (single_column) then
-       if (use_camiop) then
+       if (use_replay) then
           call getfil (ncdata, ncdata_loc)
           call cam_pio_openfile (piofile, trim(ncdata_loc), PIO_NOWRITE)
           call shr_scam_getCloseLatLon(piofile,scmlat,scmlon,closelat,closelon,latidx,lonidx)

components/cam/src/chemistry/utils/tracer_data.F90

@@ -1188,7 +1188,7 @@ subroutine read_next_trcdata(flds, file, state )
 
     ! If single column do not interpolate aerosol data, just use the step function. 
     !   The exception is if we are trying to "replay" a column from the full model
-    if(single_column .and. .not. use_camiop) then
+    if(single_column .and. .not. use_replay) then
       file%stepTime = .true.
     endif
 

components/cam/src/control/cam_comp.F90

@@ -87,7 +87,7 @@ subroutine cam_init( cam_out, cam_in, mpicom_atm, &
    use physpkg,          only: phys_init, phys_register
 
    use dycore,           only: dycore_is
-#if (defined BFB_CAM_SCAM_IOP)
+#if (defined E3SM_SCM_REPLAY)
    use history_defaults, only: initialize_iop_history
 #endif
 !   use shr_orb_mod,      only: shr_orb_params
@@ -169,7 +169,7 @@ subroutine cam_init( cam_out, cam_in, mpicom_atm, &
      ! Commented out the hub2atm_alloc call as it overwrite cam_in, which is undesirable. The fields in cam_in are necessary for getting BFB restarts
 	 ! There are no side effects of commenting out this call as this call allocates cam_in and cam_in allocation has already been done in cam_init
      !call hub2atm_alloc( cam_in )
-#if (defined BFB_CAM_SCAM_IOP)
+#if (defined E3SM_SCM_REPLAY)
       call initialize_iop_history()
 #endif
    end if

components/cam/src/control/cam_history.F90

@@ -3408,7 +3408,7 @@ subroutine h_inquire (t)
       ierr=pio_inq_varid (tape(t)%File,'time_bnds',   tape(t)%tbndid)
       ierr=pio_inq_varid (tape(t)%File,'date_written',tape(t)%date_writtenid)
       ierr=pio_inq_varid (tape(t)%File,'time_written',tape(t)%time_writtenid)
-#if ( defined BFB_CAM_SCAM_IOP )
+#if ( defined E3SM_SCM_REPLAY )
       ierr=pio_inq_varid (tape(t)%File,'tsec    ',tape(t)%tsecid)
       ierr=pio_inq_varid (tape(t)%File,'bdate   ',tape(t)%bdateid)
 #endif
@@ -3761,8 +3761,8 @@ subroutine h_define (t, restart)
     str = 'CF-1.0'
     ierr=pio_put_att (tape(t)%File, PIO_GLOBAL, 'Conventions', trim(str))
     ierr=pio_put_att (tape(t)%File, PIO_GLOBAL, 'source', 'CAM')
-#if ( defined BFB_CAM_SCAM_IOP )
-    ierr=pio_put_att (tape(t)%File, PIO_GLOBAL, 'CAM_GENERATED_FORCING','create SCAM IOP dataset')
+#if defined (E3SM_SCM_REPLAY)
+    ierr=pio_put_att (tape(t)%File, PIO_GLOBAL, 'E3SM_GENERATED_FORCING','create SCM IOP dataset')
 #endif
     ierr=pio_put_att (tape(t)%File, PIO_GLOBAL, 'case',caseid)
     ierr=pio_put_att (tape(t)%File, PIO_GLOBAL, 'title',ctitle)
@@ -3822,7 +3822,7 @@ subroutine h_define (t, restart)
       str = 'base date (YYYYMMDD)'
       ierr=pio_put_att (tape(t)%File, tape(t)%nbdateid, 'long_name', trim(str))
 
-#if ( defined BFB_CAM_SCAM_IOP )
+#if ( defined E3SM_SCM_REPLAY )
       ierr=pio_def_var (tape(t)%File,'bdate',PIO_INT,tape(t)%bdateid)
       str = 'base date (YYYYMMDD)'
       ierr=pio_put_att (tape(t)%File, tape(t)%bdateid, 'long_name', trim(str))
@@ -3899,7 +3899,7 @@ subroutine h_define (t, restart)
       end if
 
 
-#if ( defined BFB_CAM_SCAM_IOP )
+#if ( defined E3SM_SCM_REPLAY )
       ierr=pio_def_var (tape(t)%File,'tsec ',pio_int,(/timdim/), tape(t)%tsecid)
       str = 'current seconds of current date needed for scam'
       ierr=pio_put_att (tape(t)%File, tape(t)%tsecid, 'long_name', trim(str))
@@ -4149,7 +4149,7 @@ subroutine h_define (t, restart)
 
       ierr = pio_put_var(tape(t)%File, tape(t)%nbdateid, (/nbdate/))
       call cam_pio_handle_error(ierr, 'h_define: cannot put nbdate')
-#if ( defined BFB_CAM_SCAM_IOP )
+#if ( defined E3SM_SCM_REPLAY )
       ierr = pio_put_var(tape(t)%File, tape(t)%bdateid, (/nbdate/))
       call cam_pio_handle_error(ierr, 'h_define: cannot put bdate')
 #endif
@@ -4530,7 +4530,7 @@ subroutine wshist (rgnht_in)
     character(len=max_string_len) :: fname ! Filename
     logical :: prev              ! Label file with previous date rather than current
     integer :: ierr
-#if ( defined BFB_CAM_SCAM_IOP )
+#if ( defined E3SM_SCM_REPLAY )
     integer :: tsec             ! day component of current time
     integer :: dtime            ! seconds component of current time
 #endif
@@ -4674,7 +4674,7 @@ subroutine wshist (rgnht_in)
           end if
 
           ierr = pio_put_var (tape(t)%File, tape(t)%datesecid,(/start/),(/count1/),(/ncsec/))
-#if ( defined BFB_CAM_SCAM_IOP )
+#if ( defined E3SM_SCM_REPLAY )
           dtime = get_step_size()
           tsec=dtime*nstep
           ierr = pio_put_var (tape(t)%File, tape(t)%tsecid,(/start/),(/count1/),(/tsec/))

components/cam/src/control/cam_history_support.F90

@@ -196,7 +196,7 @@ module cam_history_support
     type(var_desc_t) :: f12vmrid         ! var id for f12 volume mixing ratio
     type(var_desc_t) :: sol_tsiid        ! var id for total solar irradiance (W/m2)
     type(var_desc_t) :: datesecid        ! var id for curent seconds of current date
-#if ( defined BFB_CAM_SCAM_IOP )
+#if ( defined E3SM_SCM_REPLAY )
     type(var_desc_t) :: bdateid         ! var id for base date
     type(var_desc_t) :: tsecid        ! var id for curent seconds of current date
 #endif

components/cam/src/control/getinterpnetcdfdata.F90

@@ -189,7 +189,7 @@ subroutine getinterpncdata( NCID, camlat, camlon, TimeIdx, &
       outdata(1) = tmp(1)
       return                 ! no need to do interpolation 
    endif
-!   if ( use_camiop .and. nlev.eq.plev) then
+!   if ( use_replay .and. nlev.eq.plev) then
    if ( nlev.eq.plev .or. nlev.eq.plev+1) then
       outData(:nlev)= tmp(:nlev)! no need to do interpolation 
    else

components/cam/src/control/history_defaults.F90

@@ -22,7 +22,7 @@ module history_defaults
 
   public :: bldfld
 
-#if ( defined BFB_CAM_SCAM_IOP )
+#if ( defined E3SM_SCM_REPLAY )
   public :: initialize_iop_history
 #endif
 
@@ -60,7 +60,7 @@ subroutine bldfld ()
   end subroutine bldfld
 
 !#######################################################################
-#if ( defined BFB_CAM_SCAM_IOP  )
+#if ( defined E3SM_SCM_REPLAY  )
   subroutine initialize_iop_history()
 !
 ! !DESCRIPTION: