Merge branch 'origin/whannah/atm/fix-physgrid-aqua-restart' into next…

… (PR #3204)

This fixes an issues where aqua planet cases were crashing on restart due to
not correctly setting the topography to zero. Instead the realistic topography
was added on restart, which caused the run to crash. This PR resolves this issue
by explicitly resetting the topography to zero when restarting a physgrid run.
This also fixes an issue running certain physgrid configurations, such as ne4pg2,
because the default NTASK was set higher than the number of physics columns.
To get the physgrid specification to match correctly in config_pesall.xml an underscore
was to be added to other grid query strings. This should not affect non-physgrid cases.
The underscore is used because it always separates the atmosphere grid from the other grids.
For example, A typical ne4 grid string is
a%ne4np4_l%ne4np4_oi%ne4np4_r%null_g%null_w%null_z%null_m%oQU240

But the string for ne4pg2 is
a%ne4np4.pg2_l%ne4np4_oi%ne4np4_r%null_g%null_w%null_z%null_m%oQU240

so in order to differentiate the grids we use "ne4np4_" and "ne4np4.pg2" to get
different default PE layouts. Using "ne4np4" without the underscore leads to an error
when running physgrid cases.
After some discussion we decided to also include a change to the default physgrid mapping algorithm
to use the high-order option. This will make all pg2 tests non-BFB.

[BFB] except for SMS_Ln5.ne4pg2_ne4pg2.FC5AV1C-L

cime/config/e3sm/allactive/config_pesall.xml

@@ -185,7 +185,7 @@
       </pes>
     </mach>
   </grid>
-  <grid name="a%ne16np4">
+  <grid name="a%ne16np4_">
     <mach name="any">
       <pes compset="any" pesize="any">
         <comment>none</comment>
@@ -222,7 +222,7 @@
       </pes>
     </mach>
   </grid>
-  <grid name="a%ne240np4">
+  <grid name="a%ne240np4_">
     <mach name="any">
       <pes compset="any" pesize="any">
         <comment>none</comment>
@@ -259,7 +259,7 @@
       </pes>
     </mach>
   </grid>
-  <grid name="a%ne240np4">
+  <grid name="a%ne240np4_">
     <mach name="any">
       <pes compset="CAM.+CLM.+DOCN." pesize="any">
         <comment>none</comment>
@@ -296,7 +296,7 @@
       </pes>
     </mach>
   </grid>
-  <grid name="a%ne120np4">
+  <grid name="a%ne120np4_">
     <mach name="titan|stampede|bluewaters|edison|eos|cori-haswell">
       <pes compset="any" pesize="any">
         <comment>none</comment>
@@ -440,7 +440,7 @@
       </pes>
     </mach>
   </grid>
-  <grid name="a%ne120np4">
+  <grid name="a%ne120np4_">
     <mach name="any">
       <pes compset="CAM.+CLM.+DOCN." pesize="any">
         <comment>none</comment>
@@ -477,7 +477,7 @@
       </pes>
     </mach>
   </grid>
-  <grid name="a%ne120np4">
+  <grid name="a%ne120np4_">
     <mach name="titan|stampede|bluewaters|edison|eos|cori-haswell">
       <pes compset="CAM.+CLM.+DOCN." pesize="any">
         <comment>none</comment>
@@ -514,7 +514,7 @@
       </pes>
     </mach>
   </grid>
-  <grid name="a%ne120np4">
+  <grid name="a%ne120np4_">
     <mach name="titan|stampede|bluewaters">
       <pes compset="CAM.+CLM.+CICE.+POP" pesize="any">
         <comment>none</comment>
@@ -551,7 +551,7 @@
       </pes>
     </mach>
   </grid>
-  <grid name="a%ne30np4">
+  <grid name="a%ne30np4_">
     <mach name="any">
       <pes compset="any" pesize="any">
         <comment>none</comment>
@@ -588,7 +588,7 @@
       </pes>
     </mach>
   </grid>
-  <grid name="a%ne30np4">
+  <grid name="a%ne30np4_">
     <mach name="any">
       <pes compset="CAM.+CLM.+DOCN." pesize="any">
         <comment>none</comment>
@@ -625,7 +625,7 @@
       </pes>
     </mach>
   </grid>
-  <grid name="a%ne30np4">
+  <grid name="a%ne30np4_">
     <mach name="melvin">
       <pes compset="any" pesize="any">
         <comment>none</comment>
@@ -662,7 +662,7 @@
       </pes>
     </mach>
   </grid>
-  <grid name="a%ne30np4">
+  <grid name="a%ne30np4_">
     <mach name="edison">
       <pes compset="any" pesize="any">
         <comment>none</comment>
@@ -699,7 +699,7 @@
       </pes>
     </mach>
   </grid>
-  <grid name="a%ne30np4">
+  <grid name="a%ne30np4_">
     <mach name="cori-haswell">
       <pes compset="any" pesize="L">
         <comment>169 nodes, 19 sypd</comment>
@@ -802,7 +802,7 @@
       </pes>
     </mach>
   </grid>
-  <grid name="a%ne30np4">
+  <grid name="a%ne30np4_">
     <mach name="cori-knl">
       <pes compset="any" pesize="L">
         <comment>cori-knl, generic ne30, 85 nodes, 64x1</comment>
@@ -906,7 +906,7 @@
       </pes>
     </mach>
   </grid>
-  <grid name="a%ne30np4">
+  <grid name="a%ne30np4_">
     <mach name="eos">
       <pes compset="any" pesize="any">
         <comment>none</comment>
@@ -943,7 +943,7 @@
       </pes>
     </mach>
   </grid>
-  <grid name="a%ne30np4">
+  <grid name="a%ne30np4_">
     <mach name="blues">
       <pes compset="XATM|DATM.+CLM" pesize="any">
         <comment>none</comment>
@@ -980,7 +980,7 @@
       </pes>
     </mach>
   </grid>
-  <grid name="a%ne30np4">
+  <grid name="a%ne30np4_">
     <mach name="titan|stampede|bluewaters">
       <pes compset="any" pesize="any">
         <comment>none</comment>
@@ -1017,7 +1017,7 @@
       </pes>
     </mach>
   </grid>
-  <grid name="a%ne30np4">
+  <grid name="a%ne30np4_">
     <mach name="titan|stampede|bluewaters">
       <pes compset="CAM.+CLM.+DOCN." pesize="any">
         <comment>none</comment>
@@ -2541,7 +2541,7 @@
       </pes>
     </mach>
   </grid>
-  <grid name="a%ne16np4">
+  <grid name="a%ne16np4_">
     <mach name="edison|cori-haswell">
       <pes compset="any" pesize="any">
         <comment>none</comment>
@@ -2578,7 +2578,7 @@
       </pes>
     </mach>
   </grid>
-  <grid name="a%ne16np4">
+  <grid name="a%ne16np4_">
     <mach name="cori-knl">
       <pes compset="any" pesize="any">
         <comment>cori-knl, 6 nodes, 64x4, sypd=2.93 (for F-compset)</comment>
@@ -2607,7 +2607,7 @@
       </pes>
     </mach>
   </grid>
-  <grid name="a%ne11np4">
+  <grid name="a%ne11np4_">
     <mach name="cori-knl">
       <pes compset="any" pesize="any">
         <comment>6 nodes, 64x2, sypd=11.1 (for F-compset)</comment>
@@ -4552,7 +4552,7 @@
       </pes>
     </mach>
   </grid>
-  <grid name="a%ne30np4_l%ne30np4_oi%ne30np4">
+  <grid name="a%ne30np4_l%ne30np4_oi%ne30np4_">
     <mach name="anvil|bebop">
       <pes compset="any" pesize="any">
         <comment>ne30_ne30 grid on 40 nodes 36 ppn pure-MPI</comment>
@@ -4810,7 +4810,7 @@
       </pes>
     </mach>
   </grid>
-  <grid name="a%ne30np4">
+  <grid name="a%ne30np4_">
     <mach name="mira|cetus">
       <pes compset="any" pesize="any">
         <comment>ne30 grid on 128x16x4 PEs</comment>
@@ -6109,7 +6109,7 @@
       </pes>
     </mach>
   </grid>
-  <grid name="a%ne30np4">
+  <grid name="a%ne30np4_">
     <mach name="edison">
       <pes compset="CAM.+CLM.+DOCN." pesize="any">
         <comment>"113-node 12x4 F-compset sypd=10.8"</comment>
@@ -6148,7 +6148,7 @@
       </pes>
     </mach>
   </grid>
-  <grid name="a%ne30np4">
+  <grid name="a%ne30np4_">
     <mach name="edison">
       <pes compset="CAM5.+CLM45.+MPASSI.+MPASO.+MOSART.+SGLC.+SWAV" pesize="any">
         <comment>"133 node version gets 6 SYPD. This will be the default and M size"</comment>
@@ -6185,7 +6185,7 @@
       </pes>
     </mach>
   </grid>
-  <grid name="a%ne30np4">
+  <grid name="a%ne30np4_">
     <mach name="edison">
       <pes compset="CAM5.+CLM45.+MPASSI.+MPASO.+MOSART.+SGLC.+SWAV" pesize="S">
         <comment>"39 node version gets 2.1 SYPD."</comment>
@@ -6222,7 +6222,7 @@
       </pes>
     </mach>
   </grid>
-  <grid name="a%ne30np4">
+  <grid name="a%ne30np4_">
     <mach name="edison">
       <pes compset="CAM5.+CLM45.+MPASSI.+MPASO.+MOSART.+SGLC.+SWAV" pesize="L">
         <comment>"285 node version gets 11.5 SYPD"</comment>
@@ -6259,7 +6259,7 @@
       </pes>
     </mach>
   </grid>
-  <grid name="a%ne30np4">
+  <grid name="a%ne30np4_">
     <mach name="cori-haswell">
       <pes compset="CAM5.+CLM45.+MPASSI.+MPASO.+MOSART.+SGLC.+SWAV" pesize="any">
         <comment>"185 nodes, 32x1, ~5sypd (wmod185)"</comment>
@@ -6329,7 +6329,7 @@
       </pes>
     </mach>
   </grid>
-  <grid name="a%ne30np4">
+  <grid name="a%ne30np4_">
     <mach name="cori-knl">
       <pes compset=".*CAM5.+CLM45.+MPASSI.+MPASO.+MOSART.*" pesize="L">
         <comment>"cori-knl ne30 coupled compest on 120 nodes, 64x1 (2 threads CPL/OCN/ICE), (kmod125) sypd=4.1"</comment>
@@ -6758,7 +6758,7 @@
       </pes>
     </mach>
   </grid>
-  <grid name="a%ne120np4">
+  <grid name="a%ne120np4_">
     <mach name="mira">
       <pes compset=".*CAM5.+CLM45.+MPASSI.+MPASO.+MOSART.*" pesize="S">
         <comment>ne120 coupled-compset on 1024 nodes</comment>
@@ -8210,7 +8210,7 @@
       </pes>
     </mach>
   </grid>
-  <grid name="a%ne4np4">
+  <grid name="a%ne4np4_">
     <mach name="cetus">
       <pes compset="any" pesize="any">
         <comment>any compset on ne4 grid</comment>
@@ -8247,7 +8247,7 @@
       </pes>
     </mach>
   </grid>
-  <grid name="a%ne4np4">
+  <grid name="a%ne4np4_">
     <mach name="cori-haswell">
       <pes compset="any" pesize="any">
         <comment>3 nodes, any compset on ne4 grid</comment>
@@ -8274,7 +8274,7 @@
       </pes>
     </mach>
   </grid>
-  <grid name="a%ne4np4">
+  <grid name="a%ne4np4_">
     <mach name="cori-knl">
       <pes compset=".*CAM5.+CLM45.+MPASSI.+MPASO.+MOSART.*" pesize="any">
         <comment>"cori-knl ne4 coupled compest on 6 nodes, sypd=22.9"</comment>
@@ -8421,6 +8421,87 @@
       </pes>
     </mach>
   </grid>
+  <grid name="a%ne4np4.pg1">
+    <mach name="any">
+      <pes compset="any" pesize="any">
+        <comment>any compset on ne4np4.pg1 grid</comment>
+        <ntasks>
+          <ntasks_atm>96</ntasks_atm>
+          <ntasks_lnd>96</ntasks_lnd>
+          <ntasks_rof>96</ntasks_rof>
+          <ntasks_ice>96</ntasks_ice>
+          <ntasks_ocn>96</ntasks_ocn>
+          <ntasks_cpl>96</ntasks_cpl>
+          <ntasks_glc>1</ntasks_glc>
+          <ntasks_wav>1</ntasks_wav>
+        </ntasks>
+        <nthrds>
+          <nthrds_atm>1</nthrds_atm>
+          <nthrds_lnd>1</nthrds_lnd>
+          <nthrds_rof>1</nthrds_rof>
+          <nthrds_ice>1</nthrds_ice>
+          <nthrds_ocn>1</nthrds_ocn>
+          <nthrds_glc>1</nthrds_glc>
+          <nthrds_wav>1</nthrds_wav>
+          <nthrds_cpl>1</nthrds_cpl>
+        </nthrds>
+      </pes>
+    </mach>
+  </grid>
+  <grid name="a%ne4np4.pg2">
+    <mach name="any">
+      <pes compset="any" pesize="any">
+        <comment>any compset on ne4np4.pg2 grid</comment>
+        <ntasks>
+          <ntasks_atm>96</ntasks_atm>
+          <ntasks_lnd>96</ntasks_lnd>
+          <ntasks_rof>96</ntasks_rof>
+          <ntasks_ice>96</ntasks_ice>
+          <ntasks_ocn>96</ntasks_ocn>
+          <ntasks_cpl>96</ntasks_cpl>
+          <ntasks_glc>1</ntasks_glc>
+          <ntasks_wav>1</ntasks_wav>
+        </ntasks>
+        <nthrds>
+          <nthrds_atm>1</nthrds_atm>
+          <nthrds_lnd>1</nthrds_lnd>
+          <nthrds_rof>1</nthrds_rof>
+          <nthrds_ice>1</nthrds_ice>
+          <nthrds_ocn>1</nthrds_ocn>
+          <nthrds_glc>1</nthrds_glc>
+          <nthrds_wav>1</nthrds_wav>
+          <nthrds_cpl>1</nthrds_cpl>
+        </nthrds>
+      </pes>
+    </mach>
+  </grid>
+  <grid name="a%ne4np4.pg3">
+    <mach name="any">
+      <pes compset="any" pesize="any">
+        <comment>any compset on ne4np4.pg3 grid</comment>
+        <ntasks>
+          <ntasks_atm>96</ntasks_atm>
+          <ntasks_lnd>96</ntasks_lnd>
+          <ntasks_rof>96</ntasks_rof>
+          <ntasks_ice>96</ntasks_ice>
+          <ntasks_ocn>96</ntasks_ocn>
+          <ntasks_cpl>96</ntasks_cpl>
+          <ntasks_glc>1</ntasks_glc>
+          <ntasks_wav>1</ntasks_wav>
+        </ntasks>
+        <nthrds>
+          <nthrds_atm>1</nthrds_atm>
+          <nthrds_lnd>1</nthrds_lnd>
+          <nthrds_rof>1</nthrds_rof>
+          <nthrds_ice>1</nthrds_ice>
+          <nthrds_ocn>1</nthrds_ocn>
+          <nthrds_glc>1</nthrds_glc>
+          <nthrds_wav>1</nthrds_wav>
+          <nthrds_cpl>1</nthrds_cpl>
+        </nthrds>
+      </pes>
+    </mach>
+  </grid>
   <grid name=".*oi%oRRS30to10.*">
     <mach name="theta">
       <pes compset=".*MPASSI.+MPASO.+" pesize="any">

components/cam/src/dynamics/se/dyn_grid.F90

@@ -84,6 +84,7 @@ module dyn_grid
     real(kind=r8), allocatable :: lat(:,:)    ! latitude
     real(kind=r8), allocatable :: corner_lon(:,:,:)    ! longitude
     real(kind=r8), allocatable :: corner_lat(:,:,:)    ! latitude
+    real(kind=r8), allocatable :: topo(:,:)   ! topography data
   end type fv_physgrid_struct
   type(fv_physgrid_struct), public, pointer :: fv_physgrid(:) => null()
 
@@ -1538,6 +1539,7 @@ subroutine fv_physgrid_init()
       allocate( fv_physgrid(ie)%lat        (fv_nphys,fv_nphys) )
       allocate( fv_physgrid(ie)%lon        (fv_nphys,fv_nphys) )
       allocate( fv_physgrid(ie)%area       (fv_nphys,fv_nphys) )
+      allocate( fv_physgrid(ie)%topo       (fv_nphys,fv_nphys) )
       allocate( fv_physgrid(ie)%corner_lat(fv_nphys,fv_nphys,4) )
       allocate( fv_physgrid(ie)%corner_lon(fv_nphys,fv_nphys,4) )
     end do ! ie
@@ -1642,6 +1644,7 @@ subroutine fv_physgrid_final()
       deallocate( fv_physgrid(ie)%lat  )
       deallocate( fv_physgrid(ie)%lon  )
       deallocate( fv_physgrid(ie)%area )
+      deallocate( fv_physgrid(ie)%topo )
       deallocate( fv_physgrid(ie)%corner_lat )
       deallocate( fv_physgrid(ie)%corner_lon )
     end do ! ie