Merge remote-tracking branch 'upstream/develop' into feature/use_jcb_atm

* upstream/develop: Add task to prepare emissions for GEFS (NOAA-EMC#2562) Single Executable for main GDAS JEDI applications (NOAA-EMC#2565) Add nest capability (NOAA-EMC#2545)
danholdaway · May 7, 2024 · b743ea0 · b743ea0
2 parents 6d0b8b6 + 9b6f840
commit b743ea0
Show file tree

Hide file tree

Showing 36 changed files with 1,376 additions and 152 deletions.
diff --git a/env/HERA.env b/env/HERA.env
@@ -49,6 +49,10 @@ elif [[ "${step}" = "prepsnowobs" ]]; then
 
     export APRUN_CALCFIMS="${launcher} -n 1"
 
+elif [[ "${step}" = "prep_emissions" ]]; then
+
+    export APRUN="${launcher} -n 1"
+
 elif [[ "${step}" = "waveinit" ]] || [[ "${step}" = "waveprep" ]] || [[ "${step}" = "wavepostsbs" ]] || [[ "${step}" = "wavepostbndpnt" ]] || [[ "${step}" = "wavepostbndpntbll" ]] || [[ "${step}" = "wavepostpnt" ]]; then
 
     export CFP_MP="YES"

diff --git a/env/HERCULES.env b/env/HERCULES.env
@@ -45,6 +45,10 @@ case ${step} in
 
     export APRUN_CALCFIMS="${launcher} -n 1"
  ;;
+ "prep_emissions")
+
+    export APRUN="${launcher} -n 1"
+ ;;
  "waveinit" | "waveprep" | "wavepostsbs" | "wavepostbndpnt" | "wavepostpnt" | "wavepostbndpntbll")
 
     export CFP_MP="YES"

diff --git a/env/JET.env b/env/JET.env
@@ -37,6 +37,10 @@ elif [[ "${step}" = "prepsnowobs" ]]; then
 
     export APRUN_CALCFIMS="${launcher} -n 1"
 
+elif [[ "${step}" = "prep_emissions" ]]; then
+
+    export APRUN="${launcher} -n 1"
+
 elif [[ "${step}" = "waveinit" ]] || [[ "${step}" = "waveprep" ]] || [[ "${step}" = "wavepostsbs" ]] || [[ "${step}" = "wavepostbndpnt" ]] || [[ "${step}" = "wavepostbndpntbll" ]] || [[ "${step}" = "wavepostpnt" ]]; then
 
     export CFP_MP="YES"

diff --git a/env/ORION.env b/env/ORION.env
@@ -44,6 +44,10 @@ elif [[ "${step}" = "prepsnowobs" ]]; then
 
     export APRUN_CALCFIMS="${launcher} -n 1"
 
+elif [[ "${step}" = "prep_emissions" ]]; then
+
+    export APRUN="${launcher} -n 1"
+
 elif [[ "${step}" = "waveinit" ]] || [[ "${step}" = "waveprep" ]] || [[ "${step}" = "wavepostsbs" ]] || \
     [[ "${step}" = "wavepostbndpnt" ]] || [[ "${step}" = "wavepostpnt" ]] || [[ "${step}" == "wavepostbndpntbll" ]]; then
 

diff --git a/env/S4.env b/env/S4.env
@@ -37,6 +37,10 @@ elif [[ "${step}" = "prepsnowobs" ]]; then
 
     export APRUN_CALCFIMS="${launcher} -n 1"
 
+elif [[ "${step}" = "prep_emissions" ]]; then
+
+    export APRUN="${launcher} -n 1"
+
 elif [[ "${step}" = "waveinit" ]] || [[ "${step}" = "waveprep" ]] || [[ "${step}" = "wavepostsbs" ]] || [[ "${step}" = "wavepostbndpnt" ]] || [[ "${step}" = "wavepostbndpntbll" ]] || [[ "${step}" = "wavepostpnt" ]]; then
 
     export CFP_MP="YES"

diff --git a/env/WCOSS2.env b/env/WCOSS2.env
@@ -31,6 +31,10 @@ elif [[ "${step}" = "prepsnowobs" ]]; then
 
     export APRUN_CALCFIMS="${launcher} -n 1"
 
+elif [[ "${step}" = "prep_emissions" ]]; then
+
+    export APRUN="${launcher} -n 1"
+
 elif [[ "${step}" = "waveinit" ]] || [[ "${step}" = "waveprep" ]] || [[ "${step}" = "wavepostsbs" ]] || [[ "${step}" = "wavepostbndpnt" ]] || [[ "${step}" = "wavepostbndpntbll"  ]] || [[ "${step}" = "wavepostpnt" ]]; then
 
     export USE_CFP="YES"

diff --git a/jobs/JGLOBAL_PREP_EMISSIONS b/jobs/JGLOBAL_PREP_EMISSIONS
@@ -0,0 +1,35 @@
+#! /usr/bin/env bash
+
+source "${HOMEgfs}/ush/preamble.sh"
+source "${HOMEgfs}/ush/jjob_header.sh" -e "prep_emissions" -c "base prep_emissions"
+
+##############################################
+# Set variables used in the script
+##############################################
+# TODO: Set local variables used in this script e.g. GDATE may be needed for previous cycle
+
+##############################################
+# Begin JOB SPECIFIC work
+##############################################
+# Generate COM variables from templates
+# TODO: Add necessary COMIN, COMOUT variables for this job
+
+###############################################################
+# Run relevant script
+EXSCRIPT=${PREP_EMISSIONS_PY:-${SCRgfs}/exglobal_prep_emissions.py}
+${EXSCRIPT}
+status=$?
+(( status != 0 )) && ( echo "FATAL ERROR: Error executing ${EXSCRIPT}, ABORT!"; exit "${status}" )
+
+##############################################
+# End JOB SPECIFIC work
+##############################################
+
+##############################################
+# Final processing
+##############################################
+if [[ -e "${pgmout}" ]] ; then
+  cat "${pgmout}"
+fi
+
+exit 0
diff --git a/jobs/rocoto/prep_emissions.sh b/jobs/rocoto/prep_emissions.sh
@@ -0,0 +1,23 @@
+#! /usr/bin/env bash
+
+source "${HOMEgfs}/ush/preamble.sh"
+
+###############################################################
+# Source UFSDA workflow modules
+source "${HOMEgfs}/ush/load_fv3gfs_modules.sh"
+status=$?
+(( status !=  0 )) && exit "${status}"
+
+export job="prep_emissions"
+export jobid="${job}.$$"
+
+###############################################################
+# setup python path for workflow utilities and tasks
+PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${HOMEgfs}/ush/python"
+export PYTHONPATH
+
+###############################################################
+# Execute the JJOB
+"${HOMEgfs}/jobs/JGLOBAL_PREP_EMISSIONS"
+status=$?
+exit "${status}"
diff --git a/parm/config/gefs/config.base b/parm/config/gefs/config.base
@@ -137,6 +137,13 @@ export DO_AERO="NO"
 export WAVE_CDUMP="" # When to include wave suite: gdas, gfs, or both
 export DOBNDPNT_WAVE="NO" # The GEFS buoys file does not currently have any boundary points
 export FRAC_GRID=".true."
+export DO_NEST="NO" # Whether to run a global-nested domain
+if [[ "${DO_NEST:-NO}" == "YES" ]] ; then
+  export ntiles=7
+  export NEST_OUTPUT_GRID="regional_latlon"
+else
+  export ntiles=6
+fi
 
 # Set operational resolution
 export OPS_RES="C768" # Do not change

diff --git a/parm/config/gefs/config.prep_emissions b/parm/config/gefs/config.prep_emissions
@@ -0,0 +1,11 @@
+#! /usr/bin/env bash
+
+########## config.prep_emissions ##########
+# aerosol emissions preprocessing specific
+
+echo "BEGIN: config.prep_emissions"
+
+# Get task specific resources
+source "${EXPDIR}/config.resources" prep_emissions
+
+echo "END: config.prep_emissions"
diff --git a/parm/config/gefs/config.resources b/parm/config/gefs/config.resources
@@ -69,6 +69,14 @@ case ${step} in
     export memory_waveinit="2GB"
     ;;
 
+  "prep_emissions")
+    export wtime_prep_emissions="00:10:00"
+    export npe_prep_emissions=1
+    export nth_prep_emissions=1
+    export npe_node_prep_emissions=$(( npe_node_max / nth_prep_emissions ))
+    export memory_prep_emissions="1GB"
+    ;;
+
   "fcst" | "efcs")
     export is_exclusive=True
 

diff --git a/parm/config/gfs/config.aeroanl b/parm/config/gfs/config.aeroanl
@@ -18,7 +18,7 @@ export JEDI_FIX_YAML="${PARMgfs}/gdas/aero_jedi_fix.yaml.j2"
 export io_layout_x=@IO_LAYOUT_X@
 export io_layout_y=@IO_LAYOUT_Y@
 
-export JEDIEXE="${EXECgfs}/fv3jedi_var.x"
+export JEDIEXE="${EXECgfs}/gdas.x"
 
 if [[ "${DOIAU}" == "YES" ]]; then
   export aero_bkg_times="3,6,9"

diff --git a/parm/config/gfs/config.atmanl b/parm/config/gfs/config.atmanl
@@ -30,6 +30,6 @@ export layout_y_atmanl=@LAYOUT_Y_ATMANL@
 export io_layout_x=@IO_LAYOUT_X@
 export io_layout_y=@IO_LAYOUT_Y@
 
-export JEDIEXE=${EXECgfs}/fv3jedi_var.x
+export JEDIEXE=${EXECgfs}/gdas.x
 
 echo "END: config.atmanl"
diff --git a/parm/config/gfs/config.atmensanl b/parm/config/gfs/config.atmensanl
@@ -19,6 +19,6 @@ export layout_y_atmensanl=@LAYOUT_Y_ATMENSANL@
 export io_layout_x=@IO_LAYOUT_X@
 export io_layout_y=@IO_LAYOUT_Y@
 
-export JEDIEXE=${EXECgfs}/fv3jedi_letkf.x
+export JEDIEXE=${EXECgfs}/gdas.x
 
 echo "END: config.atmensanl"
diff --git a/parm/config/gfs/config.base b/parm/config/gfs/config.base
@@ -184,6 +184,13 @@ export DO_AERO="NO"
 export WAVE_CDUMP="" # When to include wave suite: gdas, gfs, or both
 export DOBNDPNT_WAVE="NO"
 export FRAC_GRID=".true."
+export DO_NEST="NO" # Whether to run a global-nested domain
+if [[ "${DO_NEST:-NO}" == "YES" ]] ; then
+  export ntiles=7
+  export NEST_OUTPUT_GRID="regional_latlon"
+else
+  export ntiles=6
+fi
 
 # Set operational resolution
 export OPS_RES="C768" # Do not change # TODO: Why is this needed and where is it used?

diff --git a/parm/config/gfs/config.fcst b/parm/config/gfs/config.fcst
@@ -254,6 +254,9 @@ export DO_SKEB=${DO_SKEB:-"NO"}
 export DO_SHUM=${DO_SHUM:-"NO"}
 export DO_LAND_PERT=${DO_LAND_PERT:-"NO"}
 export DO_CA=${DO_CA:-"YES"}
+if [[ "${DO_NEST:-NO}" == "YES" ]] ; then
+  export DO_CA="NO"  # CA does not work with nesting.
+fi
 
 #coupling settings
 export cplmode="ufs.frac"

diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources
@@ -606,6 +606,13 @@ case ${step} in
         [[ "${DO_ICE}" == "YES" ]] && nthreads_cice6=1
       fi
 
+      if (( ntiles > 6 )); then
+        export layout_x_nest=${layout_x_nest:-10}
+        export layout_y_nest=${layout_y_nest:-10}
+        export npx_nest=${npx_nest:-1441}
+        export npy_nest=${npy_nest:-961}
+      fi
+
       # PETS for the atmosphere dycore
       (( FV3PETS = ntasks_fv3 * nthreads_fv3 ))
       echo "FV3 using (nthreads, PETS) = (${nthreads_fv3}, ${FV3PETS})"

diff --git a/parm/config/gfs/config.snowanl b/parm/config/gfs/config.snowanl
@@ -11,7 +11,7 @@ source "${EXPDIR}/config.resources" snowanl
 export OBS_LIST="${PARMgfs}/gdas/snow/obs/lists/gdas_snow.yaml.j2"
 
 # Name of the JEDI executable and its yaml template
-export JEDIEXE="${EXECgfs}/fv3jedi_letkf.x"
+export JEDIEXE="${EXECgfs}/gdas.x"
 export JEDIYAML="${PARMgfs}/gdas/snow/letkfoi/letkfoi.yaml.j2"
 
 # Ensemble member properties

diff --git a/parm/config/gfs/config.stage_ic b/parm/config/gfs/config.stage_ic
@@ -38,4 +38,8 @@ case "${CASE}" in
     ;;
 esac
 
+if [[ "${DO_NEST:-NO}" == "YES" ]] ; then
+  export CPL_ATMIC="GLOBAL-NEST_${CASE}"
+fi
+
 echo "END: config.stage_ic"