Replace old post script with initial post refactor

Refactors the post job to only run the offline post portion.
Interpolation of grib files will be moved to a new product
generation job using less resources. So will WAFS/GOES post,
though those will reuse the base python script.

New script is primarily python-based. The shell exscript is
responsible for providing the python script with the appropriate
YAML settings file for the cdump and forecast hour. The YAML
file, located in `parm/post`, is responsible for providing all
of the needed settings (many via environment variables).

This is still a work in progress. Only offline post for gfs
forecast hours is confirmed to work. Support for other cdumps,
as well as WAFS and GOES, will be implemented later.

Refs: NOAA-EMC#270

jobs/JGLOBAL_ATMOS_POST

@@ -2,6 +2,7 @@
 
 source "${HOMEgfs}/ush/preamble.sh"
 
+
 ##############################################
 # make temp directory
 ##############################################
@@ -78,10 +79,9 @@ fi
 ##############################################
 export APRUNP=${APRUN:-${APRUN_NP}}
 export RERUN=${RERUN:-NO}
-export HOMECRTM=${HOMECRTM:-${PACKAGEROOT}/lib/crtm/${crtm_ver}}
-export FIXCRTM=${CRTM_FIX:-${HOMECRTM}/fix}
+
 export PARMpost=${PARMpost:-${HOMEgfs}/parm/post}
-export INLINE_POST=${WRITE_DOPOST:-".false."}
+# export INLINE_POST=${WRITE_DOPOST:-".false."}
 
 export COMIN=${COMIN:-${ROTDIR}/${RUN}.${PDY}/${cyc}/atmos}
 export COMOUT=${COMOUT:-${ROTDIR}/${RUN}.${PDY}/${cyc}/atmos}
@@ -90,46 +90,14 @@ export COMOUT=${COMOUT:-${ROTDIR}/${RUN}.${PDY}/${cyc}/atmos}
 [[ ! -d "${COMOUT}" ]] && mkdir -m 775 -p "${COMOUT}"
 # shellcheck disable=
 
-if [ "${RUN}" = gfs ];then
-  export FHOUT_PGB=${FHOUT_GFS:-3} #Output frequency of gfs pgb file at 1.0 and 0.5 deg.
-fi
-if [ "${RUN}" = gdas ]; then
-  export IGEN_GFS="gfs_avn"
-  export IGEN_ANL="anal_gfs"
-  export IGEN_FCST="gfs_avn"
-  export IGEN_GDAS_ANL="anal_gdas"
-  export FHOUT_PGB=${FHOUT:-1} #Output frequency of gfs pgb file at 1.0 and 0.5 deg.
-fi
-
-if [ "${GRIBVERSION}" = grib2 ]; then
-  export IGEN_ANL="anal_gfs"
-  export IGEN_FCST="gfs_avn"
-  export IGEN_GFS="gfs_avn"
-fi
-
-#######################################
-# Specify Restart File Name to Key Off
-#######################################
-export restart_file=${COMIN}/${RUN}.t${cyc}z.logf
-
-####################################
-# Specify Timeout Behavior of Post
-#
-# SLEEP_TIME - Amount of time to wait for
-#              a restart file before exiting
-# SLEEP_INT  - Amount of time to wait between
-#              checking for restart files
-####################################
-export SLEEP_TIME=900
-export SLEEP_INT=5
-
-
 ###############################################################
 # Run relevant exglobal script
-
-"${SCRgfs}/ex${RUN}_atmos_post.sh"
+###############################################################
+"${HOMEgfs}/scripts/exglobal_atmos_post.sh"
 status=$?
-(( status != 0 )) && exit "${status}"
+if (( status != 0 )); then
+  exit "${status}"
+fi  
 
 ##############################################
 # End JOB SPECIFIC work

jobs/rocoto/post.sh

@@ -25,10 +25,9 @@ fi
 
 
 #---------------------------------------------------------------
-for fhr in ${fhrlst}; do
-    export post_times=${fhr}
-    export job="post${post_times}"
-    export jobid="${job}.$$"
+
+for fhr in $fhrlst; do
+    export fhr
     ${HOMEgfs}/jobs/JGLOBAL_ATMOS_POST
     status=$?
     [[ ${status} -ne 0 ]] && exit ${status}

modulefiles/module_base.orion.lua

@@ -57,7 +57,7 @@ prepend_path("MODULEPATH", "/work2/noaa/global/wkolczyn/save/hpc-stack/modulefil
 load(pathJoin("hpc", "1.2.0"))
 load(pathJoin("hpc-intel", "2018.4"))
 load(pathJoin("hpc-miniconda3", "4.6.14"))
-load(pathJoin("ufswm", "1.0.0"))
+load(pathJoin("gfs_workflow", "1.0.0"))
 load(pathJoin("met", "9.1"))
 load(pathJoin("metplus", "3.1"))
 

parm/config/config.post

@@ -13,30 +13,15 @@ echo "BEGIN: config.post"
 
 # No. of concurrent post jobs [0 implies sequential]
 export NPOSTGRP=42
-export OUTTYP=4
-export MODEL_OUT_FORM=binarynemsiompiio
-if [ $OUTPUT_FILE = "netcdf" ]; then
-    export MODEL_OUT_FORM=netcdfpara
-fi
 
 # Post driver job that calls gfs_post.sh and downstream jobs
 export POSTJJOBSH="$HOMEpost/jobs/JGLOBAL_POST"
-export GFSDOWNSH="$HOMEpost/ush/fv3gfs_downstream_nems.sh"
-export GFSDWNSH="$HOMEpost/ush/fv3gfs_dwn_nems.sh"
-
-export POSTGPSH="$HOMEpost/ush/gfs_post.sh"
-export POSTGPEXEC="$HOMEpost/exec/upp.x"
 export GOESF=NO                              # goes image
-export FLXF=YES                               # grib2 flux file written by post
 
 export npe_postgp=$npe_post
 export nth_postgp=1
 
-export GFS_DOWNSTREAM="YES"
-export downset=2
-export npe_dwn=24
 
-export GRIBVERSION='grib2'
 export SENDCOM="YES"
 
 echo "END: config.post"

parm/post/atm_post.yaml

@@ -0,0 +1,20 @@
+work_dir: !ENV ${DATA}
+tmpl_file: !ENV ${PARMpost}/post.nml.j2
+mp_file: !ENV ${PARMpost}/nam_micro_lookup.dat
+grib_out: !ENV ${COMOUT}/${RUN}.${cycle}.master.grb2f${fhr}
+grib_idx_out: !ENV ${COMOUT}/${RUN}.${cycle}.master.grb2if${fhr}
+exe_log_file: !ENV ${pgmout}
+out_form: 'netcdfpara'
+cdate: !ENV ${CDATE}
+fhr: !ENV ${fhr}
+grib_version: 'grib2'
+grib_table: !ENV ${g2tmpl_ROOT}/share/params_grib2_tbl_new
+post_variables: 'KPO=57,PO=1000.,975.,950.,925.,900.,875.,850.,825.,800.,775.,750.,725.,700.,675.,650.,625.,600.,575.,550.,525.,500.,475.,450.,425.,400.,375.,350.,325.,300.,275.,250.,225.,200.,175.,150.,125.,100.,70.,50.,40.,30.,20.,15.,10.,7.,5.,3.,2.,1.,0.7,0.4,0.2,0.1,0.07,0.04,0.02,0.01,'
+post_exe: !ENV ${HOMEgfs}/exec/upp.x
+grib_idx_exe: !ENV ${GRB2INDEX}
+mpi_run: !ENV ${APRUNP}
+sleep_max: 900
+sleep_interval: 5
+send_com: !ENV ${SENDCOM}
+send_dbn: !ENV ${SENDDBN}
+dbn_alert: !ENV ${DBNROOT}/bin/dbn_alert

parm/post/atm_post_gfs_anl.yaml

@@ -0,0 +1,15 @@
+include:
+  - !INC ${PARMpost}/atm_post.yaml
+
+atm_file: !ENV ${COMIN}/${RUN}.${cycle}.atmanl.nc
+sfc_file: !ENV ${COMIN}/${RUN}.${cycle}.sfcanl.nc
+trigger_file: !ENV ${COMIN}/${RUN}.${cycle}.atmanl.nc
+
+igen: anal_gfs
+flat_file: !ENV ${PARMpost}/postxconfig-NT-GFS-ANL.txt
+ctrl_file: !ENV ${PARMpost}/postcntrl_gfs_anl.xml
+grib2_table: !ENV ${g2tmpl_ROOT}/share/params_grib2_tbl_new
+
+dbn_signals:
+  GFS_SA: !ENV ${COMIN}/${RUN}.${cycle}.atmanl.nc
+  GFS_MSC_sfcanal: !ENV ${COMIN}/${RUN}.${cycle}.sfcanl.nc

parm/post/atm_post_gfs_f000.yaml

@@ -0,0 +1,15 @@
+include:
+  - !INC ${PARMpost}/atm_post.yaml
+
+atm_file: !ENV ${COMIN}/${RUN}.${cycle}.atmf000.nc
+sfc_file: !ENV ${COMIN}/${RUN}.${cycle}.sfcf000.nc
+trigger_file: !ENV ${COMIN}/${CDUMP}.t${cyc}z.logf000.txt
+
+igen: gfs_avn
+flat_file: !ENV ${PARMpost}/postxconfig-NT-GFS-F00.txt
+ctrl_file: !ENV ${PARMpost}/postcntrl_gfs_f00.xml
+grib2_table: !ENV ${g2tmpl_ROOT}/share/params_grib2_tbl_new
+
+dbn_signals:
+  GFS_SA: !ENV ${COMIN}/${RUN}.${cycle}.atmf000.nc
+  GFS_MSC_sfcanal: !ENV ${COMIN}/${RUN}.${cycle}.sfcf000.nc

parm/post/atm_post_gfs_fhr.yaml

@@ -0,0 +1,11 @@
+include:
+  - !INC ${PARMpost}/atm_post.yaml
+
+atm_file: !ENV ${COMIN}/${RUN}.${cycle}.atmf${fhr}.nc
+sfc_file: !ENV ${COMIN}/${RUN}.${cycle}.sfcf${fhr}.nc
+trigger_file: !ENV ${COMIN}/${CDUMP}.t${cyc}z.logf${fhr}.txt
+
+igen: gfs_avn
+flat_file: !ENV ${PARMpost}/postxconfig-NT-GFS.txt
+ctrl_file: !ENV ${PARMpost}/postcntrl_gfs.xml
+grib2_table: !ENV ${g2tmpl_ROOT}/share/params_grib2_tbl_new

parm/post/post.nml.j2

@@ -0,0 +1,12 @@
+&model_inputs
+	fileName='atm_file'
+	fileNameFlux='sfc_file'
+	IOFORM='{{out_form}}'
+	grib='{{grib_version}}'
+	DateStr='%Y-%m-%d_%H:%M:%S'
+	MODELNAME='GFS'
+/
+
+&NAMPGB
+	{{post_variables}}
+/

scripts/exglobal_atmos_post.sh

@@ -0,0 +1,17 @@
+#! /usr/bin/env bash
+
+source "${HOMEgfs}/ush/preamble.sh"
+
+# Determine which yaml file to use
+case ${fhr} in
+	anl) yaml_cat='anl' ;;
+	000) yaml_cat='f000';;
+	*)   yaml_cat='fhr' ;;
+esac
+post_settings="${PARMpost}/atm_post_${CDUMP}_${yaml_cat}.yaml"
+
+# Run post
+"${HOMEgfs}/ush/atm_post.py" "${post_settings}"
+err=$?
+
+exit ${err}