diff --git a/jobs/JGLOBAL_ATMOS_POST b/jobs/JGLOBAL_ATMOS_POST index 534eb62872..b8c5c03d7f 100755 --- a/jobs/JGLOBAL_ATMOS_POST +++ b/jobs/JGLOBAL_ATMOS_POST @@ -2,6 +2,7 @@ source "${HOMEgfs}/ush/preamble.sh" + ############################################## # make temp directory ############################################## @@ -78,10 +79,9 @@ fi ############################################## export APRUNP=${APRUN:-${APRUN_NP}} export RERUN=${RERUN:-NO} -export HOMECRTM=${HOMECRTM:-${PACKAGEROOT}/lib/crtm/${crtm_ver}} -export FIXCRTM=${CRTM_FIX:-${HOMECRTM}/fix} + export PARMpost=${PARMpost:-${HOMEgfs}/parm/post} -export INLINE_POST=${WRITE_DOPOST:-".false."} +# export INLINE_POST=${WRITE_DOPOST:-".false."} export COMIN=${COMIN:-${ROTDIR}/${RUN}.${PDY}/${cyc}/atmos} export COMOUT=${COMOUT:-${ROTDIR}/${RUN}.${PDY}/${cyc}/atmos} @@ -90,46 +90,14 @@ export COMOUT=${COMOUT:-${ROTDIR}/${RUN}.${PDY}/${cyc}/atmos} [[ ! -d "${COMOUT}" ]] && mkdir -m 775 -p "${COMOUT}" # shellcheck disable= -if [ "${RUN}" = gfs ];then - export FHOUT_PGB=${FHOUT_GFS:-3} #Output frequency of gfs pgb file at 1.0 and 0.5 deg. -fi -if [ "${RUN}" = gdas ]; then - export IGEN_GFS="gfs_avn" - export IGEN_ANL="anal_gfs" - export IGEN_FCST="gfs_avn" - export IGEN_GDAS_ANL="anal_gdas" - export FHOUT_PGB=${FHOUT:-1} #Output frequency of gfs pgb file at 1.0 and 0.5 deg. -fi - -if [ "${GRIBVERSION}" = grib2 ]; then - export IGEN_ANL="anal_gfs" - export IGEN_FCST="gfs_avn" - export IGEN_GFS="gfs_avn" -fi - -####################################### -# Specify Restart File Name to Key Off -####################################### -export restart_file=${COMIN}/${RUN}.t${cyc}z.logf - -#################################### -# Specify Timeout Behavior of Post -# -# SLEEP_TIME - Amount of time to wait for -# a restart file before exiting -# SLEEP_INT - Amount of time to wait between -# checking for restart files -#################################### -export SLEEP_TIME=900 -export SLEEP_INT=5 - - ############################################################### # Run relevant exglobal script - -"${SCRgfs}/ex${RUN}_atmos_post.sh" +############################################################### +"${HOMEgfs}/scripts/exglobal_atmos_post.sh" status=$? -(( status != 0 )) && exit "${status}" +if (( status != 0 )); then + exit "${status}" +fi ############################################## # End JOB SPECIFIC work diff --git a/jobs/rocoto/post.sh b/jobs/rocoto/post.sh index bc7da5709a..9549b3e74f 100755 --- a/jobs/rocoto/post.sh +++ b/jobs/rocoto/post.sh @@ -25,10 +25,9 @@ fi #--------------------------------------------------------------- -for fhr in ${fhrlst}; do - export post_times=${fhr} - export job="post${post_times}" - export jobid="${job}.$$" + +for fhr in $fhrlst; do + export fhr ${HOMEgfs}/jobs/JGLOBAL_ATMOS_POST status=$? [[ ${status} -ne 0 ]] && exit ${status} diff --git a/modulefiles/module_base.orion.lua b/modulefiles/module_base.orion.lua index e838355555..ff78c3825e 100644 --- a/modulefiles/module_base.orion.lua +++ b/modulefiles/module_base.orion.lua @@ -57,7 +57,7 @@ prepend_path("MODULEPATH", "/work2/noaa/global/wkolczyn/save/hpc-stack/modulefil load(pathJoin("hpc", "1.2.0")) load(pathJoin("hpc-intel", "2018.4")) load(pathJoin("hpc-miniconda3", "4.6.14")) -load(pathJoin("ufswm", "1.0.0")) +load(pathJoin("gfs_workflow", "1.0.0")) load(pathJoin("met", "9.1")) load(pathJoin("metplus", "3.1")) diff --git a/parm/config/config.post b/parm/config/config.post index cdea64e2a3..780abd8a6b 100755 --- a/parm/config/config.post +++ b/parm/config/config.post @@ -13,30 +13,15 @@ echo "BEGIN: config.post" # No. of concurrent post jobs [0 implies sequential] export NPOSTGRP=42 -export OUTTYP=4 -export MODEL_OUT_FORM=binarynemsiompiio -if [ $OUTPUT_FILE = "netcdf" ]; then - export MODEL_OUT_FORM=netcdfpara -fi # Post driver job that calls gfs_post.sh and downstream jobs export POSTJJOBSH="$HOMEpost/jobs/JGLOBAL_POST" -export GFSDOWNSH="$HOMEpost/ush/fv3gfs_downstream_nems.sh" -export GFSDWNSH="$HOMEpost/ush/fv3gfs_dwn_nems.sh" - -export POSTGPSH="$HOMEpost/ush/gfs_post.sh" -export POSTGPEXEC="$HOMEpost/exec/upp.x" export GOESF=NO # goes image -export FLXF=YES # grib2 flux file written by post export npe_postgp=$npe_post export nth_postgp=1 -export GFS_DOWNSTREAM="YES" -export downset=2 -export npe_dwn=24 -export GRIBVERSION='grib2' export SENDCOM="YES" echo "END: config.post" diff --git a/parm/post/atm_post.yaml b/parm/post/atm_post.yaml new file mode 100644 index 0000000000..08a07dc16f --- /dev/null +++ b/parm/post/atm_post.yaml @@ -0,0 +1,20 @@ +work_dir: !ENV ${DATA} +tmpl_file: !ENV ${PARMpost}/post.nml.j2 +mp_file: !ENV ${PARMpost}/nam_micro_lookup.dat +grib_out: !ENV ${COMOUT}/${RUN}.${cycle}.master.grb2f${fhr} +grib_idx_out: !ENV ${COMOUT}/${RUN}.${cycle}.master.grb2if${fhr} +exe_log_file: !ENV ${pgmout} +out_form: 'netcdfpara' +cdate: !ENV ${CDATE} +fhr: !ENV ${fhr} +grib_version: 'grib2' +grib_table: !ENV ${g2tmpl_ROOT}/share/params_grib2_tbl_new +post_variables: 'KPO=57,PO=1000.,975.,950.,925.,900.,875.,850.,825.,800.,775.,750.,725.,700.,675.,650.,625.,600.,575.,550.,525.,500.,475.,450.,425.,400.,375.,350.,325.,300.,275.,250.,225.,200.,175.,150.,125.,100.,70.,50.,40.,30.,20.,15.,10.,7.,5.,3.,2.,1.,0.7,0.4,0.2,0.1,0.07,0.04,0.02,0.01,' +post_exe: !ENV ${HOMEgfs}/exec/upp.x +grib_idx_exe: !ENV ${GRB2INDEX} +mpi_run: !ENV ${APRUNP} +sleep_max: 900 +sleep_interval: 5 +send_com: !ENV ${SENDCOM} +send_dbn: !ENV ${SENDDBN} +dbn_alert: !ENV ${DBNROOT}/bin/dbn_alert diff --git a/parm/post/atm_post_gfs_anl.yaml b/parm/post/atm_post_gfs_anl.yaml new file mode 100644 index 0000000000..aec361eea8 --- /dev/null +++ b/parm/post/atm_post_gfs_anl.yaml @@ -0,0 +1,15 @@ +include: + - !INC ${PARMpost}/atm_post.yaml + +atm_file: !ENV ${COMIN}/${RUN}.${cycle}.atmanl.nc +sfc_file: !ENV ${COMIN}/${RUN}.${cycle}.sfcanl.nc +trigger_file: !ENV ${COMIN}/${RUN}.${cycle}.atmanl.nc + +igen: anal_gfs +flat_file: !ENV ${PARMpost}/postxconfig-NT-GFS-ANL.txt +ctrl_file: !ENV ${PARMpost}/postcntrl_gfs_anl.xml +grib2_table: !ENV ${g2tmpl_ROOT}/share/params_grib2_tbl_new + +dbn_signals: + GFS_SA: !ENV ${COMIN}/${RUN}.${cycle}.atmanl.nc + GFS_MSC_sfcanal: !ENV ${COMIN}/${RUN}.${cycle}.sfcanl.nc diff --git a/parm/post/atm_post_gfs_f000.yaml b/parm/post/atm_post_gfs_f000.yaml new file mode 100644 index 0000000000..cd93113931 --- /dev/null +++ b/parm/post/atm_post_gfs_f000.yaml @@ -0,0 +1,15 @@ +include: + - !INC ${PARMpost}/atm_post.yaml + +atm_file: !ENV ${COMIN}/${RUN}.${cycle}.atmf000.nc +sfc_file: !ENV ${COMIN}/${RUN}.${cycle}.sfcf000.nc +trigger_file: !ENV ${COMIN}/${CDUMP}.t${cyc}z.logf000.txt + +igen: gfs_avn +flat_file: !ENV ${PARMpost}/postxconfig-NT-GFS-F00.txt +ctrl_file: !ENV ${PARMpost}/postcntrl_gfs_f00.xml +grib2_table: !ENV ${g2tmpl_ROOT}/share/params_grib2_tbl_new + +dbn_signals: + GFS_SA: !ENV ${COMIN}/${RUN}.${cycle}.atmf000.nc + GFS_MSC_sfcanal: !ENV ${COMIN}/${RUN}.${cycle}.sfcf000.nc \ No newline at end of file diff --git a/parm/post/atm_post_gfs_fhr.yaml b/parm/post/atm_post_gfs_fhr.yaml new file mode 100644 index 0000000000..c72886c8b8 --- /dev/null +++ b/parm/post/atm_post_gfs_fhr.yaml @@ -0,0 +1,11 @@ +include: + - !INC ${PARMpost}/atm_post.yaml + +atm_file: !ENV ${COMIN}/${RUN}.${cycle}.atmf${fhr}.nc +sfc_file: !ENV ${COMIN}/${RUN}.${cycle}.sfcf${fhr}.nc +trigger_file: !ENV ${COMIN}/${CDUMP}.t${cyc}z.logf${fhr}.txt + +igen: gfs_avn +flat_file: !ENV ${PARMpost}/postxconfig-NT-GFS.txt +ctrl_file: !ENV ${PARMpost}/postcntrl_gfs.xml +grib2_table: !ENV ${g2tmpl_ROOT}/share/params_grib2_tbl_new \ No newline at end of file diff --git a/parm/post/post.nml.j2 b/parm/post/post.nml.j2 new file mode 100644 index 0000000000..5c363b4c5e --- /dev/null +++ b/parm/post/post.nml.j2 @@ -0,0 +1,12 @@ +&model_inputs + fileName='atm_file' + fileNameFlux='sfc_file' + IOFORM='{{out_form}}' + grib='{{grib_version}}' + DateStr='%Y-%m-%d_%H:%M:%S' + MODELNAME='GFS' +/ + +&NAMPGB + {{post_variables}} +/ diff --git a/scripts/exglobal_atmos_post.sh b/scripts/exglobal_atmos_post.sh new file mode 100755 index 0000000000..1d610ac707 --- /dev/null +++ b/scripts/exglobal_atmos_post.sh @@ -0,0 +1,17 @@ +#! /usr/bin/env bash + +source "${HOMEgfs}/ush/preamble.sh" + +# Determine which yaml file to use +case ${fhr} in + anl) yaml_cat='anl' ;; + 000) yaml_cat='f000';; + *) yaml_cat='fhr' ;; +esac +post_settings="${PARMpost}/atm_post_${CDUMP}_${yaml_cat}.yaml" + +# Run post +"${HOMEgfs}/ush/atm_post.py" "${post_settings}" +err=$? + +exit ${err} diff --git a/ush/atm_post.py b/ush/atm_post.py new file mode 100755 index 0000000000..d0cda7070f --- /dev/null +++ b/ush/atm_post.py @@ -0,0 +1,300 @@ +#! /usr/bin/env python3 + +import os +import shutil +import glob +import subprocess +import time +from datetime import timedelta +from argparse import ArgumentParser +from python.pygw.src.pygw import timetools +from python.pygw.src.pygw.template import Template, TemplateConstants +from python.pygw.src.pygw.fsutils import mkdir, chdir, rm_p +from python.pygw.src.pygw.yaml_file import YAMLFile +from pprint import pprint +from functools import partial + +# Make sure print is flushed immediately +print = partial(print, flush=True) + + +def run_post(settings: dict) -> None: + ''' + Runs UPP as an MPI job, then creates an index file. + + Parameters + ---------- + settings : dict + Dictionary of settings needed to run post. Dictionary must include + the following keys/value pairs: + mpi_run : str + Command to execute an mpi job + grib_idx_exe : str + Path to program that creates a grib index file + exe_log_file : str + Path to log file for executables + + Returns + ------- + None + + Input files + ----------- + The following must be in the current directory: + upp.x : Post executable + itag : Post namelist + + Output files + ------------ + pgbfile : GRiB2 file for input data on gaussian grid + pgifile : GRiB2 index of pgbfile + + ''' + with open('itag', 'r') as file: + print(f''' + Executing {settings['post_exe']} (copied as upp.x) with the + following namelist: + {file.read()} + + Output will be written to {settings['exe_log_file']} + + ''' + ) + os.environ['PGBOUT'] = 'pgbfile' + subprocess.run(f"{settings['mpi_run']} upp.x > {settings['exe_log_file']}", shell=True, check=True) + subprocess.run(f"{settings['grib_idx_exe']} pgbfile pgifile >> {settings['exe_log_file']}", + shell=True, check=True) + + +def make_namelist(settings: dict) -> str: + ''' + Takes a namelist template and substitutes in variables and the verification time. + + Parameters + ---------- + settings : dict + Dictionary contaning the name of the template file, the initial time and + forecast hour, and the variables to substitute. At a minimum, the + dictionary must have the following key/value pairs: + tmpl_file : str + Path to the template + cdate : str + Initial time in YYYYMMDDHH format + fhr : str + Forecast hour (parsable to int) or anl + + Returns + ------- + str + String representation of the template file, with substitutions for all of the + variables in the template that are present in settings, and for all strftime + format codes (%Y, %m, etc.). + + ''' + with open(settings['tmpl_file'], 'r') as file: + tmpl = file.read() + + tmpl = Template.substitute_structure(tmpl, TemplateConstants.DOUBLE_CURLY_BRACES, settings.get) + delta = 0 if settings['fhr'] in ['anl'] else int(settings['fhr']) + when = timetools.strptime(settings['cdate'], "%Y%m%d%H") + timedelta(hours=delta) + tmpl = timetools.strftime(when, tmpl) + + return(tmpl) + + +def wait_for_model_output(settings: dict) -> None: + ''' + Repeatedly sleeps while waiting for trigger file to be available. Will return + once the file is present. If the file does not exist after the maximum wait + time, an exception will be thrown. + + Parameters + ---------- + settings : dict + Dictionary containing the needed settings. The dictional must specify + the following key/value pairs: + trigger_file : str + Path to file we are waiting for + sleep_interval : int + Time to sleep between checks (in seconds) + sleep_max : int + Maximum number of seconds to wait + + Returns + ------- + None + + Raises + ------ + RuntimeException + If trigger_file still does not exist after sleep_max seconds + + ''' + + sleep_max = settings['sleep_max'] + sleep_interval = settings['sleep_interval'] + trigger_file = settings['trigger_file'] + + for timer in range(0, sleep_max // sleep_interval): + if os.path.isfile(trigger_file): + # File exists, job can proceed + return + + time.sleep(sleep_interval) + + raise RuntimeError(f"File {trigger_file} does not exist after waiting {sleep_max}s") + + +def test_make_namelist() -> None: + settings = { + "cdate": '2021061512', + "fhr": '012', + "atm_file": "dummy_atm_file", + "flux_file": "dummy_flux_file", + "out_form": "netcdfpara", + "grib_version": "2", + "post_variables": "test list of post vars", + "tmpl_file": "../parm/post/post.nml.j2" + } + print(make_namelist(settings)) + + +def stage_post(settings: dict, nml_filename: str = 'itag') -> None: + ''' + Stages all the necessary files and executables to run post. + + Parameters + ---------- + settings : dict + Dictionary containing all the needed settings to stage. Dictionary must + contain the following key/value pairs: + work_dir : str + The temporary working directory to use + atm_file : str + Raw atmosphere model output in NetCDF format. Will be copied + to work_dir as atm_file. + sfc_file : str + Raw atmosphere flux output in NetCDF format. Will be copied + to work_dir as sfc_file. + flat_file : str + Post 'flat' file. Will be copied to work_dir as + postxconfig-NT.txt. + grib_table : str + Grib2 table. Will be copied to work_dir. + mp_file : str + Microphysics .dat file. Will be copied to work_dir as + eta_micro_lookup.dat. + post_exe : str + Path to the post executable. Will be copied to work_dir as + upp.x + + Additionally, the following key/value pair may be defined: + is_ens : bool + Whether data is an ensemble member (default is False) + + nml_filename : str + File name for the namelist file. UPP currently expects this to be 'itag'. + [Default: 'itag'] + + ''' + work_dir = settings['work_dir'] + mkdir(work_dir) + chdir(work_dir) + + namelist = make_namelist(settings) + with open(nml_filename, 'w') as file: + file.write(namelist) + + shutil.copy(settings['flat_file'], 'postxconfig-NT.txt') + shutil.copy(settings['grib_table'], './') + shutil.copy(settings['mp_file'], 'eta_micro_lookup.dat') + shutil.copy(settings['atm_file'], 'atm_file') + shutil.copy(settings['sfc_file'], 'sfc_file') + shutil.copy(settings['post_exe'], 'upp.x') + + if settings.get('is_ens', False): + # TODO replace negatively_post_fcst with ${ens_pert_type} in postxconfig-NT.txt + # sed < "${PostFlatFile}" -e "s#negatively_pert_fcst#${ens_pert_type}#" > ./postxconfig-NT.txt + pass + + +def send_com(settings: dict) -> None: + ''' + Copies post output from current directory to final destination. The current directory + must have files names pgbfile and pgifile for the master grib file and its index, + repsectively. + + Parameters + ---------- + settings : dict + Dictionary with paths for the final file names. The following key/value + pairs must be defined: + grib_out : str + File name for the master grib file + grib_idx_out : str + File naem for the master grib file index + + Returns + ------- + None + + ''' + shutil.copyfile('pgbfile', settings['grib_out']) + shutil.copyfile('pgifile', settings['grib_idx_out']) + + +def send_dbn(settings: dict) -> None: + ''' + Sends specified alerts to the data broadcast network (DBN). + + Parameters + ---------- + settings : dict + Dictionary with paths for the final file names. The following key/value + pairs must be defined: + dbn_alert : str + The command to send a DBN alert + dbn_signals : dict + Sub-dictionary with a list of signals (keys) and the files they are + signalling (values) + + ''' + for signal in settings.get('dbn_signals', []): + signal_file = settings['dbn_signals'][signal] + subprocess.run(f"{settings['dbn_alert']} {signal} {signal_file}", shell=True, check=True) + + +if __name__ == '__main__': + ''' + Runs post using a given settings file. + + Parameters + ---------- + settings_file : YAML file containing the settings to use for post + + Returns + ------- + None + + ''' + parser = ArgumentParser() + parser.add_argument('settings_file', help='Path to the YAML file containing the post settings') + + args = parser.parse_args() + settings = YAMLFile(path=args.settings_file) + # Move all settings that are defined under include to the top level + # TODO make this recursive + for inc in settings.pop('include', []): + settings.update(inc) + + print("Running post using the following settings:") + pprint(settings) + + stage_post(settings=settings) + wait_for_model_output(settings=settings) + run_post(settings=settings) + if settings['send_com'] in ["YES"]: + send_com(settings) + + if settings['send_dbn'] in ["YES"]: + send_dbn(settings)