From 5a5fc2be7555f094a0f90fd3a3df22d071ccdfd4 Mon Sep 17 00:00:00 2001 From: Jessica Meixner Date: Fri, 14 Jun 2024 11:04:41 -0400 Subject: [PATCH 1/9] Remove ocean daily files (#2689) This PR removes the ocn_daily files that are produced by the ocean component. These files can be recreated by averaging data that exists in the 6 hour aveaged files if needed. Fixes https://github.com/NOAA-EMC/global-workflow/issues/2675 Fixes https://github.com/NOAA-EMC/global-workflow/issues/2659 (by removing them and making this obsolete) --- parm/archive/master_gfs.yaml.j2 | 1 - parm/archive/ocean_daily.yaml.j2 | 8 -------- parm/ufs/fv3/diag_table | 20 -------------------- ush/forecast_postdet.sh | 7 ------- 4 files changed, 36 deletions(-) delete mode 100644 parm/archive/ocean_daily.yaml.j2 diff --git a/parm/archive/master_gfs.yaml.j2 b/parm/archive/master_gfs.yaml.j2 index e96def7a03..67cde482a2 100644 --- a/parm/archive/master_gfs.yaml.j2 +++ b/parm/archive/master_gfs.yaml.j2 @@ -56,7 +56,6 @@ datasets: # Ocean forecasts {% filter indent(width=4) %} {% include "ocean_6hravg.yaml.j2" %} -{% include "ocean_daily.yaml.j2" %} {% include "ocean_grib2.yaml.j2" %} {% include "gfs_flux_1p00.yaml.j2" %} {% endfilter %} diff --git a/parm/archive/ocean_daily.yaml.j2 b/parm/archive/ocean_daily.yaml.j2 deleted file mode 100644 index 0f45264973..0000000000 --- a/parm/archive/ocean_daily.yaml.j2 +++ /dev/null @@ -1,8 +0,0 @@ -ocean_daily: - {% set head = "gfs.ocean.t" + cycle_HH + "z." %} - name: "OCEAN_DAILY" - target: "{{ ATARDIR }}/{{ cycle_YMDH }}/ocean_daily.tar" - required: - {% for fhr in range(24, FHMAX_GFS + 24, 24) %} - - "{{ COM_OCEAN_HISTORY | relpath(ROTDIR) }}/{{ head }}daily.f{{ '%03d' % fhr }}.nc" - {% endfor %} diff --git a/parm/ufs/fv3/diag_table b/parm/ufs/fv3/diag_table index 83991cb223..dad8b6fac6 100644 --- a/parm/ufs/fv3/diag_table +++ b/parm/ufs/fv3/diag_table @@ -1,7 +1,6 @@ "fv3_history", 0, "hours", 1, "hours", "time" "fv3_history2d", 0, "hours", 1, "hours", "time" "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr", @[FHOUT_OCN], "hours", 1, "hours", "time", @[FHOUT_OCN], "hours", "@[SYEAR] @[SMONTH] @[SDAY] @[CHOUR] 0 0" -"@[MOM6_OUTPUT_DIR]/ocn_daily%4yr%2mo%2dy", 1, "days", 1, "days", "time", 1, "days", "@[SYEAR] @[SMONTH] @[SDAY] @[CHOUR] 0 0" ############## # Ocean fields @@ -57,25 +56,6 @@ "ocean_model", "LwLatSens", "LwLatSens", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr","all",.true.,"none",2 "ocean_model", "Heat_PmE", "Heat_PmE", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr","all",.true.,"none",2 -# Daily fields -"ocean_model", "geolon", "geolon", "@[MOM6_OUTPUT_DIR]/ocn_daily%4yr%2mo%2dy", "all", .false., "none", 2 -"ocean_model", "geolat", "geolat", "@[MOM6_OUTPUT_DIR]/ocn_daily%4yr%2mo%2dy", "all", .false., "none", 2 -"ocean_model", "geolon_c", "geolon_c", "@[MOM6_OUTPUT_DIR]/ocn_daily%4yr%2mo%2dy", "all", .false., "none", 2 -"ocean_model", "geolat_c", "geolat_c", "@[MOM6_OUTPUT_DIR]/ocn_daily%4yr%2mo%2dy", "all", .false., "none", 2 -"ocean_model", "geolon_u", "geolon_u", "@[MOM6_OUTPUT_DIR]/ocn_daily%4yr%2mo%2dy", "all", .false., "none", 2 -"ocean_model", "geolat_u", "geolat_u", "@[MOM6_OUTPUT_DIR]/ocn_daily%4yr%2mo%2dy", "all", .false., "none", 2 -"ocean_model", "geolon_v", "geolon_v", "@[MOM6_OUTPUT_DIR]/ocn_daily%4yr%2mo%2dy", "all", .false., "none", 2 -"ocean_model", "geolat_v", "geolat_v", "@[MOM6_OUTPUT_DIR]/ocn_daily%4yr%2mo%2dy", "all", .false., "none", 2 -"ocean_model", "SST", "sst", "@[MOM6_OUTPUT_DIR]/ocn_daily%4yr%2mo%2dy", "all", .true., "none", 2 -"ocean_model", "latent", "latent", "@[MOM6_OUTPUT_DIR]/ocn_daily%4yr%2mo%2dy", "all", .true., "none", 2 -"ocean_model", "sensible", "sensible", "@[MOM6_OUTPUT_DIR]/ocn_daily%4yr%2mo%2dy", "all", .true., "none", 2 -"ocean_model", "SW", "SW", "@[MOM6_OUTPUT_DIR]/ocn_daily%4yr%2mo%2dy", "all", .true., "none", 2 -"ocean_model", "LW", "LW", "@[MOM6_OUTPUT_DIR]/ocn_daily%4yr%2mo%2dy", "all", .true., "none", 2 -"ocean_model", "evap", "evap", "@[MOM6_OUTPUT_DIR]/ocn_daily%4yr%2mo%2dy", "all", .true., "none", 2 -"ocean_model", "lprec", "lprec", "@[MOM6_OUTPUT_DIR]/ocn_daily%4yr%2mo%2dy", "all", .true., "none", 2 -"ocean_model", "taux", "taux", "@[MOM6_OUTPUT_DIR]/ocn_daily%4yr%2mo%2dy", "all", .true., "none", 2 -"ocean_model", "tauy", "tauy", "@[MOM6_OUTPUT_DIR]/ocn_daily%4yr%2mo%2dy", "all", .true., "none", 2 - ################### # Atmosphere fields ################### diff --git a/ush/forecast_postdet.sh b/ush/forecast_postdet.sh index a4e10dffb9..2cc34eaacd 100755 --- a/ush/forecast_postdet.sh +++ b/ush/forecast_postdet.sh @@ -455,13 +455,6 @@ MOM6_postdet() { dest_file="${RUN}.ocean.t${cyc}z.${interval}hr_avg.f${fhr3}.nc" ${NLN} "${COMOUT_OCEAN_HISTORY}/${dest_file}" "${DATA}/MOM6_OUTPUT/${source_file}" - # Daily output - if (( fhr > 0 & fhr % 24 == 0 )); then - source_file="ocn_daily_${vdate:0:4}_${vdate:4:2}_${vdate:6:2}.nc" - dest_file="${RUN}.ocean.t${cyc}z.daily.f${fhr3}.nc" - ${NLN} "${COMOUT_OCEAN_HISTORY}/${dest_file}" "${DATA}/MOM6_OUTPUT/${source_file}" - fi - last_fhr=${fhr} done From 6c93b4554e235fcb4d0004e99a4c4498d55d461b Mon Sep 17 00:00:00 2001 From: Yaping Wang <49168260+ypwang19@users.noreply.github.com> Date: Fri, 14 Jun 2024 10:18:17 -0500 Subject: [PATCH 2/9] Add observation preparation job for aerosols DA to workflow (#2624) Add a prepaeroobs job to prepare aerosol obs files for DA. This job does quality control of the VIIRS aerosol raw observations and convert them to ioda format. Resolves #2623 --------- Co-authored-by: ypwang19 Co-authored-by: TerrenceMcGuinness-NOAA Co-authored-by: Cory Martin Co-authored-by: David Huber <69919478+DavidHuber-NOAA@users.noreply.github.com> --- env/HERA.env | 7 + env/HERCULES.env | 6 + env/JET.env | 7 + env/ORION.env | 7 + env/S4.env | 7 + env/WCOSS2.env | 7 + jobs/JGLOBAL_PREP_OBS_AERO | 43 +++++ jobs/rocoto/prepobsaero.sh | 24 +++ parm/archive/arcdir.yaml.j2 | 5 + parm/archive/gdas.yaml.j2 | 4 + parm/archive/gfsa.yaml.j2 | 4 + parm/config/gfs/config.base | 1 + parm/config/gfs/config.prepobsaero | 17 ++ parm/config/gfs/config.resources | 10 +- scripts/exglobal_archive.py | 4 +- scripts/exglobal_prep_obs_aero.py | 23 +++ ush/python/pygfs/task/aero_prepobs.py | 236 ++++++++++++++++++++++++++ workflow/applications/applications.py | 1 + workflow/applications/gfs_cycled.py | 6 + workflow/rocoto/gfs_tasks.py | 25 +++ 20 files changed, 441 insertions(+), 3 deletions(-) create mode 100755 jobs/JGLOBAL_PREP_OBS_AERO create mode 100755 jobs/rocoto/prepobsaero.sh create mode 100644 parm/config/gfs/config.prepobsaero create mode 100755 scripts/exglobal_prep_obs_aero.py create mode 100644 ush/python/pygfs/task/aero_prepobs.py diff --git a/env/HERA.env b/env/HERA.env index 2157e90031..94bab36703 100755 --- a/env/HERA.env +++ b/env/HERA.env @@ -102,6 +102,13 @@ elif [[ "${step}" = "atmanlfv3inc" ]]; then [[ ${NTHREADS_ATMANLFV3INC} -gt ${nth_max} ]] && export NTHREADS_ATMANLFV3INC=${nth_max} export APRUN_ATMANLFV3INC="${launcher} -n ${npe_atmanlfv3inc} --cpus-per-task=${NTHREADS_ATMANLFV3INC}" +elif [[ "${step}" = "prepobsaero" ]]; then + + nth_max=$((npe_node_max / npe_node_prepobsaero)) + + export NTHREADS_PREPOBSAERO=${nth_prepobsaero:-1} + export APRUN_PREPOBSAERO="${launcher} -n ${npe_prepobsaero} --cpus-per-task=${NTHREADS_PREPOBSAERO}" + elif [[ "${step}" = "snowanl" ]]; then nth_max=$((npe_node_max / npe_node_snowanl)) diff --git a/env/HERCULES.env b/env/HERCULES.env index 0824ba913a..d43dedad8d 100755 --- a/env/HERCULES.env +++ b/env/HERCULES.env @@ -99,6 +99,12 @@ case ${step} in [[ ${NTHREADS_AEROANL} -gt ${nth_max} ]] && export NTHREADS_AEROANL=${nth_max} export APRUN_AEROANL="${launcher} -n ${npe_aeroanlrun} --cpus-per-task=${NTHREADS_AEROANL}" ;; + "prepobsaero") + nth_max=$((npe_node_max / npe_node_prepobsaero)) + + export NTHREADS_PREPOBSAERO=${nth_prepobsaero:-1} + export APRUN_PREPOBSAERO="${launcher} -n ${npe_prepobsaero} --cpus-per-task=${NTHREADS_PREPOBSAERO}" +;; "snowanl") nth_max=$((npe_node_max / npe_node_snowanl)) diff --git a/env/JET.env b/env/JET.env index 5bd88dc93a..668ec1c2e4 100755 --- a/env/JET.env +++ b/env/JET.env @@ -82,6 +82,13 @@ elif [[ "${step}" = "aeroanlrun" ]]; then [[ ${NTHREADS_AEROANL} -gt ${nth_max} ]] && export NTHREADS_AEROANL=${nth_max} export APRUN_AEROANL="${launcher} -n ${npe_aeroanlrun}" +elif [[ "${step}" = "prepobsaero" ]]; then + + nth_max=$((npe_node_max / npe_node_prepobsaero)) + + export NTHREADS_PREPOBSAERO=${nth_prepobsaero:-1} + export APRUN_PREPOBSAERO="${launcher} -n ${npe_prepobsaero} --cpus-per-task=${NTHREADS_PREPOBSAERO}" + elif [[ "${step}" = "snowanl" ]]; then nth_max=$((npe_node_max / npe_node_snowanl)) diff --git a/env/ORION.env b/env/ORION.env index f701e55aa2..afd1cda052 100755 --- a/env/ORION.env +++ b/env/ORION.env @@ -90,6 +90,13 @@ elif [[ "${step}" = "aeroanlrun" ]]; then [[ ${NTHREADS_AEROANL} -gt ${nth_max} ]] && export NTHREADS_AEROANL=${nth_max} export APRUN_AEROANL="${launcher} -n ${npe_aeroanlrun} --cpus-per-task=${NTHREADS_AEROANL}" +elif [[ "${step}" = "prepobsaero" ]]; then + + nth_max=$((npe_node_max / npe_node_prepobsaero)) + + export NTHREADS_PREPOBSAERO=${nth_prepobsaero:-1} + export APRUN_PREPOBSAERO="${launcher} -n ${npe_prepobsaero} --cpus-per-task=${NTHREADS_PREPOBSAERO}" + elif [[ "${step}" = "snowanl" ]]; then nth_max=$((npe_node_max / npe_node_snowanl)) diff --git a/env/S4.env b/env/S4.env index 9ba3a61b01..8a368bf1d6 100755 --- a/env/S4.env +++ b/env/S4.env @@ -82,6 +82,13 @@ elif [[ "${step}" = "aeroanlrun" ]]; then [[ ${NTHREADS_AEROANL} -gt ${nth_max} ]] && export NTHREADS_AEROANL=${nth_max} export APRUN_AEROANL="${launcher} -n ${npe_aeroanlrun}" +elif [[ "${step}" = "prepobsaero" ]]; then + + nth_max=$((npe_node_max / npe_node_prepobsaero)) + + export NTHREADS_PREPOBSAERO=${nth_prepobsaero:-1} + export APRUN_PREPOBSAERO="${launcher} -n ${npe_prepobsaero} --cpus-per-task=${NTHREADS_PREPOBSAERO}" + elif [[ "${step}" = "snowanl" ]]; then nth_max=$((npe_node_max / npe_node_snowanl)) diff --git a/env/WCOSS2.env b/env/WCOSS2.env index 0876e4127d..9fe9179e6b 100755 --- a/env/WCOSS2.env +++ b/env/WCOSS2.env @@ -76,6 +76,13 @@ elif [[ "${step}" = "aeroanlrun" ]]; then [[ ${NTHREADS_AEROANL} -gt ${nth_max} ]] && export NTHREADS_AEROANL=${nth_max} export APRUN_AEROANL="${launcher} -n ${npe_aeroanlrun}" +elif [[ "${step}" = "prepobsaero" ]]; then + + nth_max=$((npe_node_max / npe_node_prepaeroobs)) + + export NTHREADS_PREPOBSAERO=${nth_prepobsaero:-1} + export APRUN_PREPOBSAERO="${launcher} -n ${npe_prepobsaero} --ppn ${npe_node_prepobsaero}--cpu-bind depth --depth=${NTHREADS_PREPOBSAERO}" + elif [[ "${step}" = "snowanl" ]]; then nth_max=$((npe_node_max / npe_node_snowanl)) diff --git a/jobs/JGLOBAL_PREP_OBS_AERO b/jobs/JGLOBAL_PREP_OBS_AERO new file mode 100755 index 0000000000..7fe701898f --- /dev/null +++ b/jobs/JGLOBAL_PREP_OBS_AERO @@ -0,0 +1,43 @@ +#! /usr/bin/env bash + +source "${HOMEgfs}/ush/preamble.sh" +source "${HOMEgfs}/ush/jjob_header.sh" -e "prepobsaero" -c "base prepobsaero" + +############################################## +# Set variables used in the script +############################################## + +export COMIN_OBS="${DATA}" +YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COMOUT_OBS:COM_OBS_TMPL + +############################################## +# Begin JOB SPECIFIC work +############################################## + +############################################################### +# Run relevant script + +EXSCRIPT=${GDASPREPAEROOBSPY:-${SCRgfs}/exglobal_prep_obs_aero.py} +${EXSCRIPT} +status=$? +[[ ${status} -ne 0 ]] && exit "${status}" + + +############################################## +# End JOB SPECIFIC work +############################################## + +############################################## +# Final processing +############################################## +if [[ -e "${pgmout}" ]] ; then + cat "${pgmout}" +fi + +########################################## +# Remove the Temporary working directory +########################################## +cd "${DATAROOT}" || exit +[[ "${KEEPDATA}" = "NO" ]] && rm -rf "${DATA}" + +exit 0 diff --git a/jobs/rocoto/prepobsaero.sh b/jobs/rocoto/prepobsaero.sh new file mode 100755 index 0000000000..89da7547e8 --- /dev/null +++ b/jobs/rocoto/prepobsaero.sh @@ -0,0 +1,24 @@ +#! /usr/bin/env bash + +source "${HOMEgfs}/ush/preamble.sh" + +############################################################### +# Source UFSDA workflow modules +. "${HOMEgfs}/ush/load_ufsda_modules.sh" +status=$? +[[ ${status} -ne 0 ]] && exit "${status}" + +export job="prepobsaero" +export jobid="${job}.$$" + +############################################################### +# setup python path for workflow utilities and tasks +wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src" +PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}" +export PYTHONPATH + +############################################################### +# Execute the JJOB +"${HOMEgfs}/jobs/JGLOBAL_PREP_OBS_AERO" +status=$? +exit "${status}" diff --git a/parm/archive/arcdir.yaml.j2 b/parm/archive/arcdir.yaml.j2 index fab93cc2a4..1ed7422761 100644 --- a/parm/archive/arcdir.yaml.j2 +++ b/parm/archive/arcdir.yaml.j2 @@ -47,6 +47,11 @@ Deterministic: &deterministic - ["{{ COM_CHEM_ANALYSIS }}/{{ head }}aerostat", "{{ ARCDIR }}/aerostat.{{ RUN }}.{{ cycle_YMDH }}"] {% endif %} + {% if DO_PREP_OBS_AERO %} + - ["{{ COM_OBS }}/{{ head }}aeroobs", "{{ ARCDIR }}/aeroobs.{{ RUN }}.{{ cycle_YMDH }}"] + - ["{{ COM_OBS }}/{{ head }}aerorawobs", "{{ ARCDIR }}/aerorawobs.{{ RUN }}.{{ cycle_YMDH }}"] + {% endif %} + {% endif %} {% if RUN == "gfs" %} diff --git a/parm/archive/gdas.yaml.j2 b/parm/archive/gdas.yaml.j2 index fd5e074111..26540156cd 100644 --- a/parm/archive/gdas.yaml.j2 +++ b/parm/archive/gdas.yaml.j2 @@ -69,6 +69,10 @@ gdas: {% if AERO_ANL_CDUMP == "gdas" or AERO_ANL_CDUMP == "both" %} - "{{ COM_CHEM_ANALYSIS | relpath(ROTDIR) }}/{{ head }}aerostat" {% endif %} + {% if DO_PREP_OBS_AERO %} + - "{{ COM_OBS | relpath(ROTDIR) }}/{{ head }}aeroobs" + - "{{ COM_OBS | relpath(ROTDIR) }}/{{ head }}aerorawobs" + {% endif %} {% if DO_JEDISNOWDA %} - "{{ COM_SNOW_ANALYSIS | relpath(ROTDIR) }}/{{ head }}snowstat.tgz" {% endif %} diff --git a/parm/archive/gfsa.yaml.j2 b/parm/archive/gfsa.yaml.j2 index df90a1a71e..0a8e65d3ef 100644 --- a/parm/archive/gfsa.yaml.j2 +++ b/parm/archive/gfsa.yaml.j2 @@ -40,6 +40,10 @@ gfsa: {% if AERO_ANL_CDUMP == "gfs" or AERO_ANL_CDUMP == "both" %} - "{{ COM_CHEM_ANALYSIS | relpath(ROTDIR) }}/{{ head }}aerostat" {% endif %} + {% if DO_PREP_OBS_AERO %} + - "{{ COM_OBS | relpath(ROTDIR) }}/{{ head }}aeroobs" + - "{{ COM_OBS | relpath(ROTDIR) }}/{{ head }}aerorawobs" + {% endif %} # BUFR inputs - "{{ COM_OBS | relpath(ROTDIR) }}/{{ head }}nsstbufr" diff --git a/parm/config/gfs/config.base b/parm/config/gfs/config.base index 6cc1b6d744..fd1ffdc415 100644 --- a/parm/config/gfs/config.base +++ b/parm/config/gfs/config.base @@ -178,6 +178,7 @@ export DO_WAVE="NO" export DO_OCN="NO" export DO_ICE="NO" export DO_AERO="NO" +export DO_PREP_OBS_AERO="NO" export AERO_FCST_CDUMP="" # When to run aerosol forecast: gdas, gfs, or both export AERO_ANL_CDUMP="" # When to run aerosol analysis: gdas, gfs, or both export WAVE_CDUMP="" # When to include wave suite: gdas, gfs, or both diff --git a/parm/config/gfs/config.prepobsaero b/parm/config/gfs/config.prepobsaero new file mode 100644 index 0000000000..f70138991c --- /dev/null +++ b/parm/config/gfs/config.prepobsaero @@ -0,0 +1,17 @@ +#!/bin/bash -x + +########## config.prepobsaero ########## +# Prepare and thin/superob aerosol observations + +echo "BEGIN: config.prepobsaero" + +# Get task specific resources +source "${EXPDIR}/config.resources" prepobsaero + +export OBSPROCYAML="${PARMgfs}/gdas/aero/obs/lists/gdas_aero_obsproc.yaml.j2" +export OBSPROCEXE="${EXECgfs}/gdas_obsprovider2ioda.x" +export VIIRS_DATA_DIR="/scratch2/NCEPDEV/stmp3/Yaping.Wang/VIIRS/AWS/" +export SENSORS="npp,n20" + + +echo "END: config.prepaeroobs" diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources index 0972f74f9c..1d1fd2e3c1 100644 --- a/parm/config/gfs/config.resources +++ b/parm/config/gfs/config.resources @@ -13,7 +13,7 @@ if (( $# != 1 )); then echo "atmanlinit atmanlvar atmanlfv3inc atmanlfinal" echo "atmensanlinit atmensanlletkf atmensanlfv3inc atmensanlfinal" echo "snowanl" - echo "aeroanlinit aeroanlrun aeroanlfinal" + echo "prepobsaero aeroanlinit aeroanlrun aeroanlfinal" echo "anal sfcanl analcalc analdiag fcst echgres" echo "upp atmos_products" echo "tracker genesis genesis_fsu" @@ -289,6 +289,14 @@ case ${step} in export npe_node_snowanl=$(( npe_node_max / nth_snowanl )) ;; + "prepobsaero") + export wtime_prepobsaero="00:30:00" + export npe_prepobsaero=1 + export nth_prepobsaero=1 + export npe_node_prepobsaero=1 + export memory_prepobsaero="96GB" + ;; + "aeroanlinit") # below lines are for creating JEDI YAML case ${CASE} in diff --git a/scripts/exglobal_archive.py b/scripts/exglobal_archive.py index e38d0abf72..bcd8d522d9 100755 --- a/scripts/exglobal_archive.py +++ b/scripts/exglobal_archive.py @@ -19,8 +19,8 @@ def main(): # Pull out all the configuration keys needed to run the rest of archive steps keys = ['ATARDIR', 'current_cycle', 'FHMIN', 'FHMAX', 'FHOUT', 'RUN', 'PDY', - 'DO_VERFRAD', 'DO_VMINMON', 'DO_VERFOZN', 'DO_ICE', 'DO_AERO', 'PARMgfs', - 'DO_OCN', 'DO_WAVE', 'WRITE_DOPOST', 'PSLOT', 'HPSSARCH', 'DO_MOS', + 'DO_VERFRAD', 'DO_VMINMON', 'DO_VERFOZN', 'DO_ICE', 'DO_AERO', 'DO_PREP_OBS_AERO', + 'PARMgfs', 'DO_OCN', 'DO_WAVE', 'WRITE_DOPOST', 'PSLOT', 'HPSSARCH', 'DO_MOS', 'DO_JEDISNOWDA', 'LOCALARCH', 'REALTIME', 'ROTDIR', 'ARCH_WARMICFREQ', 'ARCH_FCSTICFREQ', 'ARCH_CYC', 'assim_freq', 'ARCDIR', 'SDATE', 'FHMIN_GFS', 'FHMAX_GFS', 'FHOUT_GFS', 'ARCH_GAUSSIAN', 'MODE', diff --git a/scripts/exglobal_prep_obs_aero.py b/scripts/exglobal_prep_obs_aero.py new file mode 100755 index 0000000000..08548e6874 --- /dev/null +++ b/scripts/exglobal_prep_obs_aero.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +# exglobal_prep_obs_aero.py +# This script collect available viirs +# obs files, combine and preprocess +# them. +import os + +from wxflow import Logger, cast_strdict_as_dtypedict +from pygfs.task.aero_prepobs import AerosolObsPrep + +# Initialize root logger +logger = Logger(level='DEBUG', colored_log=True) + + +if __name__ == '__main__': + + # Take configuration from environment and cast it as python dictionary + config = cast_strdict_as_dtypedict(os.environ) + + AeroObs = AerosolObsPrep(config) + AeroObs.initialize() + AeroObs.runConverter() + AeroObs.finalize() diff --git a/ush/python/pygfs/task/aero_prepobs.py b/ush/python/pygfs/task/aero_prepobs.py new file mode 100644 index 0000000000..f2344241a9 --- /dev/null +++ b/ush/python/pygfs/task/aero_prepobs.py @@ -0,0 +1,236 @@ +#!/usr/bin/env python3 + +import os +import glob +import gzip +import tarfile +import re +from logging import getLogger +from typing import List, Dict, Any, Union + +from wxflow import (AttrDict, FileHandler, rm_p, rmdir, + Task, add_to_datetime, to_timedelta, to_datetime, + datetime_to_YMD, + chdir, Executable, WorkflowException, + parse_j2yaml, save_as_yaml, logit) + +logger = getLogger(__name__.split('.')[-1]) + + +class AerosolObsPrep(Task): + """ + Class for preparing and managing aerosol observations + """ + def __init__(self, config: Dict[str, Any]) -> None: + super().__init__(config) + + _window_begin = add_to_datetime(self.runtime_config.current_cycle, -to_timedelta(f"{self.config['assim_freq']}H") / 2) + _window_end = add_to_datetime(self.runtime_config.current_cycle, +to_timedelta(f"{self.config['assim_freq']}H") / 2) + + local_dict = AttrDict( + { + 'window_begin': _window_begin, + 'window_end': _window_end, + 'sensors': str(self.config['SENSORS']).split(','), + 'data_dir': self.config['VIIRS_DATA_DIR'], + 'input_files': '', + 'OPREFIX': f"{self.runtime_config.RUN}.t{self.runtime_config.cyc:02d}z.", + 'APREFIX': f"{self.runtime_config.RUN}.t{self.runtime_config.cyc:02d}z." + } + ) + + # task_config is everything that this task should need + self.task_config = AttrDict(**self.config, **self.runtime_config, **local_dict) + + @logit(logger) + def initialize(self) -> None: + """ + List needed raw obs files. + Copy the raw obs files to $DATA/obs. + Link over the needed executable. + Generate corresponding YAML file. + Run IODA converter. + """ + self.task_config.DATA_OBS = os.path.join(self.task_config.DATA, 'obs') + if os.path.exists(self.task_config.DATA_OBS): + rmdir(self.task_config.DATA_OBS) + FileHandler({'mkdir': [self.task_config.DATA_OBS]}).sync() + + self.task_config.prepaero_yaml = [] + for sensor in self.task_config.sensors: + raw_files = self.list_raw_files(sensor) + self.task_config.input_files = self.copy_obs(raw_files) + self.link_obsconvexe() + self.task_config.prepaero_config = self.get_obsproc_config(sensor) + + # generate converter YAML file + template = f"{self.runtime_config.CDUMP}.t{self.runtime_config['cyc']:02d}z.prepaero_viirs_{sensor}.yaml" + _prepaero_yaml = os.path.join(self.runtime_config.DATA, template) + self.task_config.prepaero_yaml.append(_prepaero_yaml) + logger.debug(f"Generate PrepAeroObs YAML file: {_prepaero_yaml}") + save_as_yaml(self.task_config.prepaero_config, _prepaero_yaml) + logger.info(f"Wrote PrepAeroObs YAML to: {_prepaero_yaml}") + + @logit(logger) + def list_raw_files(self, sensor) -> List[str]: + """ + List all files in the predefined directory that match the predefined sensor and within the time window. + """ + if sensor == 'n20': + sensor = 'j01' + dir1 = os.path.join(self.task_config.data_dir, datetime_to_YMD(self.task_config.window_begin)) + dir2 = os.path.join(self.task_config.data_dir, datetime_to_YMD(self.task_config.window_end)) + + if dir1 == dir2: + files = os.listdir(dir1) + allfiles = [os.path.join(dir1, file) for file in files] + allfiles.sort() + else: + files_1 = os.listdir(dir1) + allfiles_1 = [os.path.join(dir1, file) for file in files_1] + files_2 = os.listdir(dir2) + allfiles_2 = [os.path.join(dir2, file) for file in files_2] + allfiles = sorted(allfiles_1, allfiles_2) + matching_files = [] + try: + for file in allfiles: + basename = os.path.basename(file) + pattern = r"s(\d{4})(\d{2})(\d{2})(\d{2})(\d{2})(\d{3})" + match = re.match(pattern, basename.split('_')[3]) + yyyy, mm, dd, HH, MM = match.group(1), match.group(2), match.group(3), match.group(4), match.group(5) + fstart = to_datetime(f'{yyyy}-{mm}-{dd}T{HH}:{MM}Z') + if sensor == basename.split('_')[2]: + # temporally select obs files based on time stamp in the filename. + if (fstart > self.task_config.window_begin) and (fstart < self.task_config.window_end): + matching_files.append(os.path.join(self.task_config.data_dir, file)) + logger.info("Found %d matching files.", len(matching_files)) + except FileNotFoundError: + logger.error("The specified file/directory does not exist.") + raise + return matching_files + + @logit(logger) + def copy_obs(self, inputfiles) -> Dict[str, Any]: + """ + Copy the raw obs files to $DATA/obs. + """ + copylist = [] + destlist = [] + for filename in inputfiles: + basename = os.path.basename(filename) + dest = os.path.join(self.task_config.DATA_OBS, basename) + copylist.append([filename, dest]) + destlist.append(dest) + FileHandler({'copy': copylist}).sync() + + return destlist + + @logit(logger) + def get_obsproc_config(self, sensor) -> Dict[str, Any]: + """ + Compile a dictionary of obs proc configuration from OBSPROCYAML template file + Parameters + ---------- + Returns + ---------- + obsproc_config : Dict + a dictionary containing the fully rendered obs proc yaml configuration + """ + self.task_config.sensor = sensor + # generate JEDI YAML file + logger.info(f"Generate gdas_obsprovider2ioda YAML config: {self.task_config.OBSPROCYAML}") + prepaero_config = parse_j2yaml(self.task_config.OBSPROCYAML, self.task_config) + + return prepaero_config + + @logit(logger) + def link_obsconvexe(self) -> None: + """ + This method links the gdas executable to the run directory + Parameters + ---------- + Task: GDAS task + Returns + ---------- + None + """ + exe_src = self.task_config.OBSPROCEXE + + logger.info(f"Link executable {exe_src} to DATA/") + exe_dest = os.path.join(self.task_config.DATA, os.path.basename(exe_src)) + if os.path.exists(exe_dest): + rm_p(exe_dest) + os.symlink(exe_src, exe_dest) + + return + + @logit(logger) + def runConverter(self) -> None: + """ + Run the IODA converter gdas_obsprovider2ioda.x + """ + chdir(self.task_config.DATA) + exec_cmd = Executable(self.task_config.APRUN_PREPOBSAERO) + exec_name = os.path.join(self.task_config.DATA, 'gdas_obsprovider2ioda.x') + exec_cmd.add_default_arg(exec_name) + + for prepaero_yaml in self.task_config.prepaero_yaml: + try: + logger.debug(f"Executing {exec_cmd} on {prepaero_yaml}") + exec_cmd(f"{prepaero_yaml}") + except OSError: + raise OSError(f"Failed to execute {exec_cmd} on {prepaero_yaml}") + except Exception: + raise WorkflowException(f"An error occured during execution of {exec_cmd} on {prepaero_yaml}") + + pass + + @logit(logger) + def finalize(self) -> None: + """ + Copy the output viirs files to COMIN_OBS. + Tar and archive the output files. + Tar and archive the raw obs files. + """ + # get list of viirs files + obsfiles = glob.glob(os.path.join(self.task_config['DATA'], '*viirs*nc4')) + copylist = [] + for obsfile in obsfiles: + basename = os.path.basename(obsfile) + src = os.path.join(self.task_config['DATA'], basename) + dest = os.path.join(self.task_config.COMOUT_OBS, basename) + copylist.append([src, dest]) + FileHandler({'copy': copylist}).sync() + + # gzip the files first + for obsfile in obsfiles: + with open(obsfile, 'rb') as f_in, gzip.open(f"{obsfile}.gz", 'wb') as f_out: + f_out.writelines(f_in) + + aeroobs = os.path.join(self.task_config.COMOUT_OBS, f"{self.task_config['APREFIX']}aeroobs") + # open tar file for writing + with tarfile.open(aeroobs, "w") as archive: + for obsfile in obsfiles: + aeroobsgzip = f"{obsfile}.gz" + archive.add(aeroobsgzip, arcname=os.path.basename(aeroobsgzip)) + # get list of raw viirs L2 files + rawfiles = glob.glob(os.path.join(self.task_config.DATA_OBS, 'JRR-AOD*')) + # gzip the raw L2 files first + for rawfile in rawfiles: + with open(rawfile, 'rb') as f_in, gzip.open(f"{rawfile}.gz", 'wb') as f_out: + f_out.writelines(f_in) + + aerorawobs = os.path.join(self.task_config.COMOUT_OBS, f"{self.task_config['APREFIX']}aerorawobs") + # open tar file for writing + with tarfile.open(aerorawobs, "w") as archive: + for rawfile in rawfiles: + aerorawobsgzip = f"{rawfile}.gz" + archive.add(aerorawobsgzip, arcname=os.path.basename(aerorawobsgzip)) + copylist = [] + for prepaero_yaml in self.task_config.prepaero_yaml: + basename = os.path.basename(prepaero_yaml) + dest = os.path.join(self.task_config.COMOUT_OBS, basename) + copylist.append([prepaero_yaml, dest]) + FileHandler({'copy': copylist}).sync() + + pass diff --git a/workflow/applications/applications.py b/workflow/applications/applications.py index 50a9a7cdd0..6a4d240fe5 100644 --- a/workflow/applications/applications.py +++ b/workflow/applications/applications.py @@ -51,6 +51,7 @@ def __init__(self, conf: Configuration) -> None: self.do_ocean = _base.get('DO_OCN', False) self.do_ice = _base.get('DO_ICE', False) self.do_aero = _base.get('DO_AERO', False) + self.do_prep_obs_aero = _base.get('DO_PREP_OBS_AERO', False) self.do_bufrsnd = _base.get('DO_BUFRSND', False) self.do_gempak = _base.get('DO_GEMPAK', False) self.do_awips = _base.get('DO_AWIPS', False) diff --git a/workflow/applications/gfs_cycled.py b/workflow/applications/gfs_cycled.py index f7f9b5b5e6..175ddb07bf 100644 --- a/workflow/applications/gfs_cycled.py +++ b/workflow/applications/gfs_cycled.py @@ -108,6 +108,8 @@ def _get_app_configs(self): if self.do_aero: configs += ['aeroanlinit', 'aeroanlrun', 'aeroanlfinal'] + if self.do_prep_obs_aero: + configs += ['prepobsaero'] if self.do_jedisnowda: configs += ['prepsnowobs', 'snowanl'] @@ -178,6 +180,8 @@ def get_task_names(self): if self.do_aero and 'gdas' in self.aero_anl_cdumps: gdas_tasks += ['aeroanlinit', 'aeroanlrun', 'aeroanlfinal'] + if self.do_prep_obs_aero: + gdas_tasks += ['prepobsaero'] gdas_tasks += ['atmanlupp', 'atmanlprod', 'fcst'] @@ -215,6 +219,8 @@ def get_task_names(self): if self.do_aero and 'gfs' in self.aero_anl_cdumps: gfs_tasks += ['aeroanlinit', 'aeroanlrun', 'aeroanlfinal'] + if self.do_prep_obs_aero: + gfs_tasks += ['prepobsaero'] gfs_tasks += ['atmanlupp', 'atmanlprod', 'fcst'] diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index fa218c6713..0fd468b3b4 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -483,10 +483,35 @@ def atmanlfinal(self): return task + def prepobsaero(self): + deps = [] + dep_dict = {'type': 'task', 'name': f'{self.cdump}prep'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) + + resources = self.get_resource('prepobsaero') + task_name = f'{self.cdump}prepobsaero' + task_dict = {'task_name': task_name, + 'resources': resources, + 'dependency': dependencies, + 'envars': self.envars, + 'cycledef': self.cdump.replace('enkf', ''), + 'command': f'{self.HOMEgfs}/jobs/rocoto/prepobsaero.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' + } + + task = rocoto.create_task(task_dict) + + return task + def aeroanlinit(self): deps = [] dep_dict = {'type': 'task', 'name': f'{self.cdump}prep'} + if self.app_config.do_prep_obs_aero: + dep_dict = {'type': 'task', 'name': f'{self.cdump}prepobsaero'} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) From 5af325a6a4e0a14d180514a418603ca79fada487 Mon Sep 17 00:00:00 2001 From: Dan Holdaway <27729500+danholdaway@users.noreply.github.com> Date: Fri, 14 Jun 2024 18:05:23 -0400 Subject: [PATCH 3/9] Update GDASapp hash to move JCB into GDASapp (#2665) This PR moves JCB into GDASapp. The PR also bumps up the hash of GDASapp to what is in `feature/move_jcb`, which at time of writing is develop plus the absorption of JCB into GDASapp. Note that I also took the changes from https://github.com/NOAA-EMC/global-workflow/pull/2641 to follow the testing @RussTreadon-NOAA has done. --- .gitmodules | 4 ---- sorc/gdas.cd | 2 +- sorc/jcb | 1 - sorc/link_workflow.sh | 10 +++++++++- 4 files changed, 10 insertions(+), 7 deletions(-) delete mode 160000 sorc/jcb diff --git a/.gitmodules b/.gitmodules index 1c3b7acfa5..5c9e569243 100644 --- a/.gitmodules +++ b/.gitmodules @@ -26,7 +26,3 @@ [submodule "sorc/gsi_monitor.fd"] path = sorc/gsi_monitor.fd url = https://github.com/NOAA-EMC/GSI-Monitor.git -[submodule "sorc/jcb"] - path = sorc/jcb - url = https://github.com/noaa-emc/jcb - fetchRecurseSubmodules = false diff --git a/sorc/gdas.cd b/sorc/gdas.cd index 2bd8ffc149..368c9c5db9 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit 2bd8ffc149f30c0ae0f8e1508477a8dc88c46a65 +Subproject commit 368c9c5db9b5ea62e72937b6d1b0f753adb9be40 diff --git a/sorc/jcb b/sorc/jcb deleted file mode 160000 index f62b9df37f..0000000000 --- a/sorc/jcb +++ /dev/null @@ -1 +0,0 @@ -Subproject commit f62b9df37f131c9ff68b62eb6e19c1109c314cf0 diff --git a/sorc/link_workflow.sh b/sorc/link_workflow.sh index 1f0de0745c..4973ab8d7d 100755 --- a/sorc/link_workflow.sh +++ b/sorc/link_workflow.sh @@ -87,13 +87,21 @@ source "${HOMEgfs}/versions/fix.ver" # Link python pacakges in ush/python # TODO: This will be unnecessary when these are part of the virtualenv -packages=("wxflow" "jcb") +packages=("wxflow") for package in "${packages[@]}"; do cd "${HOMEgfs}/ush/python" || exit 1 [[ -s "${package}" ]] && rm -f "${package}" ${LINK} "${HOMEgfs}/sorc/${package}/src/${package}" . done +# Link GDASapp python packages in ush/python +packages=("jcb") +for package in "${packages[@]}"; do + cd "${HOMEgfs}/ush/python" || exit 1 + [[ -s "${package}" ]] && rm -f "${package}" + ${LINK} "${HOMEgfs}/sorc/gdas.cd/sorc/${package}/src/${package}" . +done + # Link wxflow in workflow and ci/scripts # TODO: This will be unnecessary when wxflow is part of the virtualenv cd "${HOMEgfs}/workflow" || exit 1 From 38f2df9fb0c074b1f80d3c637080be79be693161 Mon Sep 17 00:00:00 2001 From: David Huber <69919478+DavidHuber-NOAA@users.noreply.github.com> Date: Mon, 17 Jun 2024 17:12:55 +0000 Subject: [PATCH 4/9] Optimize wavepostpnt (#2657) Optimize the gfswavepostpnt, gfswavepostbndpntbll, and gfswavepostbndpnt jobs This is done by 1) reducing the number of calls to `sed`, `awk`, `grep`, and `cat` by - performing operations on all files at once instead of looping over each file - removing piped `cat` calls (e.g. `cat | sed 'something'`) - combining `sed` and `grep` calls when possible - adding logic to `awk` calls instead of handling that logic in bash 2) minimizing as much as possible the amount of data on disk that has to be read in (e.g. limiting sed to read only the line numbers it needs) --------- Co-authored-by: Walter Kolczynski - NOAA --- parm/config/gefs/config.resources | 2 +- parm/config/gefs/config.wave | 2 +- parm/config/gfs/config.resources | 2 +- parm/config/gfs/config.wave | 2 +- scripts/exgfs_wave_post_pnt.sh | 60 +++++++++++++++---------------- ush/wave_outp_spec.sh | 39 ++++++++------------ workflow/rocoto/gefs_tasks.py | 7 ++-- workflow/rocoto/gfs_tasks.py | 6 +++- 8 files changed, 58 insertions(+), 62 deletions(-) diff --git a/parm/config/gefs/config.resources b/parm/config/gefs/config.resources index 04d55ae082..b2ca5b3e51 100644 --- a/parm/config/gefs/config.resources +++ b/parm/config/gefs/config.resources @@ -255,7 +255,7 @@ case ${step} in # The wavepost*pnt* jobs are I/O heavy and do not scale well to large nodes. # Limit the number of tasks/node to 40. "wavepostbndpnt") - export wtime_wavepostbndpnt="01:00:00" + export wtime_wavepostbndpnt="03:00:00" export npe_wavepostbndpnt=240 export nth_wavepostbndpnt=1 export npe_node_wavepostbndpnt=$(( npe_node_max / nth_wavepostbndpnt )) diff --git a/parm/config/gefs/config.wave b/parm/config/gefs/config.wave index b61a2f6e54..bef3437adf 100644 --- a/parm/config/gefs/config.wave +++ b/parm/config/gefs/config.wave @@ -87,7 +87,7 @@ export WAVEWND_FID= # The start time reflects the number of hindcast hours prior to the cycle initial time export WAVHINDH=0 export FHMAX_WAV_IBP=180 -if (( FHMAX_WAV < FHMAX_WAV_IBP )); then export FHMAX_WAV_IBP=${FHMAX_GFS} ; fi +if (( FHMAX_WAV < FHMAX_WAV_IBP )); then export FHMAX_WAV_IBP=${FHMAX_WAV} ; fi # gridded and point output rate export DTFLD_WAV=$(( FHOUT_HF_WAV * 3600 )) diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources index 1d1fd2e3c1..e5f741cf7e 100644 --- a/parm/config/gfs/config.resources +++ b/parm/config/gfs/config.resources @@ -144,7 +144,7 @@ case ${step} in # The wavepost*pnt* jobs are I/O heavy and do not scale well to large nodes. # Limit the number of tasks/node to 40. "wavepostbndpnt") - export wtime_wavepostbndpnt="01:00:00" + export wtime_wavepostbndpnt="03:00:00" export npe_wavepostbndpnt=240 export nth_wavepostbndpnt=1 export npe_node_wavepostbndpnt=$(( npe_node_max / nth_wavepostbndpnt )) diff --git a/parm/config/gfs/config.wave b/parm/config/gfs/config.wave index 7253ef1396..568aeb1e1c 100644 --- a/parm/config/gfs/config.wave +++ b/parm/config/gfs/config.wave @@ -102,7 +102,7 @@ if [[ "${RUN}" == "gfs" ]]; then fi export WAVHINDH=0 export FHMAX_WAV_IBP=180 -if (( FHMAX_WAV < FHMAX_WAV_IBP )); then export FHMAX_WAV_IBP=${FHMAX_GFS} ; fi +if (( FHMAX_WAV < FHMAX_WAV_IBP )); then export FHMAX_WAV_IBP=${FHMAX_WAV} ; fi # gridded and point output rate export DTFLD_WAV=$(( FHOUT_HF_WAV * 3600 )) diff --git a/scripts/exgfs_wave_post_pnt.sh b/scripts/exgfs_wave_post_pnt.sh index 56cb83be21..93bdbeaf32 100755 --- a/scripts/exgfs_wave_post_pnt.sh +++ b/scripts/exgfs_wave_post_pnt.sh @@ -290,14 +290,14 @@ source "${USHgfs}/preamble.sh" fi # Create new buoy_log.ww3 - cat buoy.loc | awk '{print $3}' | sed 's/'\''//g' > ibp_tags + awk '{print $3}' buoy.loc | sed 's/'\''//g' > ibp_tags grep -F -f ibp_tags buoy_log.ww3 > buoy_log.tmp rm -f buoy_log.dat mv buoy_log.tmp buoy_log.dat grep -F -f ibp_tags buoy_lst.loc > buoy_tmp1.loc #sed '$d' buoy_tmp1.loc > buoy_tmp2.loc - buoys=$(awk '{ print $1 }' buoy_tmp1.loc) + awk '{ print $1 }' buoy_tmp1.loc > buoy_lst.txt Nb=$(wc buoy_tmp1.loc | awk '{ print $1 }') rm -f buoy_tmp1.loc @@ -350,6 +350,8 @@ source "${USHgfs}/preamble.sh" # 1.a.2 Loop over forecast time to generate post files fhr=$FHMIN_WAV + # Generated sed-searchable paths + escaped_USHgfs="${USHgfs//\//\\\/}" while [ $fhr -le $FHMAX_WAV_PNT ]; do echo " Creating the wave point scripts at : $(date)" @@ -366,6 +368,7 @@ source "${USHgfs}/preamble.sh" # Create instances of directories for spec and gridded output export SPECDATA=${DATA}/output_$YMDHMS + escaped_SPECDATA="${SPECDATA//\//\\\/}" export BULLDATA=${DATA}/output_$YMDHMS cp $DATA/mod_def.${waveuoutpGRD} mod_def.${waveuoutpGRD} @@ -386,19 +389,15 @@ source "${USHgfs}/preamble.sh" if [ "$DOSPC_WAV" = 'YES' ] then export dtspec=3600. - for buoy in $buoys - do - echo "${USHgfs}/wave_outp_spec.sh $buoy $ymdh spec $SPECDATA > $SPECDATA/spec_$buoy.out 2>&1" >> tmpcmdfile.$FH3 - done + # Construct the wave_outp_spec (spec) command to run on each buoy in buoy_lst.txt + sed "s/^\(.*\)$/${escaped_USHgfs}\/wave_outp_spec.sh \1 ${ymdh} spec ${escaped_SPECDATA} > ${escaped_SPECDATA}\/spec_\1.out 2>\&1/" buoy_lst.txt >> "tmpcmdfile.${FH3}" fi if [ "$DOBLL_WAV" = 'YES' ] then export dtspec=3600. - for buoy in $buoys - do - echo "${USHgfs}/wave_outp_spec.sh $buoy $ymdh bull $SPECDATA > $SPECDATA/bull_$buoy.out 2>&1" >> tmpcmdfile.$FH3 - done + # Construct the wave_outp_spec (bull) command to run on each buoy in buoy_lst.txt + sed "s/^\(.*\)$/${escaped_USHgfs}\/wave_outp_spec.sh \1 ${ymdh} bull ${escaped_SPECDATA} > ${escaped_SPECDATA}\/bull_\1.out 2>\&1/" buoy_lst.txt >> "tmpcmdfile.${FH3}" fi split -n l/1/10 tmpcmdfile.$FH3 > cmdfile.${FH3}.01 @@ -504,27 +503,24 @@ source "${USHgfs}/preamble.sh" cd $DATA - echo "Before create cmdfile for cat bouy : $(date)" - rm -f cmdfile.bouy - touch cmdfile.bouy - chmod 744 cmdfile.bouy + echo "Before create cmdfile for cat buoy : $(date)" + rm -f cmdfile.buoy + touch cmdfile.buoy + chmod 744 cmdfile.buoy CATOUTDIR=${DATA}/pnt_cat_out + escaped_CATOUTDIR="${CATOUTDIR//\//\\\/}" mkdir -p ${CATOUTDIR} if [ "$DOSPC_WAV" = 'YES' ] then - for buoy in $buoys - do - echo "${USHgfs}/wave_outp_cat.sh $buoy $FHMAX_WAV_PNT spec > ${CATOUTDIR}/spec_cat_$buoy.out 2>&1" >> cmdfile.bouy - done + # Construct wave_outp_cat (spec) call for each buoy in buoy_lst.txt + sed "s/^\(.*\)$/${escaped_USHgfs}\/wave_outp_cat.sh \1 ${FHMAX_WAV_PNT} spec > ${escaped_CATOUTDIR}\/spec_cat_\1.out 2>\&1/" buoy_lst.txt >> cmdfile.buoy fi if [ "$DOBLL_WAV" = 'YES' ] then - for buoy in $buoys - do - echo "${USHgfs}/wave_outp_cat.sh $buoy $FHMAX_WAV_PNT bull > ${CATOUTDIR}/bull_cat_$buoy.out 2>&1" >> cmdfile.bouy - done + # Construct wave_outp_cat (bull) call for each buoy in buoy_lst.txt + sed "s/^\(.*\)$/${escaped_USHgfs}\/wave_outp_cat.sh \1 ${FHMAX_WAV_PNT} bull > ${escaped_CATOUTDIR}\/bull_cat_\1.out 2>\&1/" buoy_lst.txt >> cmdfile.buoy fi if [ ${CFP_MP:-"NO"} = "YES" ]; then @@ -532,18 +528,18 @@ source "${USHgfs}/preamble.sh" ifile=0 iline=1 ifirst='yes' - nlines=$( wc -l cmdfile.bouy | awk '{print $1}' ) + nlines=$( wc -l < cmdfile.buoy) while [ $iline -le $nlines ]; do - line=$( sed -n ''$iline'p' cmdfile.bouy ) + line=$( sed -n ''$iline'p' cmdfile.buoy ) if [ -z "$line" ]; then break else if [ "$ifirst" = 'yes' ]; then - echo "#!/bin/sh" > cmdfile.bouy.$nfile - echo "$nfile cmdfile.bouy.$nfile" >> cmdmprogbouy - chmod 744 cmdfile.bouy.$nfile + echo "#!/bin/sh" > cmdfile.buoy.$nfile + echo "$nfile cmdfile.buoy.$nfile" >> cmdmprogbuoy + chmod 744 cmdfile.buoy.$nfile fi - echo $line >> cmdfile.bouy.$nfile + echo $line >> cmdfile.buoy.$nfile nfile=$(( nfile + 1 )) if [ $nfile -eq $NTASKS ]; then nfile=0 @@ -554,7 +550,7 @@ source "${USHgfs}/preamble.sh" done fi - wavenproc=$(wc -l cmdfile.bouy | awk '{print $1}') + wavenproc=$(wc -l < cmdfile.buoy) wavenproc=$(echo $((${wavenproc}<${NTASKS}?${wavenproc}:${NTASKS}))) set +x @@ -567,9 +563,11 @@ source "${USHgfs}/preamble.sh" if [ "$wavenproc" -gt '1' ] then if [ ${CFP_MP:-"NO"} = "YES" ]; then - ${wavempexec} -n ${wavenproc} ${wave_mpmd} cmdmprogbouy + # shellcheck disable=SC2086 + ${wavempexec} -n "${wavenproc}" ${wave_mpmd} cmdmprogbuoy else - ${wavempexec} ${wavenproc} ${wave_mpmd} cmdfile.bouy + # shellcheck disable=SC2086 + ${wavempexec} "${wavenproc}" ${wave_mpmd} cmdfile.buoy fi exit=$? else diff --git a/ush/wave_outp_spec.sh b/ush/wave_outp_spec.sh index 159d0eb2cf..37accbae49 100755 --- a/ush/wave_outp_spec.sh +++ b/ush/wave_outp_spec.sh @@ -74,21 +74,7 @@ source "${USHgfs}/preamble.sh" exit 1 else buoy=$bloc - grep $buoy ${DATA}/buoy_log.ww3 > tmp_list.loc - while read line - do - buoy_name=$(echo $line | awk '{print $2}') - if [ $buoy = $buoy_name ] - then - point=$(echo $line | awk '{ print $1 }') - set +x - echo " Location ID/# : $buoy (${point})" - echo " Spectral output start time : $ymdh " - echo ' ' - set_trace - break - fi - done < tmp_list.loc + point=$(awk "{if (\$2 == \"${buoy}\"){print \$1; exit} }" "${DATA}/buoy_log.ww3") if [ -z "$point" ] then set +x @@ -98,6 +84,11 @@ source "${USHgfs}/preamble.sh" echo ' ' set_trace exit 2 + else + set +x + echo " Location ID/# : $buoy (${point})" + echo " Spectral output start time : $ymdh " + echo ' ' fi fi @@ -201,27 +192,27 @@ source "${USHgfs}/preamble.sh" then if [ "$specdir" = "bull" ] then - cat $outfile | sed -e '9,$d' >> ${STA_DIR}/${specdir}fhr/$WAV_MOD_TAG.${ymdh}.$buoy.bull - cat $coutfile | sed -e '8,$d' >> ${STA_DIR}/c${specdir}fhr/$WAV_MOD_TAG.${ymdh}.$buoy.cbull + sed '9,$d' "${outfile}" >> "${STA_DIR}/${specdir}fhr/${WAV_MOD_TAG}.${ymdh}.${buoy}.bull" + sed '8,$d' "${coutfile}" >> "${STA_DIR}/c${specdir}fhr/${WAV_MOD_TAG}.${ymdh}.${buoy}.cbull" else - cat $outfile >> ${STA_DIR}/${specdir}fhr/$WAV_MOD_TAG.${ymdh}.$buoy.spec + cat $outfile >> "${STA_DIR}/${specdir}fhr/${WAV_MOD_TAG}.${ymdh}.${buoy}.spec" fi elif [ "${ymdh}" = "${YMDHE}" ] then if [ "$specdir" = "bull" ] then - cat $outfile | sed -e '1,7d' >> ${STA_DIR}/${specdir}fhr/$WAV_MOD_TAG.${ymdh}.$buoy.bull - cat $coutfile | sed -e '1,6d' >> ${STA_DIR}/c${specdir}fhr/$WAV_MOD_TAG.${ymdh}.$buoy.cbull + sed '1,7d' "${outfile}" >> "${STA_DIR}/${specdir}fhr/${WAV_MOD_TAG}.${ymdh}.${buoy}.bull" + sed '1,6d' "${coutfile}" >> "${STA_DIR}/c${specdir}fhr/${WAV_MOD_TAG}.${ymdh}.${buoy}.cbull" else - cat $outfile | sed -n "/^${YMD} ${HMS}$/,\$p" >> ${STA_DIR}/${specdir}fhr/$WAV_MOD_TAG.${ymdh}.$buoy.spec + sed -n "/^${YMD} ${HMS}$/,\$p" "${outfile}" >> "${STA_DIR}/${specdir}fhr/${WAV_MOD_TAG}.${ymdh}.${buoy}.spec" fi else if [ "$specdir" = "bull" ] then - cat $outfile | sed -e '1,7d' | sed -e '2,$d' >> ${STA_DIR}/${specdir}fhr/$WAV_MOD_TAG.${ymdh}.$buoy.bull - cat $coutfile | sed -e '1,6d' | sed -e '2,$d' >> ${STA_DIR}/c${specdir}fhr/$WAV_MOD_TAG.${ymdh}.$buoy.cbull + sed '8q;d' "${outfile}" >> "${STA_DIR}/${specdir}fhr/${WAV_MOD_TAG}.${ymdh}.${buoy}.bull" + sed '7q;d' "${coutfile}" >> "${STA_DIR}/c${specdir}fhr/${WAV_MOD_TAG}.${ymdh}.${buoy}.cbull" else - cat $outfile | sed -n "/^${YMD} ${HMS}$/,\$p" >> ${STA_DIR}/${specdir}fhr/$WAV_MOD_TAG.${ymdh}.$buoy.spec + sed -n "/^${YMD} ${HMS}$/,\$p" "${outfile}" >> "${STA_DIR}/${specdir}fhr/${WAV_MOD_TAG}.${ymdh}.${buoy}.spec" fi fi else diff --git a/workflow/rocoto/gefs_tasks.py b/workflow/rocoto/gefs_tasks.py index 6899a655b6..6fffc881e0 100644 --- a/workflow/rocoto/gefs_tasks.py +++ b/workflow/rocoto/gefs_tasks.py @@ -408,8 +408,11 @@ def wavepostbndpnt(self): def wavepostbndpntbll(self): deps = [] atmos_hist_path = self._template_to_rocoto_cycstring(self._base["COM_ATMOS_HISTORY_TMPL"], {'MEMDIR': 'mem#member#'}) - # Is there any reason this is 180? - data = f'{atmos_hist_path}/{self.cdump}.t@Hz.atm.logf180.txt' + + # The wavepostbndpntbll job runs on forecast hours up to FHMAX_WAV_IBP + last_fhr = self._configs['wave']['FHMAX_WAV_IBP'] + + data = f'{atmos_hist_path}/{self.cdump}.t@Hz.atm.logf{last_fhr:03d}.txt' dep_dict = {'type': 'data', 'data': data} deps.append(rocoto.add_dependency(dep_dict)) diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index 0fd468b3b4..2c74d0f854 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -1200,9 +1200,13 @@ def wavepostbndpnt(self): return task def wavepostbndpntbll(self): + + # The wavepostbndpntbll job runs on forecast hours up to FHMAX_WAV_IBP + last_fhr = self._configs['wave']['FHMAX_WAV_IBP'] + deps = [] atmos_hist_path = self._template_to_rocoto_cycstring(self._base["COM_ATMOS_HISTORY_TMPL"]) - data = f'{atmos_hist_path}/{self.cdump}.t@Hz.atm.logf180.txt' + data = f'{atmos_hist_path}/{self.cdump}.t@Hz.atm.logf{last_fhr:03d}.txt' dep_dict = {'type': 'data', 'data': data} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep=deps) From 47b3a581c8257fa24411fb400df8bb0e1e04972a Mon Sep 17 00:00:00 2001 From: Walter Kolczynski - NOAA Date: Mon, 17 Jun 2024 22:55:38 -0400 Subject: [PATCH 5/9] Turn on high-frequency output in extended test (#2679) Turns on high-frequency (hourly) output in the extended products test to exercise that aspect of the code. This test only runs on WCOSS. Also adds the hooks to optionally turn on the metplus jobs, but that is deferred as they are not currently working correctly. --- ci/cases/yamls/gfs_extended_ci.yaml | 1 + parm/config/gefs/config.base | 2 +- parm/config/gefs/yaml/defaults.yaml | 3 ++- parm/config/gfs/config.base | 6 +++--- parm/config/gfs/yaml/defaults.yaml | 2 ++ 5 files changed, 9 insertions(+), 5 deletions(-) diff --git a/ci/cases/yamls/gfs_extended_ci.yaml b/ci/cases/yamls/gfs_extended_ci.yaml index f3a84c8fde..42ee612f3a 100644 --- a/ci/cases/yamls/gfs_extended_ci.yaml +++ b/ci/cases/yamls/gfs_extended_ci.yaml @@ -10,3 +10,4 @@ base: DO_NPOESS: "YES" DO_GENESIS_FSU: "NO" FHMAX_GFS: 384 + FHMAX_HF_GFS: 120 diff --git a/parm/config/gefs/config.base b/parm/config/gefs/config.base index 0bb29e31ae..e98eb41208 100644 --- a/parm/config/gefs/config.base +++ b/parm/config/gefs/config.base @@ -237,7 +237,7 @@ export FHMIN_GFS=0 export FHMIN=${FHMIN_GFS} export FHMAX_GFS=@FHMAX_GFS@ export FHOUT_GFS=6 -export FHMAX_HF_GFS=0 +export FHMAX_HF_GFS=@FHMAX_HF_GFS@ export FHOUT_HF_GFS=1 export FHOUT_OCN_GFS=6 export FHOUT_ICE_GFS=6 diff --git a/parm/config/gefs/yaml/defaults.yaml b/parm/config/gefs/yaml/defaults.yaml index 2341c35d05..f6797758da 100644 --- a/parm/config/gefs/yaml/defaults.yaml +++ b/parm/config/gefs/yaml/defaults.yaml @@ -9,5 +9,6 @@ base: DO_AWIPS: "NO" KEEPDATA: "NO" FHMAX_GFS: 120 + FHMAX_HF_GFS: 0 REPLAY_ICS: "NO" - + USE_OCN_PERTURB_FILES: "false" diff --git a/parm/config/gfs/config.base b/parm/config/gfs/config.base index fd1ffdc415..f893eaaf4e 100644 --- a/parm/config/gfs/config.base +++ b/parm/config/gfs/config.base @@ -291,8 +291,8 @@ export gfs_cyc=@gfs_cyc@ # 0: no GFS cycle, 1: 00Z only, 2: 00Z and 12Z only, 4: # GFS output and frequency export FHMIN_GFS=0 export FHMAX_GFS=@FHMAX_GFS@ -export FHOUT_GFS=3 # Must be 6 for S2S until #1629 is addressed; 3 for ops -export FHMAX_HF_GFS=0 +export FHOUT_GFS=3 # 3 for ops +export FHMAX_HF_GFS=@FHMAX_HF_GFS@ export FHOUT_HF_GFS=1 export FHOUT_OCN_GFS=6 export FHOUT_ICE_GFS=6 @@ -445,7 +445,7 @@ export netcdf_diag=".true." export binary_diag=".false." # Verification options -export DO_METP="NO" # Run METPLUS jobs - set METPLUS settings in config.metp; not supported with spack-stack +export DO_METP="@DO_METP@" # Run METPLUS jobs - set METPLUS settings in config.metp export DO_FIT2OBS="YES" # Run fit to observations package export DO_VRFY_OCEANDA="@DO_VRFY_OCEANDA@" # Run SOCA Ocean and Seaice DA verification tasks diff --git a/parm/config/gfs/yaml/defaults.yaml b/parm/config/gfs/yaml/defaults.yaml index d4a423f17e..b074b11dbf 100644 --- a/parm/config/gfs/yaml/defaults.yaml +++ b/parm/config/gfs/yaml/defaults.yaml @@ -13,7 +13,9 @@ base: DO_TRACKER: "YES" DO_GENESIS: "YES" DO_GENESIS_FSU: "NO" + DO_METP: "NO" FHMAX_GFS: 120 + FHMAX_HF_GFS: 0 DO_VRFY_OCEANDA: "NO" GSI_SOILANAL: "NO" EUPD_CYC: "gdas" From 35d4d99eaac669721add9ddcc793153e5ab3b30a Mon Sep 17 00:00:00 2001 From: "Henry R. Winterbottom" <49202169+HenryWinterbottom-NOAA@users.noreply.github.com> Date: Tue, 18 Jun 2024 08:06:53 -0600 Subject: [PATCH 6/9] Update archive job to use COMIN/COMOUT (#2668) NCO has requested that each COM variable specify whether it is an input or an output. This completes that process for the global-workflow archive task. Refs #2451 --------- Co-authored-by: Walter Kolczynski - NOAA Co-authored-by: Rahul Mahajan Co-authored-by: David Huber <69919478+DavidHuber-NOAA@users.noreply.github.com> --- jobs/JGDAS_ENKF_ARCHIVE | 4 +- jobs/JGLOBAL_ARCHIVE | 50 +++++++++---- parm/archive/arcdir.yaml.j2 | 52 ++++++------- parm/archive/chem.yaml.j2 | 2 +- parm/archive/enkf.yaml.j2 | 16 ++-- parm/archive/enkf_grp.yaml.j2 | 16 ++-- parm/archive/enkf_restarta_grp.yaml.j2 | 26 +++---- parm/archive/enkf_restartb_grp.yaml.j2 | 10 +-- parm/archive/gdas.yaml.j2 | 98 ++++++++++++------------- parm/archive/gdas_restarta.yaml.j2 | 34 ++++----- parm/archive/gdas_restartb.yaml.j2 | 10 +-- parm/archive/gdasice.yaml.j2 | 6 +- parm/archive/gdasice_restart.yaml.j2 | 2 +- parm/archive/gdasocean.yaml.j2 | 4 +- parm/archive/gdasocean_analysis.yaml.j2 | 26 +++---- parm/archive/gdasocean_restart.yaml.j2 | 4 +- parm/archive/gdaswave.yaml.j2 | 4 +- parm/archive/gdaswave_restart.yaml.j2 | 2 +- parm/archive/gfs_downstream.yaml.j2 | 10 +-- parm/archive/gfs_flux.yaml.j2 | 4 +- parm/archive/gfs_flux_1p00.yaml.j2 | 4 +- parm/archive/gfs_netcdfa.yaml.j2 | 12 +-- parm/archive/gfs_netcdfb.yaml.j2 | 4 +- parm/archive/gfs_pgrb2b.yaml.j2 | 16 ++-- parm/archive/gfs_restarta.yaml.j2 | 8 +- parm/archive/gfsa.yaml.j2 | 56 +++++++------- parm/archive/gfsb.yaml.j2 | 16 ++-- parm/archive/gfswave.yaml.j2 | 20 ++--- parm/archive/ice_6hravg.yaml.j2 | 4 +- parm/archive/ice_grib2.yaml.j2 | 12 +-- parm/archive/master_enkf.yaml.j2 | 26 +++---- parm/archive/ocean_6hravg.yaml.j2 | 2 +- parm/archive/ocean_grib2.yaml.j2 | 12 +-- scripts/exgdas_enkf_earc.py | 4 +- scripts/exglobal_archive.py | 4 +- ush/python/pygfs/task/archive.py | 19 ++--- 36 files changed, 312 insertions(+), 287 deletions(-) diff --git a/jobs/JGDAS_ENKF_ARCHIVE b/jobs/JGDAS_ENKF_ARCHIVE index 5ac46a73e7..7496acd8d4 100755 --- a/jobs/JGDAS_ENKF_ARCHIVE +++ b/jobs/JGDAS_ENKF_ARCHIVE @@ -11,8 +11,8 @@ export CDUMP=${RUN/enkf} YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COM_TOP MEMDIR="ensstat" YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \ - COM_ATMOS_ANALYSIS_ENSSTAT:COM_ATMOS_ANALYSIS_TMPL \ - COM_ATMOS_HISTORY_ENSSTAT:COM_ATMOS_HISTORY_TMPL + COMIN_ATMOS_ANALYSIS_ENSSTAT:COM_ATMOS_ANALYSIS_TMPL \ + COMIN_ATMOS_HISTORY_ENSSTAT:COM_ATMOS_HISTORY_TMPL ############################################################### # Run archive script diff --git a/jobs/JGLOBAL_ARCHIVE b/jobs/JGLOBAL_ARCHIVE index cec3505000..28bd820de1 100755 --- a/jobs/JGLOBAL_ARCHIVE +++ b/jobs/JGLOBAL_ARCHIVE @@ -9,21 +9,45 @@ source "${HOMEgfs}/ush/jjob_header.sh" -e "arch" -c "base arch" ############################################## export CDUMP=${RUN/enkf} -YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COM_ATMOS_ANALYSIS COM_ATMOS_BUFR COM_ATMOS_GEMPAK \ - COM_ATMOS_GENESIS COM_ATMOS_HISTORY COM_ATMOS_INPUT COM_ATMOS_MASTER COM_ATMOS_RESTART \ - COM_ATMOS_TRACK COM_ATMOS_WMO \ - COM_CHEM_HISTORY COM_CHEM_ANALYSIS\ - COM_MED_RESTART \ - COM_SNOW_ANALYSIS \ - COM_ICE_HISTORY COM_ICE_INPUT COM_ICE_RESTART COM_ICE_GRIB \ - COM_OBS COM_TOP \ - COM_OCEAN_HISTORY COM_OCEAN_RESTART COM_OCEAN_GRIB COM_OCEAN_NETCDF \ - COM_OCEAN_ANALYSIS \ - COM_WAVE_GRID COM_WAVE_HISTORY COM_WAVE_STATION COM_WAVE_RESTART \ - COM_ATMOS_OZNMON COM_ATMOS_RADMON COM_ATMOS_MINMON COM_CONF +YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \ + COMIN_ATMOS_ANALYSIS:COM_ATMOS_ANALYSIS_TMPL \ + COMIN_ATMOS_BUFR:COM_ATMOS_BUFR_TMPL \ + COMIN_ATMOS_GEMPAK:COM_ATMOS_GEMPAK_TMPL \ + COMIN_ATMOS_GENESIS:COM_ATMOS_GENESIS_TMPL \ + COMIN_ATMOS_HISTORY:COM_ATMOS_HISTORY_TMPL \ + COMIN_ATMOS_INPUT:COM_ATMOS_INPUT_TMPL \ + COMIN_ATMOS_MASTER:COM_ATMOS_MASTER_TMPL \ + COMIN_ATMOS_RESTART:COM_ATMOS_RESTART_TMPL \ + COMIN_ATMOS_TRACK:COM_ATMOS_TRACK_TMPL \ + COMIN_ATMOS_WMO:COM_ATMOS_WMO_TMPL \ + COMIN_CHEM_HISTORY:COM_CHEM_HISTORY_TMPL \ + COMIN_CHEM_ANALYSIS:COM_CHEM_ANALYSIS_TMPL \ + COMIN_MED_RESTART:COM_MED_RESTART_TMPL \ + COMIN_SNOW_ANALYSIS:COM_SNOW_ANALYSIS_TMPL \ + COMIN_ICE_HISTORY:COM_ICE_HISTORY_TMPL \ + COMIN_ICE_INPUT:COM_ICE_INPUT_TMPL \ + COMIN_ICE_RESTART:COM_ICE_RESTART_TMPL \ + COMIN_ICE_GRIB:COM_ICE_GRIB_TMPL \ + COMIN_OBS:COM_OBS_TMPL \ + COMIN_TOP:COM_TOP_TMPL \ + COMIN_OCEAN_HISTORY:COM_OCEAN_HISTORY_TMPL \ + COMIN_OCEAN_RESTART:COM_OCEAN_RESTART_TMPL \ + COMIN_OCEAN_GRIB:COM_OCEAN_GRIB_TMPL \ + COMIN_OCEAN_NETCDF:COM_OCEAN_NETCDF_TMPL \ + COMIN_OCEAN_ANALYSIS:COM_OCEAN_ANALYSIS_TMPL \ + COMIN_WAVE_GRID:COM_WAVE_GRID_TMPL \ + COMIN_WAVE_HISTORY:COM_WAVE_HISTORY_TMPL \ + COMIN_WAVE_STATION:COM_WAVE_STATION_TMPL \ + COMIN_WAVE_RESTART:COM_WAVE_RESTART_TMPL \ + COMIN_ATMOS_OZNMON:COM_ATMOS_OZNMON_TMPL \ + COMIN_ATMOS_RADMON:COM_ATMOS_RADMON_TMPL \ + COMIN_ATMOS_MINMON:COM_ATMOS_MINMON_TMPL \ + COMIN_CONF:COM_CONF_TMPL \ + COMOUT_ATMOS_TRACK:COM_ATMOS_TRACK_TMPL for grid in "0p25" "0p50" "1p00"; do - YMD=${PDY} HH=${cyc} GRID=${grid} declare_from_tmpl -rx "COM_ATMOS_GRIB_${grid}:COM_ATMOS_GRIB_GRID_TMPL" + YMD=${PDY} HH=${cyc} GRID=${grid} declare_from_tmpl -rx \ + "COMIN_ATMOS_GRIB_${grid}:COM_ATMOS_GRIB_GRID_TMPL" done ############################################################### diff --git a/parm/archive/arcdir.yaml.j2 b/parm/archive/arcdir.yaml.j2 index 1ed7422761..d6b1899ebc 100644 --- a/parm/archive/arcdir.yaml.j2 +++ b/parm/archive/arcdir.yaml.j2 @@ -15,15 +15,15 @@ Deterministic: &deterministic copy: # Cyclone forecasts, produced for both gdas and gfs cycles ## Only created if tracking is on and there were systems to track - {% if path_exists(COM_ATMOS_TRACK ~ "/atcfunix." ~ RUN ~ "." ~ cycle_YMDH) %} - - ["{{ COM_ATMOS_TRACK }}/atcfunix.{{ RUN }}.{{ cycle_YMDH }}", "{{ ARCDIR }}/atcfunix.{{ RUN }}.{{ cycle_YMDH }}"] - - ["{{ COM_ATMOS_TRACK }}/atcfunixp.{{ RUN }}.{{ cycle_YMDH }}", "{{ ARCDIR }}/atcfunixp.{{ RUN }}.{{ cycle_YMDH }}"] + {% if path_exists(COMIN_ATMOS_TRACK ~ "/atcfunix." ~ RUN ~ "." ~ cycle_YMDH) %} + - ["{{ COMIN_ATMOS_TRACK }}/atcfunix.{{ RUN }}.{{ cycle_YMDH }}", "{{ ARCDIR }}/atcfunix.{{ RUN }}.{{ cycle_YMDH }}"] + - ["{{ COMIN_ATMOS_TRACK }}/atcfunixp.{{ RUN }}.{{ cycle_YMDH }}", "{{ ARCDIR }}/atcfunixp.{{ RUN }}.{{ cycle_YMDH }}"] {% endif %} # Cyclone tracking data {% for basin in ["epac", "natl"] %} - {% if path_exists(COM_ATMOS_TRACK + "/" + basin) %} - - ["{{ COM_ATMOS_TRACK }}/{{ basin }}", "{{ ARCDIR }}/{{ basin }}"] + {% if path_exists(COMIN_ATMOS_TRACK + "/" + basin) %} + - ["{{ COMIN_ATMOS_TRACK }}/{{ basin }}", "{{ ARCDIR }}/{{ basin }}"] {% endif %} {% endfor %} @@ -31,25 +31,25 @@ Deterministic: &deterministic analysis: copy: # Analysis data (if we are running in cycled mode) - - ["{{ COM_ATMOS_GRIB_1p00 }}/{{ head }}pgrb2.1p00.anl", "{{ ARCDIR }}/pgbanl.{{ RUN }}.{{ cycle_YMDH }}.grib2"] + - ["{{ COMIN_ATMOS_GRIB_1p00 }}/{{ head }}pgrb2.1p00.anl", "{{ ARCDIR }}/pgbanl.{{ RUN }}.{{ cycle_YMDH }}.grib2"] {% if DO_JEDIATMVAR %} - - ["{{ COM_ATMOS_ANALYSIS }}/{{ head }}atmstat", "{{ ARCDIR }}/atmstat.{{ RUN }}.{{ cycle_YMDH }}"] + - ["{{ COMIN_ATMOS_ANALYSIS }}/{{ head }}atmstat", "{{ ARCDIR }}/atmstat.{{ RUN }}.{{ cycle_YMDH }}"] {% else %} - - ["{{ COM_ATMOS_ANALYSIS }}/{{ head }}gsistat", "{{ ARCDIR }}/gsistat.{{ RUN }}.{{ cycle_YMDH }}"] + - ["{{ COMIN_ATMOS_ANALYSIS }}/{{ head }}gsistat", "{{ ARCDIR }}/gsistat.{{ RUN }}.{{ cycle_YMDH }}"] {% endif %} {% if DO_JEDISNOWDA %} - - ["{{ COM_SNOW_ANALYSIS }}/{{ head }}snowstat.tgz", "{{ ARCDIR }}/snowstat.{{ RUN }}.{{ cycle_YMDH }}.tgz"] + - ["{{ COMIN_SNOW_ANALYSIS }}/{{ head }}snowstat.tgz", "{{ ARCDIR }}/snowstat.{{ RUN }}.{{ cycle_YMDH }}.tgz"] {% endif %} {% if AERO_ANL_CDUMP == RUN or AERO_ANL_CDUMP == "both" %} - - ["{{ COM_CHEM_ANALYSIS }}/{{ head }}aerostat", "{{ ARCDIR }}/aerostat.{{ RUN }}.{{ cycle_YMDH }}"] + - ["{{ COMIN_CHEM_ANALYSIS }}/{{ head }}aerostat", "{{ ARCDIR }}/aerostat.{{ RUN }}.{{ cycle_YMDH }}"] {% endif %} {% if DO_PREP_OBS_AERO %} - - ["{{ COM_OBS }}/{{ head }}aeroobs", "{{ ARCDIR }}/aeroobs.{{ RUN }}.{{ cycle_YMDH }}"] - - ["{{ COM_OBS }}/{{ head }}aerorawobs", "{{ ARCDIR }}/aerorawobs.{{ RUN }}.{{ cycle_YMDH }}"] + - ["{{ COMIN_OBS }}/{{ head }}aeroobs", "{{ ARCDIR }}/aeroobs.{{ RUN }}.{{ cycle_YMDH }}"] + - ["{{ COMIN_OBS }}/{{ head }}aerorawobs", "{{ ARCDIR }}/aerorawobs.{{ RUN }}.{{ cycle_YMDH }}"] {% endif %} {% endif %} @@ -62,18 +62,18 @@ gfs: # GFS specific gfs: copy: {% for fhr in range(0, FHMAX_GFS + 1, FHOUT_GFS) %} - - ["{{ COM_ATMOS_GRIB_1p00 }}/{{ head }}pgrb2.1p00.f{{ '%03d' % fhr }}", "{{ ARCDIR }}/pgbf{{ '%02d' % fhr }}.{{ RUN }}.{{ cycle_YMDH }}.grib2"] + - ["{{ COMIN_ATMOS_GRIB_1p00 }}/{{ head }}pgrb2.1p00.f{{ '%03d' % fhr }}", "{{ ARCDIR }}/pgbf{{ '%02d' % fhr }}.{{ RUN }}.{{ cycle_YMDH }}.grib2"] {% endfor %} # Cyclone genesis data (only present if there are storms) - {% if path_exists(COM_ATMOS_GENESIS ~ "/storms.gfso.atcf_gen." ~ cycle_YMDH) %} - - ["{{ COM_ATMOS_GENESIS }}/storms.gfso.atcf_gen.{{ cycle_YMDH }}", "{{ ARCDIR }}/storms.gfso.atcf_gen.{{ cycle_YMDH }}"] - - ["{{ COM_ATMOS_GENESIS }}/storms.gfso.atcf_gen.altg.{{ cycle_YMDH }}", "{{ ARCDIR }}/storms.gfso.atcf_gen.altg.{{ cycle_YMDH }}"] + {% if path_exists(COMIN_ATMOS_GENESIS ~ "/storms.gfso.atcf_gen." ~ cycle_YMDH) %} + - ["{{ COMIN_ATMOS_GENESIS }}/storms.gfso.atcf_gen.{{ cycle_YMDH }}", "{{ ARCDIR }}/storms.gfso.atcf_gen.{{ cycle_YMDH }}"] + - ["{{ COMIN_ATMOS_GENESIS }}/storms.gfso.atcf_gen.altg.{{ cycle_YMDH }}", "{{ ARCDIR }}/storms.gfso.atcf_gen.altg.{{ cycle_YMDH }}"] {% endif %} - {% if path_exists(COM_ATMOS_GENESIS ~ "/trak.gfso.atcfunix." ~ cycle_YMDH) %} - - ["{{ COM_ATMOS_GENESIS }}/trak.gfso.atcfunix.{{ cycle_YMDH }}", "{{ ARCDIR }}/trak.gfso.atcfunix.{{ cycle_YMDH }}"] - - ["{{ COM_ATMOS_GENESIS }}/trak.gfso.atcfunix.altg.{{ cycle_YMDH }}", "{{ ARCDIR }}/trak.gfso.atcfunix.altg.{{ cycle_YMDH }}"] + {% if path_exists(COMIN_ATMOS_GENESIS ~ "/trak.gfso.atcfunix." ~ cycle_YMDH) %} + - ["{{ COMIN_ATMOS_GENESIS }}/trak.gfso.atcfunix.{{ cycle_YMDH }}", "{{ ARCDIR }}/trak.gfso.atcfunix.{{ cycle_YMDH }}"] + - ["{{ COMIN_ATMOS_GENESIS }}/trak.gfso.atcfunix.altg.{{ cycle_YMDH }}", "{{ ARCDIR }}/trak.gfso.atcfunix.altg.{{ cycle_YMDH }}"] {% endif %} {% if DO_FIT2OBS %} @@ -87,8 +87,8 @@ gfs: # GFS specific {% for fhr in range(0, FHMAX_FITS + 1, 6) %} {% set sfcfile = "/" + head + "sfcf" + '%03d'|format(fhr) + ".nc" %} {% set sigfile = "/" + head + "atmf" + '%03d'|format(fhr) + ".nc" %} - - ["{{COM_ATMOS_HISTORY}}/{{ sfcfile }}", "{{ VFYARC }}/{{ RUN }}.{{ cycle_YMD }}/{{ cycle_HH }}/{{ sfcfile }}"] - - ["{{COM_ATMOS_HISTORY}}/{{ sigfile }}", "{{ VFYARC }}/{{ RUN }}.{{ cycle_YMD }}/{{ cycle_HH }}/{{ sigfile }}"] + - ["{{COMIN_ATMOS_HISTORY}}/{{ sfcfile }}", "{{ VFYARC }}/{{ RUN }}.{{ cycle_YMD }}/{{ cycle_HH }}/{{ sfcfile }}"] + - ["{{COMIN_ATMOS_HISTORY}}/{{ sigfile }}", "{{ VFYARC }}/{{ RUN }}.{{ cycle_YMD }}/{{ cycle_HH }}/{{ sigfile }}"] {% endfor %} {% endif %} @@ -101,7 +101,7 @@ gdas: # GDAS specific gdas: copy: {% for fhr in range(0, FHMAX + 1, FHOUT) %} - - ["{{ COM_ATMOS_GRIB_1p00 }}/{{ head }}pgrb2.1p00.f{{ '%03d' % fhr }}", "{{ ARCDIR }}/pgbf{{ '%02d' % fhr }}.{{ RUN }}.{{ cycle_YMDH }}.grib2"] + - ["{{ COMIN_ATMOS_GRIB_1p00 }}/{{ head }}pgrb2.1p00.f{{ '%03d' % fhr }}", "{{ ARCDIR }}/pgbf{{ '%02d' % fhr }}.{{ RUN }}.{{ cycle_YMDH }}.grib2"] {% endfor %} {% endif %} @@ -110,11 +110,11 @@ Ensemble: &ensemble # ENKFGDAS or ENKFGFS copy: # Copy ensemble analyses {% if DO_JEDIATMENS %} - - ["{{ COM_ATMOS_ANALYSIS_ENSSTAT }}/{{ head }}atmensstat", "{{ ARCDIR }}/atmensstat.{{ RUN }}.{{ cycle_YMDH }}"] - - ["{{ COM_ATMOS_ANALYSIS_ENSSTAT }}/{{ head }}atminc.ensmean.nc", "{{ ARCDIR }}/atmensstat.{{ RUN }}.{{ cycle_YMDH }}.ensmean.nc"] + - ["{{ COMIN_ATMOS_ANALYSIS_ENSSTAT }}/{{ head }}atmensstat", "{{ ARCDIR }}/atmensstat.{{ RUN }}.{{ cycle_YMDH }}"] + - ["{{ COMIN_ATMOS_ANALYSIS_ENSSTAT }}/{{ head }}atminc.ensmean.nc", "{{ ARCDIR }}/atmensstat.{{ RUN }}.{{ cycle_YMDH }}.ensmean.nc"] {% else %} - - ["{{ COM_ATMOS_ANALYSIS_ENSSTAT }}/{{ head }}enkfstat", "{{ ARCDIR }}/enkfstat.{{ RUN }}.{{ cycle_YMDH }}"] - - ["{{ COM_ATMOS_ANALYSIS_ENSSTAT }}/{{ head }}gsistat.ensmean", "{{ ARCDIR }}/gsistat.{{ RUN }}.{{ cycle_YMDH }}.ensmean"] + - ["{{ COMIN_ATMOS_ANALYSIS_ENSSTAT }}/{{ head }}enkfstat", "{{ ARCDIR }}/enkfstat.{{ RUN }}.{{ cycle_YMDH }}"] + - ["{{ COMIN_ATMOS_ANALYSIS_ENSSTAT }}/{{ head }}gsistat.ensmean", "{{ ARCDIR }}/gsistat.{{ RUN }}.{{ cycle_YMDH }}.ensmean"] {% endif %} {% if "enkf" in RUN %} diff --git a/parm/archive/chem.yaml.j2 b/parm/archive/chem.yaml.j2 index 7796912b1a..33c6dcef57 100644 --- a/parm/archive/chem.yaml.j2 +++ b/parm/archive/chem.yaml.j2 @@ -4,4 +4,4 @@ chem: target: "{{ ATARDIR }}/{{ cycle_YMDH }}/chem.tar" required: # TODO explicitize this set - - "{{ COM_CHEM_HISTORY | relpath(ROTDIR) }}/{{ head }}*" + - "{{ COMIN_CHEM_HISTORY | relpath(ROTDIR) }}/{{ head }}*" diff --git a/parm/archive/enkf.yaml.j2 b/parm/archive/enkf.yaml.j2 index ed50a1749d..bc5ef03cb8 100644 --- a/parm/archive/enkf.yaml.j2 +++ b/parm/archive/enkf.yaml.j2 @@ -34,10 +34,10 @@ enkf: # Ensemble mean and spread {% for fhr in range(3, fhmax + 1, 3) %} - - "{{ COM_ATMOS_HISTORY_ENSSTAT | relpath(ROTDIR) }}/{{ head }}atmf{{ '%03d' % fhr }}.ensmean.nc" - - "{{ COM_ATMOS_HISTORY_ENSSTAT | relpath(ROTDIR) }}/{{ head }}sfcf{{ '%03d' % fhr }}.ensmean.nc" + - "{{ COMIN_ATMOS_HISTORY_ENSSTAT | relpath(ROTDIR) }}/{{ head }}atmf{{ '%03d' % fhr }}.ensmean.nc" + - "{{ COMIN_ATMOS_HISTORY_ENSSTAT | relpath(ROTDIR) }}/{{ head }}sfcf{{ '%03d' % fhr }}.ensmean.nc" {% if ENKF_SPREAD %} - - "{{ COM_ATMOS_HISTORY_ENSSTAT | relpath(ROTDIR) }}/{{ head }}atmf{{ '%03d' % fhr }}.ensspread.nc" + - "{{ COMIN_ATMOS_HISTORY_ENSSTAT | relpath(ROTDIR) }}/{{ head }}atmf{{ '%03d' % fhr }}.ensspread.nc" {% endif %} {% endfor %} @@ -54,15 +54,15 @@ enkf: "atmensstat"] %} {% endif %} {% for file in da_files %} - - "{{ COM_ATMOS_ANALYSIS_ENSSTAT | relpath(ROTDIR) }}/{{ head }}{{ file }}" + - "{{ COMIN_ATMOS_ANALYSIS_ENSSTAT | relpath(ROTDIR) }}/{{ head }}{{ file }}" {% endfor %} # Ensemble mean analyses/increments # 6-hr analysis/increment {% if do_calc_increment %} - - "{{ COM_ATMOS_ANALYSIS_ENSSTAT | relpath(ROTDIR) }}/{{ head }}atmanl.ensmean.nc" + - "{{ COMIN_ATMOS_ANALYSIS_ENSSTAT | relpath(ROTDIR) }}/{{ head }}atmanl.ensmean.nc" {% else %} - - "{{ COM_ATMOS_ANALYSIS_ENSSTAT | relpath(ROTDIR) }}/{{ head }}atminc.ensmean.nc" + - "{{ COMIN_ATMOS_ANALYSIS_ENSSTAT | relpath(ROTDIR) }}/{{ head }}atminc.ensmean.nc" {% endif %} {% if DOIAU %} @@ -71,10 +71,10 @@ enkf: {% for fhr in iaufhrs if fhr != 6 %} {% if do_calc_increment %} # Store analyses instead of increments - - "{{ COM_ATMOS_ANALYSIS_ENSSTAT | relpath(ROTDIR) }}/{{ head }}atma{{ '%03d' % fhr }}.ensmean.nc" + - "{{ COMIN_ATMOS_ANALYSIS_ENSSTAT | relpath(ROTDIR) }}/{{ head }}atma{{ '%03d' % fhr }}.ensmean.nc" {% else %} # Store increments - - "{{ COM_ATMOS_ANALYSIS_ENSSTAT | relpath(ROTDIR) }}/{{ head }}atmi{{ '%03d' % fhr }}.ensmean.nc" + - "{{ COMIN_ATMOS_ANALYSIS_ENSSTAT | relpath(ROTDIR) }}/{{ head }}atmi{{ '%03d' % fhr }}.ensmean.nc" {% endif %} {% endfor %} diff --git a/parm/archive/enkf_grp.yaml.j2 b/parm/archive/enkf_grp.yaml.j2 index 85d5854e3b..933ca45caf 100644 --- a/parm/archive/enkf_grp.yaml.j2 +++ b/parm/archive/enkf_grp.yaml.j2 @@ -5,25 +5,25 @@ enkf_grp: {% for mem in range(first_group_mem, last_group_mem + 1) %} {% set imem = mem - first_group_mem %} # Construct member COM directories - {% set COM_ATMOS_ANALYSIS_MEM = COM_ATMOS_ANALYSIS_MEM_list[imem] %} - {% set COM_ATMOS_HISTORY_MEM = COM_ATMOS_HISTORY_MEM_list[imem] %} - {% set COM_ATMOS_RESTART_MEM = COM_ATMOS_RESTART_MEM_list[imem] %} + {% set COMIN_ATMOS_ANALYSIS_MEM = COMIN_ATMOS_ANALYSIS_MEM_list[imem] %} + {% set COMIN_ATMOS_HISTORY_MEM = COMIN_ATMOS_HISTORY_MEM_list[imem] %} + {% set COMIN_ATMOS_RESTART_MEM = COMIN_ATMOS_RESTART_MEM_list[imem] %} # Forecast data {% for fhr in range(3, 10, 3) %} - - "{{ COM_ATMOS_HISTORY_MEM | relpath(ROTDIR) }}/{{ head }}atmf{{ "%03d" % fhr }}.nc" + - "{{ COMIN_ATMOS_HISTORY_MEM | relpath(ROTDIR) }}/{{ head }}atmf{{ "%03d" % fhr }}.nc" {% endfor %} # Only store the 6-hour surface forecast - - "{{ COM_ATMOS_HISTORY_MEM | relpath(ROTDIR) }}/{{ head }}sfcf006.nc" + - "{{ COMIN_ATMOS_HISTORY_MEM | relpath(ROTDIR) }}/{{ head }}sfcf006.nc" # Store the individual member analysis data {% if not lobsdiag_forenkf %} - - "{{ COM_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ head }}gsistat" + - "{{ COMIN_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ head }}gsistat" {% endif %} {% if do_calc_increment %} - - "{{ COM_ATMOS_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ head }}atmanl.nc" + - "{{ COMIN_ATMOS_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ head }}atmanl.nc" {% else %} - - "{{ COM_ATMOS_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ head }}ratminc.nc" + - "{{ COMIN_ATMOS_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ head }}ratminc.nc" {% endif %} {% endfor %} # first_group_mem to last_group_mem diff --git a/parm/archive/enkf_restarta_grp.yaml.j2 b/parm/archive/enkf_restarta_grp.yaml.j2 index 8a4ea18bc1..41e03edc92 100644 --- a/parm/archive/enkf_restarta_grp.yaml.j2 +++ b/parm/archive/enkf_restarta_grp.yaml.j2 @@ -5,16 +5,16 @@ enkf_restarta_grp: {% for mem in range(first_group_mem, last_group_mem + 1) %} {% set imem = mem - first_group_mem %} # Construct the pertinent member COM directories - {% set COM_ATMOS_ANALYSIS_MEM = COM_ATMOS_ANALYSIS_MEM_list[imem] %} - {% set COM_ATMOS_HISTORY_MEM = COM_ATMOS_HISTORY_MEM_list[imem] %} - {% set COM_ATMOS_RESTART_MEM = COM_ATMOS_RESTART_MEM_list[imem] %} + {% set COMIN_ATMOS_ANALYSIS_MEM = COMIN_ATMOS_ANALYSIS_MEM_list[imem] %} + {% set COMIN_ATMOS_HISTORY_MEM = COMIN_ATMOS_HISTORY_MEM_list[imem] %} + {% set COMIN_ATMOS_RESTART_MEM = COMIN_ATMOS_RESTART_MEM_list[imem] %} # Store bias data {% if not lobsdiag_forenkf %} - - "{{ COM_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ head }}abias" - - "{{ COM_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ head }}abias_air" - - "{{ COM_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ head }}abias_int" - - "{{ COM_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ head }}abias_pc" + - "{{ COMIN_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ head }}abias" + - "{{ COMIN_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ head }}abias_air" + - "{{ COMIN_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ head }}abias_int" + - "{{ COMIN_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ head }}abias_pc" {% endif %} # Member surface analysis data @@ -25,29 +25,29 @@ enkf_restarta_grp: {% endif %} {% set anl_time = current_cycle | add_to_datetime(anl_delta) %} {% for itile in range(1, 7) %} - - "{{ COM_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ anl_time | to_YMD }}.{{ anl_time | strftime("%H") }}0000.sfcanl_data.tile{{ itile }}.nc" + - "{{ COMIN_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ anl_time | to_YMD }}.{{ anl_time | strftime("%H") }}0000.sfcanl_data.tile{{ itile }}.nc" {% endfor %} # Member atmospheric analysis data {% if do_calc_increment %} - - "{{ COM_ATMOS_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ head }}atmanl.nc" + - "{{ COMIN_ATMOS_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ head }}atmanl.nc" {% else %} - - "{{ COM_ATMOS_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ head }}ratminc.nc" + - "{{ COMIN_ATMOS_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ head }}ratminc.nc" {% endif %} # Member increments {% for iaufhr in iaufhrs if iaufhr != 6 %} {% set iaufhr = iaufhr %} {% if do_calc_increment %} - - "{{ COM_ATMOS_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ head }}atma{{ '%03d' % iaufhr }}.nc" + - "{{ COMIN_ATMOS_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ head }}atma{{ '%03d' % iaufhr }}.nc" {% else %} - - "{{ COM_ATMOS_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ head }}ratmi{{ '%03d' % iaufhr }}.nc" + - "{{ COMIN_ATMOS_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ head }}ratmi{{ '%03d' % iaufhr }}.nc" {% endif %} {% endfor %} # iaufhr in iaufhrs # Conventional data {% if not lobsdiag_forenkf and not DO_JEDIATMENS %} - - "{{ COM_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ head }}cnvstat" + - "{{ COMIN_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ head }}cnvstat" {% endif %} {% endfor %} # first_group_mem to last_group_mem diff --git a/parm/archive/enkf_restartb_grp.yaml.j2 b/parm/archive/enkf_restartb_grp.yaml.j2 index 34fde9d7ca..7cd799f0a9 100644 --- a/parm/archive/enkf_restartb_grp.yaml.j2 +++ b/parm/archive/enkf_restartb_grp.yaml.j2 @@ -4,7 +4,7 @@ enkf_restartb_grp: required: {% for mem in range(first_group_mem, last_group_mem + 1) %} {% set imem = mem - first_group_mem %} - {% set COM_ATMOS_RESTART_MEM = COM_ATMOS_RESTART_MEM_list[imem] %} + {% set COMIN_ATMOS_RESTART_MEM = COMIN_ATMOS_RESTART_MEM_list[imem] %} # Grab surface analysis data. # If IAU is on, grab the beginning of the window, otherwise grab the center. @@ -18,7 +18,7 @@ enkf_restartb_grp: {% set offset_HH = offset_dt | strftime("%H") %} {% set prefix = offset_YMD + "." + offset_HH + "0000" %} {% for itile in range(1, 7) %} - - "{{ COM_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ prefix }}.sfcanl_data.tile{{ itile }}.nc" + - "{{ COMIN_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ prefix }}.sfcanl_data.tile{{ itile }}.nc" {% endfor %} # Now get the restart files. @@ -28,10 +28,10 @@ enkf_restartb_grp: {% set r_prefix = r_dt | to_YMD + "." + r_dt | strftime("%H") + "0000" %} {% for itile in range(1, 7) %} {% for datatype in ["ca_data", "fv_core.res", "fv_srf_wnd.res", "fv_tracer.res", "phy_data", "sfc_data"] %} - - "{{ COM_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ r_prefix }}.{{datatype}}.tile{{ itile }}.nc" + - "{{ COMIN_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ r_prefix }}.{{datatype}}.tile{{ itile }}.nc" {% endfor %} {% endfor %} - - "{{ COM_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ r_prefix }}.coupler.res" - - "{{ COM_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ r_prefix }}.fv_core.res.nc" + - "{{ COMIN_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ r_prefix }}.coupler.res" + - "{{ COMIN_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ r_prefix }}.fv_core.res.nc" {% endfor %} {% endfor %} diff --git a/parm/archive/gdas.yaml.j2 b/parm/archive/gdas.yaml.j2 index 26540156cd..12b89f2e61 100644 --- a/parm/archive/gdas.yaml.j2 +++ b/parm/archive/gdas.yaml.j2 @@ -35,22 +35,22 @@ gdas: {% endif %} # Analysis GRIB2 (sub-sampled) data - - "{{ COM_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p25.anl" - - "{{ COM_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p25.anl.idx" - - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2.1p00.anl" - - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2.1p00.anl.idx" + - "{{ COMIN_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p25.anl" + - "{{ COMIN_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p25.anl.idx" + - "{{ COMIN_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2.1p00.anl" + - "{{ COMIN_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2.1p00.anl.idx" # Analysis netCDF (raw) data - - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmanl.nc" - - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}sfcanl.nc" + - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmanl.nc" + - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}sfcanl.nc" {% if DOHYBVAR %} # Ensemble analysis residual - - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmanl.ensres.nc" + - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmanl.ensres.nc" {% if DOIAU %} # Ensemble IAU analysis residuals {% for fhr in iaufhrs if fhr != 6 %} - - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atma{{ '%03d' % fhr }}.ensres.nc" + - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atma{{ '%03d' % fhr }}.ensres.nc" {% endfor %} {% endif %} # End of ensemble analysis mean residuals @@ -58,51 +58,51 @@ gdas: # Analysis state {% if DO_JEDIATMVAR %} - - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmvar.yaml" - - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmstat" + - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmvar.yaml" + - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmstat" {% else %} - - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}gsistat" - - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}cnvstat" - - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}oznstat" - - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}radstat" + - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}gsistat" + - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}cnvstat" + - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}oznstat" + - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}radstat" {% endif %} {% if AERO_ANL_CDUMP == "gdas" or AERO_ANL_CDUMP == "both" %} - - "{{ COM_CHEM_ANALYSIS | relpath(ROTDIR) }}/{{ head }}aerostat" + - "{{ COMIN_CHEM_ANALYSIS | relpath(ROTDIR) }}/{{ head }}aerostat" {% endif %} {% if DO_PREP_OBS_AERO %} - - "{{ COM_OBS | relpath(ROTDIR) }}/{{ head }}aeroobs" - - "{{ COM_OBS | relpath(ROTDIR) }}/{{ head }}aerorawobs" + - "{{ COMIN_OBS | relpath(ROTDIR) }}/{{ head }}aeroobs" + - "{{ COMIN_OBS | relpath(ROTDIR) }}/{{ head }}aerorawobs" {% endif %} {% if DO_JEDISNOWDA %} - - "{{ COM_SNOW_ANALYSIS | relpath(ROTDIR) }}/{{ head }}snowstat.tgz" + - "{{ COMIN_SNOW_ANALYSIS | relpath(ROTDIR) }}/{{ head }}snowstat.tgz" {% endif %} # Ozone verification {% if DO_VERFOZN %} - - "{{ COM_ATMOS_OZNMON | relpath(ROTDIR) }}/time/bad_cnt.{{ cycle_YMDH }}" - - "{{ COM_ATMOS_OZNMON | relpath(ROTDIR) }}/time/bad_diag.{{ cycle_YMDH }}" - - "{{ COM_ATMOS_OZNMON | relpath(ROTDIR) }}/time/bad_pen.{{ cycle_YMDH }}" - - "{{ COM_ATMOS_OZNMON | relpath(ROTDIR) }}/time/stdout.time.tar.gz" - - "{{ COM_ATMOS_OZNMON | relpath(ROTDIR) }}/horiz/stdout.horiz.tar.gz" + - "{{ COMIN_ATMOS_OZNMON | relpath(ROTDIR) }}/time/bad_cnt.{{ cycle_YMDH }}" + - "{{ COMIN_ATMOS_OZNMON | relpath(ROTDIR) }}/time/bad_diag.{{ cycle_YMDH }}" + - "{{ COMIN_ATMOS_OZNMON | relpath(ROTDIR) }}/time/bad_pen.{{ cycle_YMDH }}" + - "{{ COMIN_ATMOS_OZNMON | relpath(ROTDIR) }}/time/stdout.time.tar.gz" + - "{{ COMIN_ATMOS_OZNMON | relpath(ROTDIR) }}/horiz/stdout.horiz.tar.gz" - "logs/{{ cycle_YMDH }}/{{ RUN }}verfozn.log" {% endif %} # Radiance verification {% if DO_VERFRAD %} - - "{{ COM_ATMOS_RADMON | relpath(ROTDIR) }}/radmon_angle.tar.gz" - - "{{ COM_ATMOS_RADMON | relpath(ROTDIR) }}/radmon_bcoef.tar.gz" - - "{{ COM_ATMOS_RADMON | relpath(ROTDIR) }}/radmon_bcor.tar.gz" - - "{{ COM_ATMOS_RADMON | relpath(ROTDIR) }}/radmon_time.tar.gz" + - "{{ COMIN_ATMOS_RADMON | relpath(ROTDIR) }}/radmon_angle.tar.gz" + - "{{ COMIN_ATMOS_RADMON | relpath(ROTDIR) }}/radmon_bcoef.tar.gz" + - "{{ COMIN_ATMOS_RADMON | relpath(ROTDIR) }}/radmon_bcor.tar.gz" + - "{{ COMIN_ATMOS_RADMON | relpath(ROTDIR) }}/radmon_time.tar.gz" - "logs/{{ cycle_YMDH }}/{{ RUN }}verfrad.log" {% endif %} # Minimization monitor {% if DO_VMINMON %} - - "{{ COM_ATMOS_MINMON | relpath(ROTDIR) }}/{{ cycle_YMDH }}.costs.txt" - - "{{ COM_ATMOS_MINMON | relpath(ROTDIR) }}/{{ cycle_YMDH }}.cost_terms.txt" - - "{{ COM_ATMOS_MINMON | relpath(ROTDIR) }}/{{ cycle_YMDH }}.gnorms.ieee_d" - - "{{ COM_ATMOS_MINMON | relpath(ROTDIR) }}/{{ cycle_YMDH }}.reduction.ieee_d" - - "{{ COM_ATMOS_MINMON | relpath(ROTDIR) }}/gnorm_data.txt" + - "{{ COMIN_ATMOS_MINMON | relpath(ROTDIR) }}/{{ cycle_YMDH }}.costs.txt" + - "{{ COMIN_ATMOS_MINMON | relpath(ROTDIR) }}/{{ cycle_YMDH }}.cost_terms.txt" + - "{{ COMIN_ATMOS_MINMON | relpath(ROTDIR) }}/{{ cycle_YMDH }}.gnorms.ieee_d" + - "{{ COMIN_ATMOS_MINMON | relpath(ROTDIR) }}/{{ cycle_YMDH }}.reduction.ieee_d" + - "{{ COMIN_ATMOS_MINMON | relpath(ROTDIR) }}/gnorm_data.txt" - "logs/{{ cycle_YMDH }}/{{ RUN }}vminmon.log" {% endif %} {% endif %} # End of cycled data @@ -118,27 +118,27 @@ gdas: {% for fhr in range(0, FHMAX + 1, 3) %} # Forecast GRIB2 data - - "{{ COM_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p25.f{{ '%03d' % fhr }}" - - "{{ COM_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p25.f{{ '%03d' % fhr }}.idx" - - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2.1p00.f{{ '%03d' % fhr }}" - - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2.1p00.f{{ '%03d' % fhr }}.idx" + - "{{ COMIN_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p25.f{{ '%03d' % fhr }}" + - "{{ COMIN_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p25.f{{ '%03d' % fhr }}.idx" + - "{{ COMIN_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2.1p00.f{{ '%03d' % fhr }}" + - "{{ COMIN_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2.1p00.f{{ '%03d' % fhr }}.idx" # Forecast GRIB2 fluxes - - "{{ COM_ATMOS_MASTER | relpath(ROTDIR) }}/{{ head }}sfluxgrbf{{ '%03d' % fhr }}.grib2" - - "{{ COM_ATMOS_MASTER | relpath(ROTDIR) }}/{{ head }}sfluxgrbf{{ '%03d' % fhr }}.grib2.idx" + - "{{ COMIN_ATMOS_MASTER | relpath(ROTDIR) }}/{{ head }}sfluxgrbf{{ '%03d' % fhr }}.grib2" + - "{{ COMIN_ATMOS_MASTER | relpath(ROTDIR) }}/{{ head }}sfluxgrbf{{ '%03d' % fhr }}.grib2.idx" # FV3 log - - "{{ COM_ATMOS_HISTORY | relpath(ROTDIR) }}/{{ head }}atm.logf{{ '%03d' % fhr }}.txt" + - "{{ COMIN_ATMOS_HISTORY | relpath(ROTDIR) }}/{{ head }}atm.logf{{ '%03d' % fhr }}.txt" # Raw netCDF forecasts - - "{{ COM_ATMOS_HISTORY | relpath(ROTDIR) }}/{{ head }}atmf{{ '%03d' % fhr }}.nc" - - "{{ COM_ATMOS_HISTORY | relpath(ROTDIR) }}/{{ head }}sfcf{{ '%03d' % fhr }}.nc" + - "{{ COMIN_ATMOS_HISTORY | relpath(ROTDIR) }}/{{ head }}atmf{{ '%03d' % fhr }}.nc" + - "{{ COMIN_ATMOS_HISTORY | relpath(ROTDIR) }}/{{ head }}sfcf{{ '%03d' % fhr }}.nc" {% endfor %} optional: {% if MODE == "cycled" %} {% if DO_VERFRAD %} # Radiance verification (only created if there are problems) - - "{{ COM_ATMOS_RADMON | relpath(ROTDIR) }}/bad_diag.{{ cycle_YMDH }}" - - "{{ COM_ATMOS_RADMON | relpath(ROTDIR) }}/bad_pen.{{ cycle_YMDH }}" - - "{{ COM_ATMOS_RADMON | relpath(ROTDIR) }}/low_count.{{ cycle_YMDH }}" - - "{{ COM_ATMOS_RADMON | relpath(ROTDIR) }}/warning.{{ cycle_YMDH }}" + - "{{ COMIN_ATMOS_RADMON | relpath(ROTDIR) }}/bad_diag.{{ cycle_YMDH }}" + - "{{ COMIN_ATMOS_RADMON | relpath(ROTDIR) }}/bad_pen.{{ cycle_YMDH }}" + - "{{ COMIN_ATMOS_RADMON | relpath(ROTDIR) }}/low_count.{{ cycle_YMDH }}" + - "{{ COMIN_ATMOS_RADMON | relpath(ROTDIR) }}/warning.{{ cycle_YMDH }}" {% endif %} {% if DO_VERFOZN %} @@ -150,10 +150,10 @@ gdas: {% for group in [ "horiz", "time" ] %} {% if group == "horiz" %} {% set suffix = ".gz" %} {% else %} {% set suffix = "" %} {% endif %} {% for type in oznmon_types %} - - "{{ COM_ATMOS_OZNMON | relpath(ROTDIR) }}/{{ group }}/{{ type }}.anl.ctl" - - "{{ COM_ATMOS_OZNMON | relpath(ROTDIR) }}/{{ group }}/{{ type }}.anl.{{ cycle_YMDH }}.ieee_d{{ suffix }}" - - "{{ COM_ATMOS_OZNMON | relpath(ROTDIR) }}/{{ group }}/{{ type }}.ges.ctl" - - "{{ COM_ATMOS_OZNMON | relpath(ROTDIR) }}/{{ group }}/{{ type }}.ges.{{ cycle_YMDH }}.ieee_d{{ suffix }}" + - "{{ COMIN_ATMOS_OZNMON | relpath(ROTDIR) }}/{{ group }}/{{ type }}.anl.ctl" + - "{{ COMIN_ATMOS_OZNMON | relpath(ROTDIR) }}/{{ group }}/{{ type }}.anl.{{ cycle_YMDH }}.ieee_d{{ suffix }}" + - "{{ COMIN_ATMOS_OZNMON | relpath(ROTDIR) }}/{{ group }}/{{ type }}.ges.ctl" + - "{{ COMIN_ATMOS_OZNMON | relpath(ROTDIR) }}/{{ group }}/{{ type }}.ges.{{ cycle_YMDH }}.ieee_d{{ suffix }}" {% endfor %} {% endfor %} {% endif %} diff --git a/parm/archive/gdas_restarta.yaml.j2 b/parm/archive/gdas_restarta.yaml.j2 index af1acbdbe7..4c0522fed7 100644 --- a/parm/archive/gdas_restarta.yaml.j2 +++ b/parm/archive/gdas_restarta.yaml.j2 @@ -4,10 +4,10 @@ gdas_restarta: target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gdas_restarta.tar" required: # Deterministic analysis increments - - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atminc.nc" + - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atminc.nc" # IAU increments {% for iaufhr in iaufhrs if iaufhr != 6 %} - - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmi{{ "%03d" % iaufhr }}.nc" + - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmi{{ "%03d" % iaufhr }}.nc" {% endfor %} # Surface analysis tiles @@ -19,36 +19,36 @@ gdas_restarta: {% set anl_timedelta = anl_offset | to_timedelta %} {% set anl_time = current_cycle | add_to_datetime(anl_timedelta) %} {% for itile in range(1,7) %} - - "{{ COM_ATMOS_RESTART | relpath(ROTDIR) }}/{{ anl_time | to_YMD }}.{{ anl_time | strftime("%H") }}0000.sfcanl_data.tile{{ itile }}.nc" + - "{{ COMIN_ATMOS_RESTART | relpath(ROTDIR) }}/{{ anl_time | to_YMD }}.{{ anl_time | strftime("%H") }}0000.sfcanl_data.tile{{ itile }}.nc" {% endfor %} # Initial biases {% if not DO_JEDIATMVAR %} - - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}abias" - - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}abias_air" - - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}abias_pc" - - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}radstat" - - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}cnvstat" - - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}abias_int" - - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}dtfanl.nc" - - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}loginc.txt" + - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}abias" + - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}abias_air" + - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}abias_pc" + - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}radstat" + - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}cnvstat" + - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}abias_int" + - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}dtfanl.nc" + - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}loginc.txt" {% endif %} # Snow surface data {% if DO_JEDISNOWDA %} {% for itile in range(1,7) %} # Snow analysis is 3dvar - - "{{ COM_SNOW_ANALYSIS | relpath(ROTDIR) }}/snowinc.{{ cycle_YMD }}.{{ cycle_HH }}0000.sfc_data.tile{{ itile }}.nc" - - "{{ COM_SNOW_ANALYSIS | relpath(ROTDIR) }}/{{ cycle_YMD }}.{{ cycle_HH }}0000.sfc_data.tile{{ itile }}.nc" + - "{{ COMIN_SNOW_ANALYSIS | relpath(ROTDIR) }}/snowinc.{{ cycle_YMD }}.{{ cycle_HH }}0000.sfc_data.tile{{ itile }}.nc" + - "{{ COMIN_SNOW_ANALYSIS | relpath(ROTDIR) }}/{{ cycle_YMD }}.{{ cycle_HH }}0000.sfc_data.tile{{ itile }}.nc" {% endfor %} {% endif %} # Snow configuration yaml {% if DO_JEDISNOWDA %} - - "{{ COM_CONF | relpath(ROTDIR) }}/{{ head }}letkfoi.yaml" + - "{{ COMIN_CONF | relpath(ROTDIR) }}/{{ head }}letkfoi.yaml" {% endif %} # Input BUFR files - - "{{ COM_OBS | relpath(ROTDIR) }}/{{ head }}nsstbufr" - - "{{ COM_OBS | relpath(ROTDIR) }}/{{ head }}prepbufr" - - "{{ COM_OBS | relpath(ROTDIR) }}/{{ head }}prepbufr.acft_profiles" + - "{{ COMIN_OBS | relpath(ROTDIR) }}/{{ head }}nsstbufr" + - "{{ COMIN_OBS | relpath(ROTDIR) }}/{{ head }}prepbufr" + - "{{ COMIN_OBS | relpath(ROTDIR) }}/{{ head }}prepbufr.acft_profiles" diff --git a/parm/archive/gdas_restartb.yaml.j2 b/parm/archive/gdas_restartb.yaml.j2 index c5cb29329f..0bbf517fb2 100644 --- a/parm/archive/gdas_restartb.yaml.j2 +++ b/parm/archive/gdas_restartb.yaml.j2 @@ -12,14 +12,14 @@ gdas_restartb: {% set offset_HH = offset_dt | strftime("%H") %} {% set prefix = offset_YMD + "." + offset_HH + "0000" %} {% for itile in range(1, 7) %} - - "{{ COM_ATMOS_RESTART | relpath(ROTDIR) }}/{{ prefix }}.sfcanl_data.tile{{ itile }}.nc" + - "{{ COMIN_ATMOS_RESTART | relpath(ROTDIR) }}/{{ prefix }}.sfcanl_data.tile{{ itile }}.nc" {% endfor %} {% endif %} # Regardless, always grab the center surface analysis data. {% set prefix = cycle_YMD + "." + cycle_HH + "0000" %} {% for itile in range(1, 7) %} - - "{{ COM_ATMOS_RESTART | relpath(ROTDIR) }}/{{ prefix }}.sfcanl_data.tile{{ itile }}.nc" + - "{{ COMIN_ATMOS_RESTART | relpath(ROTDIR) }}/{{ prefix }}.sfcanl_data.tile{{ itile }}.nc" {% endfor %} # Now get the restart files. @@ -31,9 +31,9 @@ gdas_restartb: {% set r_prefix = r_YMD + "." + r_HH + "0000" %} {% for itile in range(1, 7) %} {% for datatype in ["ca_data", "fv_core.res", "fv_srf_wnd.res", "fv_tracer.res", "phy_data", "sfc_data"] %} - - "{{ COM_ATMOS_RESTART | relpath(ROTDIR) }}/{{ r_prefix }}.{{datatype}}.tile{{ itile }}.nc" + - "{{ COMIN_ATMOS_RESTART | relpath(ROTDIR) }}/{{ r_prefix }}.{{datatype}}.tile{{ itile }}.nc" {% endfor %} {% endfor %} - - "{{ COM_ATMOS_RESTART | relpath(ROTDIR) }}/{{ r_prefix }}.coupler.res" - - "{{ COM_ATMOS_RESTART | relpath(ROTDIR) }}/{{ r_prefix }}.fv_core.res.nc" + - "{{ COMIN_ATMOS_RESTART | relpath(ROTDIR) }}/{{ r_prefix }}.coupler.res" + - "{{ COMIN_ATMOS_RESTART | relpath(ROTDIR) }}/{{ r_prefix }}.fv_core.res.nc" {% endfor %} diff --git a/parm/archive/gdasice.yaml.j2 b/parm/archive/gdasice.yaml.j2 index 23337a8856..da02decf83 100644 --- a/parm/archive/gdasice.yaml.j2 +++ b/parm/archive/gdasice.yaml.j2 @@ -3,8 +3,8 @@ gdasice: name: "GDASICE" target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gdasice.tar" required: - - "{{ COM_ICE_HISTORY | relpath(ROTDIR) }}/{{ head }}ic.nc" + - "{{ COMIN_ICE_HISTORY | relpath(ROTDIR) }}/{{ head }}ic.nc" {% for fhr in range(FHOUT_ICE, FHMAX+1, FHOUT_ICE) %} - - "{{ COM_ICE_HISTORY | relpath(ROTDIR) }}/{{ head }}inst.f{{ '%03d' % fhr }}.nc" + - "{{ COMIN_ICE_HISTORY | relpath(ROTDIR) }}/{{ head }}inst.f{{ '%03d' % fhr }}.nc" {% endfor %} - - '{{ COM_CONF | relpath(ROTDIR) }}/ufs.ice_in' + - '{{ COMIN_CONF | relpath(ROTDIR) }}/ufs.ice_in' diff --git a/parm/archive/gdasice_restart.yaml.j2 b/parm/archive/gdasice_restart.yaml.j2 index 39877674fb..15c8ba627d 100644 --- a/parm/archive/gdasice_restart.yaml.j2 +++ b/parm/archive/gdasice_restart.yaml.j2 @@ -4,4 +4,4 @@ gdasice_restart: target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gdasice_restart.tar" required: # TODO explicitly name the restart files to archive - - '{{ COM_ICE_RESTART | relpath(ROTDIR) }}/*' + - '{{ COMIN_ICE_RESTART | relpath(ROTDIR) }}/*' diff --git a/parm/archive/gdasocean.yaml.j2 b/parm/archive/gdasocean.yaml.j2 index 4e92bba1a0..9e6ca38851 100644 --- a/parm/archive/gdasocean.yaml.j2 +++ b/parm/archive/gdasocean.yaml.j2 @@ -4,6 +4,6 @@ gdasocean: target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gdasocean.tar" required: {% for fhr in range(FHMIN, FHMAX + 1, FHOUT_OCN) %} - - "{{ COM_OCEAN_HISTORY | relpath(ROTDIR) }}/{{ head }}inst.f{{ '%03d' % fhr }}.nc" + - "{{ COMIN_OCEAN_HISTORY | relpath(ROTDIR) }}/{{ head }}inst.f{{ '%03d' % fhr }}.nc" {% endfor %} - - '{{ COM_CONF | relpath(ROTDIR) }}/ufs.MOM_input' + - '{{ COMIN_CONF | relpath(ROTDIR) }}/ufs.MOM_input' diff --git a/parm/archive/gdasocean_analysis.yaml.j2 b/parm/archive/gdasocean_analysis.yaml.j2 index 2417ddf6e7..d127ee0b75 100644 --- a/parm/archive/gdasocean_analysis.yaml.j2 +++ b/parm/archive/gdasocean_analysis.yaml.j2 @@ -3,23 +3,23 @@ gdasocean_analysis: name: "GDASOCEAN_ANALYSIS" target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gdasocean_analysis.tar" required: - - '{{ COM_OCEAN_ANALYSIS | relpath(ROTDIR) }}/{{ head }}ocninc.nc' + - '{{ COMIN_OCEAN_ANALYSIS | relpath(ROTDIR) }}/{{ head }}ocninc.nc' {% set ocngrid_cycle = '%02d' % (((cycle_HH | int) - 3) % 24) %} - - '{{ COM_OCEAN_ANALYSIS | relpath(ROTDIR) }}/gdas.t{{ ocngrid_cycle }}z.ocngrid.nc' + - '{{ COMIN_OCEAN_ANALYSIS | relpath(ROTDIR) }}/gdas.t{{ ocngrid_cycle }}z.ocngrid.nc' {% for domain in ["ocn", "ice"] %} - - '{{ COM_OCEAN_ANALYSIS | relpath(ROTDIR) }}/{{ head }}{{domain}}.bkgerr_stddev.nc' - - '{{ COM_OCEAN_ANALYSIS | relpath(ROTDIR) }}/{{ head }}{{domain}}.incr.nc' - - '{{ COM_OCEAN_ANALYSIS | relpath(ROTDIR) }}/{{ head }}{{domain}}ana.nc' + - '{{ COMIN_OCEAN_ANALYSIS | relpath(ROTDIR) }}/{{ head }}{{domain}}.bkgerr_stddev.nc' + - '{{ COMIN_OCEAN_ANALYSIS | relpath(ROTDIR) }}/{{ head }}{{domain}}.incr.nc' + - '{{ COMIN_OCEAN_ANALYSIS | relpath(ROTDIR) }}/{{ head }}{{domain}}ana.nc' {% if NMEM_ENS > 2 %} - - '{{ COM_OCEAN_ANALYSIS | relpath(ROTDIR) }}/{{ head }}{{domain}}.recentering_error.nc' + - '{{ COMIN_OCEAN_ANALYSIS | relpath(ROTDIR) }}/{{ head }}{{domain}}.recentering_error.nc' {% endif %} {% endfor %} {% if NMEM_ENS > 2 %} - - '{{ COM_OCEAN_ANALYSIS | relpath(ROTDIR) }}/{{ head }}ocn.ssh_steric_stddev.nc' - - '{{ COM_OCEAN_ANALYSIS | relpath(ROTDIR) }}/{{ head }}ocn.ssh_unbal_stddev.nc' - - '{{ COM_OCEAN_ANALYSIS | relpath(ROTDIR) }}/{{ head }}ocn.ssh_total_stddev.nc' - - '{{ COM_OCEAN_ANALYSIS | relpath(ROTDIR) }}/{{ head }}ocn.steric_explained_variance.nc' + - '{{ COMIN_OCEAN_ANALYSIS | relpath(ROTDIR) }}/{{ head }}ocn.ssh_steric_stddev.nc' + - '{{ COMIN_OCEAN_ANALYSIS | relpath(ROTDIR) }}/{{ head }}ocn.ssh_unbal_stddev.nc' + - '{{ COMIN_OCEAN_ANALYSIS | relpath(ROTDIR) }}/{{ head }}ocn.ssh_total_stddev.nc' + - '{{ COMIN_OCEAN_ANALYSIS | relpath(ROTDIR) }}/{{ head }}ocn.steric_explained_variance.nc' {% endif %} - - '{{ COM_OCEAN_ANALYSIS | relpath(ROTDIR) }}/{{ head }}ocn.*.stats.csv' - - '{{ COM_OCEAN_ANALYSIS | relpath(ROTDIR) }}/diags/*.nc4' - - '{{ COM_OCEAN_ANALYSIS | relpath(ROTDIR) }}/yaml/*.yaml' + - '{{ COMIN_OCEAN_ANALYSIS | relpath(ROTDIR) }}/{{ head }}ocn.*.stats.csv' + - '{{ COMIN_OCEAN_ANALYSIS | relpath(ROTDIR) }}/diags/*.nc4' + - '{{ COMIN_OCEAN_ANALYSIS | relpath(ROTDIR) }}/yaml/*.yaml' diff --git a/parm/archive/gdasocean_restart.yaml.j2 b/parm/archive/gdasocean_restart.yaml.j2 index 21bfc3955f..f2b6bfb875 100644 --- a/parm/archive/gdasocean_restart.yaml.j2 +++ b/parm/archive/gdasocean_restart.yaml.j2 @@ -4,5 +4,5 @@ gdasocean_restart: target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gdasocean_restart.tar" required: # TODO explicitly name the restart files to archive - - '{{ COM_OCEAN_RESTART | relpath(ROTDIR) }}/*' - - '{{ COM_MED_RESTART | relpath(ROTDIR) }}/*' + - '{{ COMIN_OCEAN_RESTART | relpath(ROTDIR) }}/*' + - '{{ COMIN_MED_RESTART | relpath(ROTDIR) }}/*' diff --git a/parm/archive/gdaswave.yaml.j2 b/parm/archive/gdaswave.yaml.j2 index 74a5a64dbf..220770b38d 100644 --- a/parm/archive/gdaswave.yaml.j2 +++ b/parm/archive/gdaswave.yaml.j2 @@ -4,5 +4,5 @@ gdaswave: target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gdaswave.tar" required: # TODO explicitly name the wave grid/station files to archive - - "{{ COM_WAVE_GRID | relpath(ROTDIR) }}/{{ head }}*" - - "{{ COM_WAVE_STATION | relpath(ROTDIR) }}/{{ head }}*" + - "{{ COMIN_WAVE_GRID | relpath(ROTDIR) }}/{{ head }}*" + - "{{ COMIN_WAVE_STATION | relpath(ROTDIR) }}/{{ head }}*" diff --git a/parm/archive/gdaswave_restart.yaml.j2 b/parm/archive/gdaswave_restart.yaml.j2 index 8387d48616..7ada504e2a 100644 --- a/parm/archive/gdaswave_restart.yaml.j2 +++ b/parm/archive/gdaswave_restart.yaml.j2 @@ -3,4 +3,4 @@ gdaswave_restart: target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gdaswave_restart.tar" required: # TODO explicitly name the wave restart files to archive - - "{{ COM_WAVE_RESTART | relpath(ROTDIR) }}/*" + - "{{ COMIN_WAVE_RESTART | relpath(ROTDIR) }}/*" diff --git a/parm/archive/gfs_downstream.yaml.j2 b/parm/archive/gfs_downstream.yaml.j2 index 23c9383d28..ed5317b42c 100644 --- a/parm/archive/gfs_downstream.yaml.j2 +++ b/parm/archive/gfs_downstream.yaml.j2 @@ -3,10 +3,10 @@ gfs_downstream: name: "GFS_DOWNSTREAM" target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gfs_downstream.tar" required: - - "{{ COM_ATMOS_GEMPAK | relpath(ROTDIR) }}/gfs_{{ cycle_YMDH }}.sfc" - - "{{ COM_ATMOS_GEMPAK | relpath(ROTDIR) }}/gfs_{{ cycle_YMDH }}.snd" + - "{{ COMIN_ATMOS_GEMPAK | relpath(ROTDIR) }}/gfs_{{ cycle_YMDH }}.sfc" + - "{{ COMIN_ATMOS_GEMPAK | relpath(ROTDIR) }}/gfs_{{ cycle_YMDH }}.snd" {% for i in range(1, NUM_SND_COLLECTIVES) %} - - "{{ COM_ATMOS_WMO | relpath(ROTDIR) }}/gfs_collective{{ i }}.postsnd_{{ cycle_HH }}" + - "{{ COMIN_ATMOS_WMO | relpath(ROTDIR) }}/gfs_collective{{ i }}.postsnd_{{ cycle_HH }}" {% endfor %} - - "{{ COM_ATMOS_BUFR | relpath(ROTDIR) }}/bufr.t{{ cycle_HH }}z" - - "{{ COM_ATMOS_BUFR | relpath(ROTDIR) }}/gfs.t{{ cycle_HH }}z.bufrsnd.tar.gz" + - "{{ COMIN_ATMOS_BUFR | relpath(ROTDIR) }}/bufr.t{{ cycle_HH }}z" + - "{{ COMIN_ATMOS_BUFR | relpath(ROTDIR) }}/gfs.t{{ cycle_HH }}z.bufrsnd.tar.gz" diff --git a/parm/archive/gfs_flux.yaml.j2 b/parm/archive/gfs_flux.yaml.j2 index 66c8221f60..46bd0624b6 100644 --- a/parm/archive/gfs_flux.yaml.j2 +++ b/parm/archive/gfs_flux.yaml.j2 @@ -4,6 +4,6 @@ gfs_flux: target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gfs_flux.tar" required: {% for fhr in range(FHMIN_GFS, FHMAX_GFS + FHOUT_GFS, FHOUT_GFS) %} - - "{{ COM_ATMOS_MASTER | relpath(ROTDIR) }}/{{ head }}sfluxgrbf{{ '%03d' % fhr }}.grib2" - - "{{ COM_ATMOS_MASTER | relpath(ROTDIR) }}/{{ head }}sfluxgrbf{{ '%03d' % fhr }}.grib2.idx" + - "{{ COMIN_ATMOS_MASTER | relpath(ROTDIR) }}/{{ head }}sfluxgrbf{{ '%03d' % fhr }}.grib2" + - "{{ COMIN_ATMOS_MASTER | relpath(ROTDIR) }}/{{ head }}sfluxgrbf{{ '%03d' % fhr }}.grib2.idx" {% endfor %} diff --git a/parm/archive/gfs_flux_1p00.yaml.j2 b/parm/archive/gfs_flux_1p00.yaml.j2 index 2f5c9c8910..97fcd6e4d2 100644 --- a/parm/archive/gfs_flux_1p00.yaml.j2 +++ b/parm/archive/gfs_flux_1p00.yaml.j2 @@ -4,6 +4,6 @@ gfs_flux_1p00: target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gfs_flux_1p00.tar" required: {% for fhr in range(FHMIN_GFS, FHMAX_GFS + FHOUT_GFS, FHOUT_GFS) %} - - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}flux.1p00.f{{ '%03d' % fhr }}" - - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}flux.1p00.f{{ '%03d' % fhr }}.idx" + - "{{ COMIN_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}flux.1p00.f{{ '%03d' % fhr }}" + - "{{ COMIN_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}flux.1p00.f{{ '%03d' % fhr }}.idx" {% endfor %} diff --git a/parm/archive/gfs_netcdfa.yaml.j2 b/parm/archive/gfs_netcdfa.yaml.j2 index 6bcafe1b89..8c0d4a813f 100644 --- a/parm/archive/gfs_netcdfa.yaml.j2 +++ b/parm/archive/gfs_netcdfa.yaml.j2 @@ -3,14 +3,14 @@ gfs_netcdfa: name: "GFS_NETCDFA" target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gfs_netcdfa.tar" required: - - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmanl.nc" - - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}sfcanl.nc" - - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atminc.nc" + - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmanl.nc" + - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}sfcanl.nc" + - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atminc.nc" {% for iauhr in iaufhrs if iauhr != 6 %} - - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmi{{ "%03d" % iauhr }}.nc" + - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmi{{ "%03d" % iauhr }}.nc" {% endfor %} optional: {% if not DO_JEDIATMVAR %} - - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}dtfanl.nc" - - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}loginc.txt" + - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}dtfanl.nc" + - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}loginc.txt" {% endif %} diff --git a/parm/archive/gfs_netcdfb.yaml.j2 b/parm/archive/gfs_netcdfb.yaml.j2 index b0393d63b6..727f054715 100644 --- a/parm/archive/gfs_netcdfb.yaml.j2 +++ b/parm/archive/gfs_netcdfb.yaml.j2 @@ -4,6 +4,6 @@ gfs_netcdfb: target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gfs_netcdfb.tar" required: {% for fhr in range(0, ARCH_GAUSSIAN_FHMAX + ARCH_GAUSSIAN_FHINC, ARCH_GAUSSIAN_FHINC) %} - - "{{ COM_ATMOS_HISTORY | relpath(ROTDIR) }}/{{ head }}atmf{{ '%03d' % fhr }}.nc" - - "{{ COM_ATMOS_HISTORY | relpath(ROTDIR) }}/{{ head }}sfcf{{ '%03d' % fhr }}.nc" + - "{{ COMIN_ATMOS_HISTORY | relpath(ROTDIR) }}/{{ head }}atmf{{ '%03d' % fhr }}.nc" + - "{{ COMIN_ATMOS_HISTORY | relpath(ROTDIR) }}/{{ head }}sfcf{{ '%03d' % fhr }}.nc" {% endfor %} diff --git a/parm/archive/gfs_pgrb2b.yaml.j2 b/parm/archive/gfs_pgrb2b.yaml.j2 index b06dd14b73..ca20d1a3d8 100644 --- a/parm/archive/gfs_pgrb2b.yaml.j2 +++ b/parm/archive/gfs_pgrb2b.yaml.j2 @@ -4,16 +4,16 @@ gfs_pgrb2b: target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gfs_pgrb2b.tar" required: {% if MODE == "cycled" %} - - "{{ COM_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2b.0p25.anl" - - "{{ COM_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2b.0p25.anl.idx" - - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2b.1p00.anl" - - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2b.1p00.anl.idx" + - "{{ COMIN_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2b.0p25.anl" + - "{{ COMIN_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2b.0p25.anl.idx" + - "{{ COMIN_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2b.1p00.anl" + - "{{ COMIN_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2b.1p00.anl.idx" {% endif %} {% if ARCH_GAUSSIAN %} {% for fhr in range(0, FHMAX_GFS + FHOUT_GFS, FHOUT_GFS) %} - - "{{ COM_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2b.0p25.f{{ '%03d' % fhr }}" - - "{{ COM_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2b.0p25.f{{ '%03d' % fhr }}.idx" - - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2b.1p00.f{{ '%03d' % fhr }}" - - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2b.1p00.f{{ '%03d' % fhr }}.idx" + - "{{ COMIN_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2b.0p25.f{{ '%03d' % fhr }}" + - "{{ COMIN_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2b.0p25.f{{ '%03d' % fhr }}.idx" + - "{{ COMIN_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2b.1p00.f{{ '%03d' % fhr }}" + - "{{ COMIN_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2b.1p00.f{{ '%03d' % fhr }}.idx" {% endfor %} {% endif %} diff --git a/parm/archive/gfs_restarta.yaml.j2 b/parm/archive/gfs_restarta.yaml.j2 index c2ad717484..8f6a0b6c10 100644 --- a/parm/archive/gfs_restarta.yaml.j2 +++ b/parm/archive/gfs_restarta.yaml.j2 @@ -12,12 +12,12 @@ gfs_restarta: {% set anl_timedelta = anl_offset | to_timedelta %} {% set anl_time = current_cycle | add_to_datetime(anl_timedelta) %} {% for i_tile in range(1, 7) %} - - "{{ COM_ATMOS_RESTART | relpath(ROTDIR) }}/{{ anl_time | to_YMD }}.{{ anl_time | strftime("%H") }}0000.sfcanl_data.tile{{ i_tile }}.nc" + - "{{ COMIN_ATMOS_RESTART | relpath(ROTDIR) }}/{{ anl_time | to_YMD }}.{{ anl_time | strftime("%H") }}0000.sfcanl_data.tile{{ i_tile }}.nc" {% endfor %} {% elif MODE == "forecast-only" %} - - "{{ COM_ATMOS_INPUT | relpath(ROTDIR) }}/gfs_ctrl.nc" + - "{{ COMIN_ATMOS_INPUT | relpath(ROTDIR) }}/gfs_ctrl.nc" {% for i_tile in range(1, 7) %} - - "{{ COM_ATMOS_INPUT | relpath(ROTDIR) }}/gfs_data.tile{{ i_tile }}.nc" - - "{{ COM_ATMOS_INPUT | relpath(ROTDIR) }}/sfc_data.tile{{ i_tile }}.nc" + - "{{ COMIN_ATMOS_INPUT | relpath(ROTDIR) }}/gfs_data.tile{{ i_tile }}.nc" + - "{{ COMIN_ATMOS_INPUT | relpath(ROTDIR) }}/sfc_data.tile{{ i_tile }}.nc" {% endfor %} {% endif %} diff --git a/parm/archive/gfsa.yaml.j2 b/parm/archive/gfsa.yaml.j2 index 0a8e65d3ef..e76c26e60e 100644 --- a/parm/archive/gfsa.yaml.j2 +++ b/parm/archive/gfsa.yaml.j2 @@ -12,57 +12,57 @@ gfsa: {% endfor %} # UFS configuration - - "{{ COM_CONF | relpath(ROTDIR) }}/ufs.input.nml" + - "{{ COMIN_CONF | relpath(ROTDIR) }}/ufs.input.nml" {% if MODE == "cycled" %} # Analysis GRIB2 (gridded) data - - "{{ COM_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p25.anl" - - "{{ COM_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p25.anl.idx" - - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2.1p00.anl" - - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2.1p00.anl.idx" + - "{{ COMIN_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p25.anl" + - "{{ COMIN_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p25.anl.idx" + - "{{ COMIN_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2.1p00.anl" + - "{{ COMIN_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2.1p00.anl.idx" {% if DO_VMINMON %} # Minimization monitor - - "{{ COM_ATMOS_MINMON | relpath(ROTDIR) }}/{{ cycle_YMDH }}.costs.txt" - - "{{ COM_ATMOS_MINMON | relpath(ROTDIR) }}/{{ cycle_YMDH }}.cost_terms.txt" - - "{{ COM_ATMOS_MINMON | relpath(ROTDIR) }}/{{ cycle_YMDH }}.gnorms.ieee_d" - - "{{ COM_ATMOS_MINMON | relpath(ROTDIR) }}/{{ cycle_YMDH }}.reduction.ieee_d" - - "{{ COM_ATMOS_MINMON | relpath(ROTDIR) }}/gnorm_data.txt" + - "{{ COMIN_ATMOS_MINMON | relpath(ROTDIR) }}/{{ cycle_YMDH }}.costs.txt" + - "{{ COMIN_ATMOS_MINMON | relpath(ROTDIR) }}/{{ cycle_YMDH }}.cost_terms.txt" + - "{{ COMIN_ATMOS_MINMON | relpath(ROTDIR) }}/{{ cycle_YMDH }}.gnorms.ieee_d" + - "{{ COMIN_ATMOS_MINMON | relpath(ROTDIR) }}/{{ cycle_YMDH }}.reduction.ieee_d" + - "{{ COMIN_ATMOS_MINMON | relpath(ROTDIR) }}/gnorm_data.txt" {% endif %} # State data {% if DO_JEDIATMVAR %} - - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmvar.yaml" - - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmstat" + - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmvar.yaml" + - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmstat" {% else %} - - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}gsistat" + - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}gsistat" {% endif %} {% if AERO_ANL_CDUMP == "gfs" or AERO_ANL_CDUMP == "both" %} - - "{{ COM_CHEM_ANALYSIS | relpath(ROTDIR) }}/{{ head }}aerostat" + - "{{ COMIN_CHEM_ANALYSIS | relpath(ROTDIR) }}/{{ head }}aerostat" {% endif %} {% if DO_PREP_OBS_AERO %} - - "{{ COM_OBS | relpath(ROTDIR) }}/{{ head }}aeroobs" - - "{{ COM_OBS | relpath(ROTDIR) }}/{{ head }}aerorawobs" + - "{{ COMIN_OBS | relpath(ROTDIR) }}/{{ head }}aeroobs" + - "{{ COMIN_OBS | relpath(ROTDIR) }}/{{ head }}aerorawobs" {% endif %} # BUFR inputs - - "{{ COM_OBS | relpath(ROTDIR) }}/{{ head }}nsstbufr" - - "{{ COM_OBS | relpath(ROTDIR) }}/{{ head }}prepbufr" - - "{{ COM_OBS | relpath(ROTDIR) }}/{{ head }}prepbufr.acft_profiles" + - "{{ COMIN_OBS | relpath(ROTDIR) }}/{{ head }}nsstbufr" + - "{{ COMIN_OBS | relpath(ROTDIR) }}/{{ head }}prepbufr" + - "{{ COMIN_OBS | relpath(ROTDIR) }}/{{ head }}prepbufr.acft_profiles" {% endif %} # Full cycle # Forecast GRIB2 products {% for fhr in range(FHMIN_GFS, FHMAX_GFS + FHOUT_GFS, FHOUT_GFS) %} - - "{{ COM_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p25.f{{ '%03d' % fhr }}" - - "{{ COM_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p25.f{{ '%03d' % fhr }}.idx" - - "{{ COM_ATMOS_HISTORY | relpath(ROTDIR) }}/{{ head }}atm.logf{{ '%03d' % fhr }}.txt" + - "{{ COMIN_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p25.f{{ '%03d' % fhr }}" + - "{{ COMIN_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p25.f{{ '%03d' % fhr }}.idx" + - "{{ COMIN_ATMOS_HISTORY | relpath(ROTDIR) }}/{{ head }}atm.logf{{ '%03d' % fhr }}.txt" {% endfor %} optional: # Cyclone tracking data; only present if there's something to track. - - "{{ COM_ATMOS_TRACK | relpath(ROTDIR) }}/avno.t{{ cycle_HH }}z.cyclone.trackatcfunix" - - "{{ COM_ATMOS_TRACK | relpath(ROTDIR) }}/avnop.t{{ cycle_HH }}z.cyclone.trackatcfunix" - - "{{ COM_ATMOS_GENESIS | relpath(ROTDIR) }}/trak.gfso.atcfunix.{{ cycle_YMDH }}" - - "{{ COM_ATMOS_GENESIS | relpath(ROTDIR) }}/trak.gfso.atcfunix.altg.{{ cycle_YMDH }}" - - "{{ COM_ATMOS_GENESIS | relpath(ROTDIR) }}/storms.gfso.atcf_gen.{{ cycle_YMDH }}" - - "{{ COM_ATMOS_GENESIS | relpath(ROTDIR) }}/storms.gfso.atcf_gen.altg.{{ cycle_YMDH }}" + - "{{ COMIN_ATMOS_TRACK | relpath(ROTDIR) }}/avno.t{{ cycle_HH }}z.cyclone.trackatcfunix" + - "{{ COMIN_ATMOS_TRACK | relpath(ROTDIR) }}/avnop.t{{ cycle_HH }}z.cyclone.trackatcfunix" + - "{{ COMIN_ATMOS_GENESIS | relpath(ROTDIR) }}/trak.gfso.atcfunix.{{ cycle_YMDH }}" + - "{{ COMIN_ATMOS_GENESIS | relpath(ROTDIR) }}/trak.gfso.atcfunix.altg.{{ cycle_YMDH }}" + - "{{ COMIN_ATMOS_GENESIS | relpath(ROTDIR) }}/storms.gfso.atcf_gen.{{ cycle_YMDH }}" + - "{{ COMIN_ATMOS_GENESIS | relpath(ROTDIR) }}/storms.gfso.atcf_gen.altg.{{ cycle_YMDH }}" diff --git a/parm/archive/gfsb.yaml.j2 b/parm/archive/gfsb.yaml.j2 index cbb4d4fad8..e6ffa05766 100644 --- a/parm/archive/gfsb.yaml.j2 +++ b/parm/archive/gfsb.yaml.j2 @@ -5,16 +5,16 @@ gfsb: required: {% if MODE == "cycled" %} # GRIB2 (subsampled) analysis data - - "{{ COM_ATMOS_GRIB_0p50 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p50.anl" - - "{{ COM_ATMOS_GRIB_0p50 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p50.anl.idx" - - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2.1p00.anl" - - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2.1p00.anl.idx" + - "{{ COMIN_ATMOS_GRIB_0p50 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p50.anl" + - "{{ COMIN_ATMOS_GRIB_0p50 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p50.anl.idx" + - "{{ COMIN_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2.1p00.anl" + - "{{ COMIN_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2.1p00.anl.idx" {% endif %} # GRIB2 orecast data {% for fhr in range(FHMIN_GFS, FHMAX_GFS + FHOUT_GFS, FHOUT_GFS) %} - - "{{ COM_ATMOS_GRIB_0p50 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p50.f{{ '%03d' % fhr }}" - - "{{ COM_ATMOS_GRIB_0p50 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p50.f{{ '%03d' % fhr }}.idx" - - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2.1p00.f{{ '%03d' % fhr }}" - - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2.1p00.f{{ '%03d' % fhr }}.idx" + - "{{ COMIN_ATMOS_GRIB_0p50 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p50.f{{ '%03d' % fhr }}" + - "{{ COMIN_ATMOS_GRIB_0p50 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p50.f{{ '%03d' % fhr }}.idx" + - "{{ COMIN_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2.1p00.f{{ '%03d' % fhr }}" + - "{{ COMIN_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2.1p00.f{{ '%03d' % fhr }}.idx" {% endfor %} diff --git a/parm/archive/gfswave.yaml.j2 b/parm/archive/gfswave.yaml.j2 index 8422377b7a..6909421757 100644 --- a/parm/archive/gfswave.yaml.j2 +++ b/parm/archive/gfswave.yaml.j2 @@ -7,24 +7,24 @@ gfswave: {% for fh in range(0, FHMAX_HF_WAV + FHOUT_HF_WAV, FHOUT_HF_WAV) %} # NOTE This is as explicit as possible without major logic to parse wavepostGRD. # Matches files of the form "gfswave.tCCz...fHHH.grib2". - - "{{ COM_WAVE_GRID | relpath(ROTDIR) }}/{{ head }}*.*.f{{ '%03d' % fh }}.grib2" - - "{{ COM_WAVE_GRID | relpath(ROTDIR) }}/{{ head }}*.*.f{{ '%03d' % fh }}.grib2.idx" + - "{{ COMIN_WAVE_GRID | relpath(ROTDIR) }}/{{ head }}*.*.f{{ '%03d' % fh }}.grib2" + - "{{ COMIN_WAVE_GRID | relpath(ROTDIR) }}/{{ head }}*.*.f{{ '%03d' % fh }}.grib2.idx" {% endfor %} # Global wave GRIB2 forecast products {% for fh in range(FHMAX_HF_WAV + FHOUT_WAV, FHMAX_WAV_GFS + FHOUT_WAV, FHOUT_WAV) %} - - "{{ COM_WAVE_GRID | relpath(ROTDIR) }}/{{ head }}*.*.f{{ '%03d' % fh }}.grib2" - - "{{ COM_WAVE_GRID | relpath(ROTDIR) }}/{{ head }}*.*.f{{ '%03d' % fh }}.grib2.idx" + - "{{ COMIN_WAVE_GRID | relpath(ROTDIR) }}/{{ head }}*.*.f{{ '%03d' % fh }}.grib2" + - "{{ COMIN_WAVE_GRID | relpath(ROTDIR) }}/{{ head }}*.*.f{{ '%03d' % fh }}.grib2.idx" {% endfor %} # Wave bulletins - - "{{ COM_WAVE_STATION | relpath(ROTDIR) }}/{{ head }}bull_tar" - - "{{ COM_WAVE_STATION | relpath(ROTDIR) }}/{{ head }}cbull_tar" - - "{{ COM_WAVE_STATION | relpath(ROTDIR) }}/{{ head }}spec_tar.gz" + - "{{ COMIN_WAVE_STATION | relpath(ROTDIR) }}/{{ head }}bull_tar" + - "{{ COMIN_WAVE_STATION | relpath(ROTDIR) }}/{{ head }}cbull_tar" + - "{{ COMIN_WAVE_STATION | relpath(ROTDIR) }}/{{ head }}spec_tar.gz" # Wave IBP bulletins {% if DOIBP_WAV %} - - "{{ COM_WAVE_STATION | relpath(ROTDIR) }}/{{ head }}ibpbull_tar" - - "{{ COM_WAVE_STATION | relpath(ROTDIR) }}/{{ head }}ibpcbull_tar" - - "{{ COM_WAVE_STATION | relpath(ROTDIR) }}/{{ head }}ibp_tar" + - "{{ COMIN_WAVE_STATION | relpath(ROTDIR) }}/{{ head }}ibpbull_tar" + - "{{ COMIN_WAVE_STATION | relpath(ROTDIR) }}/{{ head }}ibpcbull_tar" + - "{{ COMIN_WAVE_STATION | relpath(ROTDIR) }}/{{ head }}ibp_tar" {% endif %} diff --git a/parm/archive/ice_6hravg.yaml.j2 b/parm/archive/ice_6hravg.yaml.j2 index 251e51b110..6eb64ae70d 100644 --- a/parm/archive/ice_6hravg.yaml.j2 +++ b/parm/archive/ice_6hravg.yaml.j2 @@ -3,7 +3,7 @@ ice_6hravg: name: "ICE_6HRAVG" target: "{{ ATARDIR }}/{{ cycle_YMDH }}/ice_6hravg.tar" required: - - "{{ COM_ICE_HISTORY | relpath(ROTDIR) }}/{{ head }}ic.nc" + - "{{ COMIN_ICE_HISTORY | relpath(ROTDIR) }}/{{ head }}ic.nc" {% for fhr in range(6, FHMAX_GFS + 6, 6) %} - - "{{ COM_ICE_HISTORY | relpath(ROTDIR) }}/{{ head }}6hr_avg.f{{ '%03d' % fhr }}.nc" + - "{{ COMIN_ICE_HISTORY | relpath(ROTDIR) }}/{{ head }}6hr_avg.f{{ '%03d' % fhr }}.nc" {% endfor %} diff --git a/parm/archive/ice_grib2.yaml.j2 b/parm/archive/ice_grib2.yaml.j2 index 9d52f174ab..04bc2f5873 100644 --- a/parm/archive/ice_grib2.yaml.j2 +++ b/parm/archive/ice_grib2.yaml.j2 @@ -7,13 +7,13 @@ ice_grib2: {% for fhr in range(FHOUT_ICE_GFS, FHMAX_GFS + FHOUT_ICE_GFS, FHOUT_ICE_GFS) %} {% set fhr3 = '%03d' % fhr %} {% if ICERES == 500 %} - - "{{ COM_ICE_GRIB | relpath(ROTDIR) }}/5p00/{{ head }}5p00.f{{ fhr3 }}.grib2" - - "{{ COM_ICE_GRIB | relpath(ROTDIR) }}/5p00/{{ head }}5p00.f{{ fhr3 }}.grib2.idx" + - "{{ COMIN_ICE_GRIB | relpath(ROTDIR) }}/5p00/{{ head }}5p00.f{{ fhr3 }}.grib2" + - "{{ COMIN_ICE_GRIB | relpath(ROTDIR) }}/5p00/{{ head }}5p00.f{{ fhr3 }}.grib2.idx" {% elif ICERES == 100 %} - - "{{ COM_ICE_GRIB | relpath(ROTDIR) }}/1p00/{{ head }}1p00.f{{ fhr3 }}.grib2" - - "{{ COM_ICE_GRIB | relpath(ROTDIR) }}/1p00/{{ head }}1p00.f{{ fhr3 }}.grib2.idx" + - "{{ COMIN_ICE_GRIB | relpath(ROTDIR) }}/1p00/{{ head }}1p00.f{{ fhr3 }}.grib2" + - "{{ COMIN_ICE_GRIB | relpath(ROTDIR) }}/1p00/{{ head }}1p00.f{{ fhr3 }}.grib2.idx" {% elif ICERES == 25 or ICERES == "025" %} - - "{{ COM_ICE_GRIB | relpath(ROTDIR) }}/0p25/{{ head }}0p25.f{{ fhr3 }}.grib2" - - "{{ COM_ICE_GRIB | relpath(ROTDIR) }}/0p25/{{ head }}0p25.f{{ fhr3 }}.grib2.idx" + - "{{ COMIN_ICE_GRIB | relpath(ROTDIR) }}/0p25/{{ head }}0p25.f{{ fhr3 }}.grib2" + - "{{ COMIN_ICE_GRIB | relpath(ROTDIR) }}/0p25/{{ head }}0p25.f{{ fhr3 }}.grib2.idx" {% endif %} {% endfor %} diff --git a/parm/archive/master_enkf.yaml.j2 b/parm/archive/master_enkf.yaml.j2 index fb6a3d30ac..2fd94597f8 100644 --- a/parm/archive/master_enkf.yaml.j2 +++ b/parm/archive/master_enkf.yaml.j2 @@ -37,12 +37,12 @@ datasets: # Archive individual member data # First, construct individual member directories from templates -# COM_ATMOS_ANALYSIS_MEM, COM_ATMOS_HISTORY_MEM, and COM_ATMOS_RESTART_MEM +# COMIN_ATMOS_ANALYSIS_MEM, COMIN_ATMOS_HISTORY_MEM, and COMIN_ATMOS_RESTART_MEM # Declare to-be-filled lists of member COM directories -{% set COM_ATMOS_ANALYSIS_MEM_list = [] %} -{% set COM_ATMOS_RESTART_MEM_list = [] %} -{% set COM_ATMOS_HISTORY_MEM_list = [] %} +{% set COMIN_ATMOS_ANALYSIS_MEM_list = [] %} +{% set COMIN_ATMOS_RESTART_MEM_list = [] %} +{% set COMIN_ATMOS_HISTORY_MEM_list = [] %} # Determine which ensemble members belong to this group {% set first_group_mem = (ENSGRP - 1) * NMEM_EARCGRP + 1 %} @@ -62,22 +62,22 @@ datasets: # This must be done in a namespace to overcome jinja scoping # Variables set inside of a for loop are lost at the end of the loop # unless they are part of a namespace -{% set com_ns = namespace(COM_ATMOS_ANALYSIS_MEM = COM_ATMOS_ANALYSIS_TMPL, - COM_ATMOS_HISTORY_MEM = COM_ATMOS_HISTORY_TMPL, - COM_ATMOS_RESTART_MEM = COM_ATMOS_RESTART_TMPL) %} +{% set com_ns = namespace(COMIN_ATMOS_ANALYSIS_MEM = COMIN_ATMOS_ANALYSIS_TMPL, + COMIN_ATMOS_HISTORY_MEM = COMIN_ATMOS_HISTORY_TMPL, + COMIN_ATMOS_RESTART_MEM = COMIN_ATMOS_RESTART_TMPL) %} {% for key in tmpl_dict.keys() %} {% set search_term = '${' + key + '}' %} {% set replace_term = tmpl_dict[key] %} -{% set com_ns.COM_ATMOS_ANALYSIS_MEM = com_ns.COM_ATMOS_ANALYSIS_MEM.replace(search_term, replace_term) %} -{% set com_ns.COM_ATMOS_HISTORY_MEM = com_ns.COM_ATMOS_HISTORY_MEM.replace(search_term, replace_term) %} -{% set com_ns.COM_ATMOS_RESTART_MEM = com_ns.COM_ATMOS_RESTART_MEM.replace(search_term, replace_term) %} +{% set com_ns.COMIN_ATMOS_ANALYSIS_MEM = com_ns.COMIN_ATMOS_ANALYSIS_MEM.replace(search_term, replace_term) %} +{% set com_ns.COMIN_ATMOS_HISTORY_MEM = com_ns.COMIN_ATMOS_HISTORY_MEM.replace(search_term, replace_term) %} +{% set com_ns.COMIN_ATMOS_RESTART_MEM = com_ns.COMIN_ATMOS_RESTART_MEM.replace(search_term, replace_term) %} {% endfor %} # Append the member COM directories -{% do COM_ATMOS_ANALYSIS_MEM_list.append(com_ns.COM_ATMOS_ANALYSIS_MEM)%} -{% do COM_ATMOS_HISTORY_MEM_list.append(com_ns.COM_ATMOS_HISTORY_MEM)%} -{% do COM_ATMOS_RESTART_MEM_list.append(com_ns.COM_ATMOS_RESTART_MEM)%} +{% do COMIN_ATMOS_ANALYSIS_MEM_list.append(com_ns.COMIN_ATMOS_ANALYSIS_MEM)%} +{% do COMIN_ATMOS_HISTORY_MEM_list.append(com_ns.COMIN_ATMOS_HISTORY_MEM)%} +{% do COMIN_ATMOS_RESTART_MEM_list.append(com_ns.COMIN_ATMOS_RESTART_MEM)%} {% endfor %} diff --git a/parm/archive/ocean_6hravg.yaml.j2 b/parm/archive/ocean_6hravg.yaml.j2 index dac3ce262a..58db08538f 100644 --- a/parm/archive/ocean_6hravg.yaml.j2 +++ b/parm/archive/ocean_6hravg.yaml.j2 @@ -4,5 +4,5 @@ ocean_6hravg: target: "{{ ATARDIR }}/{{ cycle_YMDH }}/ocean_6hravg.tar" required: {% for fhr in range(6, FHMAX_GFS + 6, 6) %} - - "{{ COM_OCEAN_HISTORY | relpath(ROTDIR) }}/{{ head }}6hr_avg.f{{ '%03d' % fhr }}.nc" + - "{{ COMIN_OCEAN_HISTORY | relpath(ROTDIR) }}/{{ head }}6hr_avg.f{{ '%03d' % fhr }}.nc" {% endfor %} diff --git a/parm/archive/ocean_grib2.yaml.j2 b/parm/archive/ocean_grib2.yaml.j2 index 784e30021d..e8f2e3170b 100644 --- a/parm/archive/ocean_grib2.yaml.j2 +++ b/parm/archive/ocean_grib2.yaml.j2 @@ -6,13 +6,13 @@ ocean_grib2: {% for fhr in range(FHOUT_OCN_GFS, FHMAX_GFS + FHOUT_OCN_GFS, FHOUT_OCN_GFS) %} {% set fhr3 = '%03d' % fhr %} {% if OCNRES == 500 %} - - "{{ COM_OCEAN_GRIB | relpath(ROTDIR) }}/5p00/{{ head }}5p00.f{{ fhr3 }}.grib2" - - "{{ COM_OCEAN_GRIB | relpath(ROTDIR) }}/5p00/{{ head }}5p00.f{{ fhr3 }}.grib2.idx" + - "{{ COMIN_OCEAN_GRIB | relpath(ROTDIR) }}/5p00/{{ head }}5p00.f{{ fhr3 }}.grib2" + - "{{ COMIN_OCEAN_GRIB | relpath(ROTDIR) }}/5p00/{{ head }}5p00.f{{ fhr3 }}.grib2.idx" {% elif OCNRES == 100 %} - - "{{ COM_OCEAN_GRIB | relpath(ROTDIR) }}/1p00/{{ head }}1p00.f{{ fhr3 }}.grib2" - - "{{ COM_OCEAN_GRIB | relpath(ROTDIR) }}/1p00/{{ head }}1p00.f{{ fhr3 }}.grib2.idx" + - "{{ COMIN_OCEAN_GRIB | relpath(ROTDIR) }}/1p00/{{ head }}1p00.f{{ fhr3 }}.grib2" + - "{{ COMIN_OCEAN_GRIB | relpath(ROTDIR) }}/1p00/{{ head }}1p00.f{{ fhr3 }}.grib2.idx" {% elif OCNRES == 25 or OCNRES == "025" %} - - "{{ COM_OCEAN_GRIB | relpath(ROTDIR) }}/0p25/{{ head }}0p25.f{{ fhr3 }}.grib2" - - "{{ COM_OCEAN_GRIB | relpath(ROTDIR) }}/0p25/{{ head }}0p25.f{{ fhr3 }}.grib2.idx" + - "{{ COMIN_OCEAN_GRIB | relpath(ROTDIR) }}/0p25/{{ head }}0p25.f{{ fhr3 }}.grib2" + - "{{ COMIN_OCEAN_GRIB | relpath(ROTDIR) }}/0p25/{{ head }}0p25.f{{ fhr3 }}.grib2.idx" {% endif %} {% endfor %} diff --git a/scripts/exgdas_enkf_earc.py b/scripts/exgdas_enkf_earc.py index 2febbc27f5..c0fc51eee5 100755 --- a/scripts/exgdas_enkf_earc.py +++ b/scripts/exgdas_enkf_earc.py @@ -34,9 +34,9 @@ def main(): for key in keys: archive_dict[key] = archive.task_config[key] - # Also import all COM* directory and template variables + # Also import all COMIN* directory and template variables for key in archive.task_config.keys(): - if key.startswith("COM"): + if key.startswith("COMIN"): archive_dict[key] = archive.task_config[key] cwd = os.getcwd() diff --git a/scripts/exglobal_archive.py b/scripts/exglobal_archive.py index bcd8d522d9..af396d382e 100755 --- a/scripts/exglobal_archive.py +++ b/scripts/exglobal_archive.py @@ -37,9 +37,9 @@ def main(): for key in keys: archive_dict[key] = archive.task_config[key] - # Also import all COM* directory and template variables + # Also import all COMIN* and COMOUT* directory and template variables for key in archive.task_config.keys(): - if key.startswith("COM"): + if key.startswith("COMIN_") or key.startswith("COMOUT_"): archive_dict[key] = archive.task_config[key] cwd = os.getcwd() diff --git a/ush/python/pygfs/task/archive.py b/ush/python/pygfs/task/archive.py index 66d94878e8..269aea0ccc 100644 --- a/ush/python/pygfs/task/archive.py +++ b/ush/python/pygfs/task/archive.py @@ -309,7 +309,7 @@ def _gen_relative_paths(self, root_path: str) -> Dict: rel_path_dict : Dict Dictionary of paths relative to root_path. Members will be named based on the dict names in self.config. For COM paths, the names will - follow COM_ --> _dir. For all other directories, the + follow COMIN_ --> _dir. For all other directories, the names will follow --> _dir. """ @@ -318,7 +318,7 @@ def _gen_relative_paths(self, root_path: str) -> Dict: if isinstance(value, str): if root_path in value: rel_path = value.replace(root_path, "") - rel_key = (key[4:] if key.startswith("COM_") else key).lower() + "_dir" + rel_key = (key[4:] if key.startswith("COMIN_") else key).lower() + "_dir" rel_path_dict[rel_key] = rel_path return rel_path_dict @@ -374,27 +374,28 @@ def _rename_cyclone_expt(arch_dict) -> None: if len(arch_dict.PSLOT) > 4: pslot4 = arch_dict.PSLOT[0:4].upper() - track_dir = arch_dict.COM_ATMOS_TRACK + track_dir_in = arch_dict.COMIN_ATMOS_TRACK + track_dir_out = arch_dict.COMOUT_ATMOS_TRACK run = arch_dict.RUN cycle_HH = strftime(arch_dict.current_cycle, "%H") if run == "gfs": - in_track_file = (track_dir + "/avno.t" + + in_track_file = (track_dir_in + "/avno.t" + cycle_HH + "z.cycle.trackatcfunix") - in_track_p_file = (track_dir + "/avnop.t" + + in_track_p_file = (track_dir_in + "/avnop.t" + cycle_HH + "z.cycle.trackatcfunixp") elif run == "gdas": - in_track_file = (track_dir + "/gdas.t" + + in_track_file = (track_dir_in + "/gdas.t" + cycle_HH + "z.cycle.trackatcfunix") - in_track_p_file = (track_dir + "/gdasp.t" + + in_track_p_file = (track_dir_in + "/gdasp.t" + cycle_HH + "z.cycle.trackatcfunixp") if not os.path.isfile(in_track_file): # Do not attempt to archive the outputs return - out_track_file = track_dir + "/atcfunix." + run + "." + to_YMDH(arch_dict.current_cycle) - out_track_p_file = track_dir + "/atcfunixp." + run + "." + to_YMDH(arch_dict.current_cycle) + out_track_file = track_dir_out + "/atcfunix." + run + "." + to_YMDH(arch_dict.current_cycle) + out_track_p_file = track_dir_out + "/atcfunixp." + run + "." + to_YMDH(arch_dict.current_cycle) def replace_string_from_to_file(filename_in, filename_out, search_str, replace_str): From 3270ac3bf00c3ebc8166c70d84647ec44431fbae Mon Sep 17 00:00:00 2001 From: "Henry R. Winterbottom" <49202169+HenryWinterbottom-NOAA@users.noreply.github.com> Date: Tue, 18 Jun 2024 12:17:59 -0600 Subject: [PATCH 7/9] Hotfix for bug in template names. (#2697) This PR is a hotfix for an incorrectly named (e.g., non-existent) `COM/` template. Resolves #2696 Refs #2451 --- parm/archive/master_enkf.yaml.j2 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/parm/archive/master_enkf.yaml.j2 b/parm/archive/master_enkf.yaml.j2 index 2fd94597f8..f663d02895 100644 --- a/parm/archive/master_enkf.yaml.j2 +++ b/parm/archive/master_enkf.yaml.j2 @@ -62,9 +62,9 @@ datasets: # This must be done in a namespace to overcome jinja scoping # Variables set inside of a for loop are lost at the end of the loop # unless they are part of a namespace -{% set com_ns = namespace(COMIN_ATMOS_ANALYSIS_MEM = COMIN_ATMOS_ANALYSIS_TMPL, - COMIN_ATMOS_HISTORY_MEM = COMIN_ATMOS_HISTORY_TMPL, - COMIN_ATMOS_RESTART_MEM = COMIN_ATMOS_RESTART_TMPL) %} +{% set com_ns = namespace(COMIN_ATMOS_ANALYSIS_MEM = COM_ATMOS_ANALYSIS_TMPL, + COMIN_ATMOS_HISTORY_MEM = COM_ATMOS_HISTORY_TMPL, + COMIN_ATMOS_RESTART_MEM = COM_ATMOS_RESTART_TMPL) %} {% for key in tmpl_dict.keys() %} {% set search_term = '${' + key + '}' %} From 0b810c888239853fedd0e4584fe62536c6aaacdf Mon Sep 17 00:00:00 2001 From: "Henry R. Winterbottom" <49202169+HenryWinterbottom-NOAA@users.noreply.github.com> Date: Tue, 18 Jun 2024 20:32:48 -0600 Subject: [PATCH 8/9] Removes misleading "No such file or directory" syntax errors from output files (#2688) This PR addresses issue #1252. The following is accomplished: - Prior to removing files, the existence of a file is checked prior to attempting to remove; this is performed as noted [here](https://github.com/NOAA-EMC/global-workflow/issues/1252#issue-1538627369); this PR only addresses the the `chgrp` issue. Refs #1252 --------- Co-authored-by: David Huber <69919478+DavidHuber-NOAA@users.noreply.github.com> --- scripts/exglobal_diag.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/scripts/exglobal_diag.sh b/scripts/exglobal_diag.sh index 2d48053bd2..e1faa7b2ee 100755 --- a/scripts/exglobal_diag.sh +++ b/scripts/exglobal_diag.sh @@ -232,9 +232,11 @@ EOFdiag # Restrict diagnostic files containing rstprod data rlist="conv_gps conv_ps conv_pw conv_q conv_sst conv_t conv_uv saphir" for rtype in $rlist; do - set +e - ${CHGRP_CMD} *${rtype}* - ${STRICT_ON:-set -e} + for rfile in *"${rtype}"*; do + if [[ -s "${rfile}" ]]; then + ${CHGRP_CMD} "${rfile}" + fi + done done # If requested, create diagnostic file tarballs From 8993b42cb91144c0ab0501dc7841ea8d675c4701 Mon Sep 17 00:00:00 2001 From: Walter Kolczynski - NOAA Date: Wed, 19 Jun 2024 21:51:22 -0400 Subject: [PATCH 9/9] Eliminate post groups (#2667) Eliminates the post groups used for upp and products jobs so that each task only processes one forecast hour. This is more efficient and greatly simplifies downstream dependencies that depend on a specific forecast hour. Resolves #2666 Refs #2642 --- .gitignore | 8 +++++ jobs/rocoto/atmos_ensstat.sh | 20 +++-------- jobs/rocoto/atmos_products.sh | 22 ++++-------- jobs/rocoto/oceanice_products.sh | 18 ++++------ jobs/rocoto/upp.sh | 17 ++++----- parm/config/gefs/config.base | 2 ++ scripts/exgdas_enkf_earc.py | 2 +- workflow/rocoto/gefs_tasks.py | 9 ++--- workflow/rocoto/gfs_tasks.py | 59 ++++++++++---------------------- 9 files changed, 56 insertions(+), 101 deletions(-) diff --git a/.gitignore b/.gitignore index 8314f5bae9..0c15a50661 100644 --- a/.gitignore +++ b/.gitignore @@ -166,6 +166,14 @@ scripts/exemcsfc_global_sfc_prep.sh scripts/exgdas_global_marine_analysis_ecen.py scripts/exglobal_prep_ocean_obs.py # ush symlinks +ush/bufr2ioda_insitu_profile_argo.py +ush/bufr2ioda_insitu_profile_bathy.py +ush/bufr2ioda_insitu_profile_glider.py +ush/bufr2ioda_insitu_profile_marinemammal.py +ush/bufr2ioda_insitu_profile_tesac.py +ush/bufr2ioda_insitu_profile_xbtctd.py +ush/bufr2ioda_insitu_surface_altkob.py +ush/bufr2ioda_insitu_surface_trkob.py ush/chgres_cube.sh ush/emcsfc_ice_blend.sh ush/emcsfc_snow.sh diff --git a/jobs/rocoto/atmos_ensstat.sh b/jobs/rocoto/atmos_ensstat.sh index cbaaff9a0f..76ed7f0a72 100755 --- a/jobs/rocoto/atmos_ensstat.sh +++ b/jobs/rocoto/atmos_ensstat.sh @@ -15,21 +15,11 @@ if (( status != 0 )); then exit "${status}"; fi export job="atmos_ensstat" export jobid="${job}.$$" -############################################################### -# shellcheck disable=SC2153,SC2001 -IFS='_' read -ra fhrs <<< "${FHRLST//f}" # strip off the 'f's and convert to array +export FORECAST_HOUR=$(( 10#${FHR3} )) -#--------------------------------------------------------------- +############################################################### # Execute the JJOB -for fhr in "${fhrs[@]}"; do - # The analysis fhr is -001. Performing math on negative, leading 0 integers is tricky. - # The negative needs to be in front of "10#", so do some regex magic to make it happen. - fhr="10#${fhr}" - fhr=${fhr//10\#-/-10\#} - export FORECAST_HOUR=$(( fhr )) - "${HOMEgfs}/jobs/JGLOBAL_ATMOS_ENSSTAT" - status=$? - if (( status != 0 )); then exit "${status}"; fi -done +############################################################### +"${HOMEgfs}/jobs/JGLOBAL_ATMOS_ENSSTAT" -exit 0 +exit $? diff --git a/jobs/rocoto/atmos_products.sh b/jobs/rocoto/atmos_products.sh index 472f202de8..f6adbcf861 100755 --- a/jobs/rocoto/atmos_products.sh +++ b/jobs/rocoto/atmos_products.sh @@ -15,21 +15,13 @@ if (( status != 0 )); then exit "${status}"; fi export job="atmos_products" export jobid="${job}.$$" -############################################################### -# shellcheck disable=SC2153,SC2001 -IFS='_' read -ra fhrs <<< "${FHRLST//f}" # strip off the 'f's and convert to array +# Negatation needs to be before the base +fhr3_base="10#${FHR3}" +export FORECAST_HOUR=$(( ${fhr3_base/10#-/-10#} )) -#--------------------------------------------------------------- +############################################################### # Execute the JJOB -for fhr in "${fhrs[@]}"; do - # The analysis fhr is -001. Performing math on negative, leading 0 integers is tricky. - # The negative needs to be in front of "10#", so do some regex magic to make it happen. - fhr="10#${fhr}" - fhr=${fhr//10\#-/-10\#} - export FORECAST_HOUR=$(( fhr )) - "${HOMEgfs}/jobs/JGLOBAL_ATMOS_PRODUCTS" - status=$? - if (( status != 0 )); then exit "${status}"; fi -done +############################################################### +"${HOMEgfs}/jobs/JGLOBAL_ATMOS_PRODUCTS" -exit 0 +exit $? diff --git a/jobs/rocoto/oceanice_products.sh b/jobs/rocoto/oceanice_products.sh index 48816fb3a1..eb704fb35f 100755 --- a/jobs/rocoto/oceanice_products.sh +++ b/jobs/rocoto/oceanice_products.sh @@ -21,17 +21,11 @@ export PYTHONPATH export job="oceanice_products" export jobid="${job}.$$" -############################################################### -# shellcheck disable=SC2153,SC2001 -IFS='_' read -ra fhrs <<< "${FHRLST//f}" # strip off the 'f's and convert to array +export FORECAST_HOUR=$(( 10#${FHR3} )) -#--------------------------------------------------------------- +############################################################### # Execute the JJOB -for fhr in "${fhrs[@]}"; do - export FORECAST_HOUR=$(( 10#${fhr} )) - "${HOMEgfs}/jobs/JGLOBAL_OCEANICE_PRODUCTS" - status=$? - if (( status != 0 )); then exit "${status}"; fi -done - -exit 0 +############################################################### +"${HOMEgfs}/jobs/JGLOBAL_OCEANICE_PRODUCTS" + +exit $? diff --git a/jobs/rocoto/upp.sh b/jobs/rocoto/upp.sh index 18d5c12cea..da0180472d 100755 --- a/jobs/rocoto/upp.sh +++ b/jobs/rocoto/upp.sh @@ -44,16 +44,11 @@ export PYTHONPATH export job="upp" export jobid="${job}.$$" -############################################################### -# shellcheck disable=SC2153,SC2001 -IFS='_' read -ra fhrs <<< "${FHRLST//f}" # strip off the 'f's convert to array +export FORECAST_HOUR=$(( 10#${FHR3} )) +############################################################### # Execute the JJOB -for fhr in "${fhrs[@]}"; do - export FORECAST_HOUR=$(( 10#${fhr} )) - "${HOMEgfs}/jobs/JGLOBAL_ATMOS_UPP" - status=$? - if (( status != 0 )); then exit "${status}"; fi -done - -exit 0 +############################################################### +"${HOMEgfs}/jobs/JGLOBAL_ATMOS_UPP" + +exit $? diff --git a/parm/config/gefs/config.base b/parm/config/gefs/config.base index e98eb41208..16e0fefaba 100644 --- a/parm/config/gefs/config.base +++ b/parm/config/gefs/config.base @@ -256,6 +256,8 @@ export ILPOST=1 # gempak output frequency up to F120 export FHMIN_ENKF=${FHMIN_GFS} export FHMAX_ENKF=${FHMAX_GFS} export FHOUT_ENKF=${FHOUT_GFS} +export FHOUT_OCN=${FHOUT_OCN_GFS} +export FHOUT_ICE=${FHOUT_ICE_GFS} # GFS restart interval in hours export restart_interval_gfs=12 diff --git a/scripts/exgdas_enkf_earc.py b/scripts/exgdas_enkf_earc.py index c0fc51eee5..a515ec9746 100755 --- a/scripts/exgdas_enkf_earc.py +++ b/scripts/exgdas_enkf_earc.py @@ -36,7 +36,7 @@ def main(): # Also import all COMIN* directory and template variables for key in archive.task_config.keys(): - if key.startswith("COMIN"): + if key.startswith("COM"): archive_dict[key] = archive.task_config[key] cwd = os.getcwd() diff --git a/workflow/rocoto/gefs_tasks.py b/workflow/rocoto/gefs_tasks.py index 6fffc881e0..cfd8fa7093 100644 --- a/workflow/rocoto/gefs_tasks.py +++ b/workflow/rocoto/gefs_tasks.py @@ -253,7 +253,7 @@ def _atmosoceaniceprod(self, component: str): postenvars = self.envars.copy() postenvar_dict = {'ENSMEM': '#member#', 'MEMDIR': 'mem#member#', - 'FHRLST': '#fhr#', + 'FHR3': '#fhr#', 'COMPONENT': component} for key, value in postenvar_dict.items(): postenvars.append(rocoto.create_envar(name=key, value=str(value))) @@ -270,11 +270,6 @@ def _atmosoceaniceprod(self, component: str): 'maxtries': '&MAXTRIES;'} fhrs = self._get_forecast_hours('gefs', self._configs[config], component) - - # ocean/ice components do not have fhr 0 as they are averaged output - if component in ['ocean', 'ice'] and 0 in fhrs: - fhrs.remove(0) - fhr_var_dict = {'fhr': ' '.join([f"{fhr:03d}" for fhr in fhrs])} fhr_metatask_dict = {'task_name': f'{component}_prod_#member#', @@ -303,7 +298,7 @@ def atmos_ensstat(self): dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) postenvars = self.envars.copy() - postenvar_dict = {'FHRLST': '#fhr#'} + postenvar_dict = {'FHR3': '#fhr#'} for key, value in postenvar_dict.items(): postenvars.append(rocoto.create_envar(name=key, value=str(value))) diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index 2c74d0f854..60a08549b6 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -939,7 +939,7 @@ def _fcst_cycled(self): def atmanlupp(self): postenvars = self.envars.copy() - postenvar_dict = {'FHRLST': 'f000', + postenvar_dict = {'FHR3': '000', 'UPP_RUN': 'analysis'} for key, value in postenvar_dict.items(): postenvars.append(rocoto.create_envar(name=key, value=str(value))) @@ -975,7 +975,7 @@ def atmanlupp(self): def atmanlprod(self): postenvars = self.envars.copy() - postenvar_dict = {'FHRLST': '-f001'} + postenvar_dict = {'FHR3': '-001'} for key, value in postenvar_dict.items(): postenvars.append(rocoto.create_envar(name=key, value=str(value))) @@ -1002,24 +1002,6 @@ def atmanlprod(self): return task - @staticmethod - def _get_ufs_postproc_grps(cdump, config, component='atmos'): - - fhrs = Tasks._get_forecast_hours(cdump, config, component=component) - - nfhrs_per_grp = config.get('NFHRS_PER_GROUP', 1) - ngrps = len(fhrs) // nfhrs_per_grp if len(fhrs) % nfhrs_per_grp == 0 else len(fhrs) // nfhrs_per_grp + 1 - - fhrs = [f'f{fhr:03d}' for fhr in fhrs] - fhrs = np.array_split(fhrs, ngrps) - fhrs = [fhr.tolist() for fhr in fhrs] - - grp = ' '.join(f'_{fhr[0]}-{fhr[-1]}' if len(fhr) > 1 else f'_{fhr[0]}' for fhr in fhrs) - dep = ' '.join([fhr[-1] for fhr in fhrs]) - lst = ' '.join(['_'.join(fhr) for fhr in fhrs]) - - return grp, dep, lst - def atmupp(self): return self._upptask(upp_run='forecast', task_id='atmupp') @@ -1032,32 +1014,28 @@ def _upptask(self, upp_run="forecast", task_id="atmupp"): if upp_run not in VALID_UPP_RUN: raise KeyError(f"{upp_run} is invalid; UPP_RUN options are: {('|').join(VALID_UPP_RUN)}") - varname1, varname2, varname3 = 'grp', 'dep', 'lst' - varval1, varval2, varval3 = self._get_ufs_postproc_grps(self.cdump, self._configs['upp']) - var_dict = {varname1: varval1, varname2: varval2, varname3: varval3} - postenvars = self.envars.copy() - postenvar_dict = {'FHRLST': '#lst#', + postenvar_dict = {'FHR3': '#fhr#', 'UPP_RUN': upp_run} for key, value in postenvar_dict.items(): postenvars.append(rocoto.create_envar(name=key, value=str(value))) atm_hist_path = self._template_to_rocoto_cycstring(self._base["COM_ATMOS_HISTORY_TMPL"]) deps = [] - data = f'{atm_hist_path}/{self.cdump}.t@Hz.atm#dep#.nc' + data = f'{atm_hist_path}/{self.cdump}.t@Hz.atmf#fhr#.nc' dep_dict = {'type': 'data', 'data': data, 'age': 120} deps.append(rocoto.add_dependency(dep_dict)) - data = f'{atm_hist_path}/{self.cdump}.t@Hz.sfc#dep#.nc' + data = f'{atm_hist_path}/{self.cdump}.t@Hz.sfcf#fhr#.nc' dep_dict = {'type': 'data', 'data': data, 'age': 120} deps.append(rocoto.add_dependency(dep_dict)) - data = f'{atm_hist_path}/{self.cdump}.t@Hz.atm.log#dep#.txt' + data = f'{atm_hist_path}/{self.cdump}.t@Hz.atm.logf#fhr#.txt' dep_dict = {'type': 'data', 'data': data, 'age': 60} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep=deps, dep_condition='and') cycledef = 'gdas_half,gdas' if self.cdump in ['gdas'] else self.cdump resources = self.get_resource('upp') - task_name = f'{self.cdump}{task_id}#{varname1}#' + task_name = f'{self.cdump}{task_id}_f#fhr#' task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, @@ -1069,9 +1047,12 @@ def _upptask(self, upp_run="forecast", task_id="atmupp"): 'maxtries': '&MAXTRIES;' } + fhrs = self._get_forecast_hours(self.cdump, self._configs['upp']) + fhr_var_dict = {'fhr': ' '.join([f"{fhr:03d}" for fhr in fhrs])} + metatask_dict = {'task_name': f'{self.cdump}{task_id}', 'task_dict': task_dict, - 'var_dict': var_dict + 'var_dict': fhr_var_dict } task = rocoto.create_task(metatask_dict) @@ -1091,25 +1072,21 @@ def _atmosoceaniceprod(self, component: str): products_dict = {'atmos': {'config': 'atmos_products', 'history_path_tmpl': 'COM_ATMOS_MASTER_TMPL', - 'history_file_tmpl': f'{self.cdump}.t@Hz.master.grb2#dep#'}, + 'history_file_tmpl': f'{self.cdump}.t@Hz.master.grb2f#fhr#'}, 'ocean': {'config': 'oceanice_products', 'history_path_tmpl': 'COM_OCEAN_HISTORY_TMPL', - 'history_file_tmpl': f'{self.cdump}.ocean.t@Hz.6hr_avg.#dep#.nc'}, + 'history_file_tmpl': f'{self.cdump}.ocean.t@Hz.6hr_avg.f#fhr#.nc'}, 'ice': {'config': 'oceanice_products', 'history_path_tmpl': 'COM_ICE_HISTORY_TMPL', - 'history_file_tmpl': f'{self.cdump}.ice.t@Hz.6hr_avg.#dep#.nc'}} + 'history_file_tmpl': f'{self.cdump}.ice.t@Hz.6hr_avg.f#fhr#.nc'}} component_dict = products_dict[component] config = component_dict['config'] history_path_tmpl = component_dict['history_path_tmpl'] history_file_tmpl = component_dict['history_file_tmpl'] - varname1, varname2, varname3 = 'grp', 'dep', 'lst' - varval1, varval2, varval3 = self._get_ufs_postproc_grps(self.cdump, self._configs[config], component=component) - var_dict = {varname1: varval1, varname2: varval2, varname3: varval3} - postenvars = self.envars.copy() - postenvar_dict = {'FHRLST': '#lst#', 'COMPONENT': component} + postenvar_dict = {'FHR3': '#fhr#', 'COMPONENT': component} for key, value in postenvar_dict.items(): postenvars.append(rocoto.create_envar(name=key, value=str(value))) @@ -1129,7 +1106,7 @@ def _atmosoceaniceprod(self, component: str): cycledef = 'gdas_half,gdas' if self.cdump in ['gdas'] else self.cdump resources = self.get_resource(component_dict['config']) - task_name = f'{self.cdump}{component}_prod#{varname1}#' + task_name = f'{self.cdump}{component}_prod_f#fhr#' task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, @@ -1141,9 +1118,11 @@ def _atmosoceaniceprod(self, component: str): 'maxtries': '&MAXTRIES;' } + fhrs = self._get_forecast_hours(self.cdump, self._configs[config], component) + fhr_var_dict = {'fhr': ' '.join([f"{fhr:03d}" for fhr in fhrs])} metatask_dict = {'task_name': f'{self.cdump}{component}_prod', 'task_dict': task_dict, - 'var_dict': var_dict + 'var_dict': fhr_var_dict } task = rocoto.create_task(metatask_dict)