Skip to content

Commit

Permalink
Update prep for COM refactor
Browse files Browse the repository at this point in the history 
Updates the prep job to use the new COM templates. These changes
require an update to the prepobs script `ush/getges_driver.sh`.
For the time being, this is being done using minimal modifications
in a private copy of prepobs. A more thorough review and update of
prepobs (and obsproc) will be required before this branch is merged
into develop, which may also result in changes to workflow scripts
as well.

Refs: NOAA-EMC#761
  • Loading branch information
@WalterKolczynski-NOAA
WalterKolczynski-NOAA committed Feb 1, 2023
1 parent 4aebba5 commit 881f237
Show file tree
Hide file tree
Showing 5 changed files with 128 additions and 104 deletions.
8 changes: 2 additions & 6 deletions jobs/JGLOBAL_ATMOS_TROPCY_QC_RELOC
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@ export RUN_ENVIR=${RUN_ENVIR:-"nco"}
##############################################
export CDATE=${CDATE:-${PDY}${cyc}}
export CDUMP=${CDUMP:-${RUN:-"gfs"}}
export COMPONENT="atmos"


##############################################
Expand All @@ -33,11 +32,8 @@ export TANK_TROPCY=${TANK_TROPCY:-${DCOMROOT}} # path to tropical cyclone reco
##############################################
# Define COM directories
##############################################
export COMIN=${ROTDIR}/${RUN}.${PDY}/${cyc}/${COMPONENT}
export COMOUT=${ROTDIR}/${RUN}.${PDY}/${cyc}/${COMPONENT}
if [ ! -d ${COMOUT} ]; then mkdir -p ${COMOUT}; fi
#export COMINgdas=${ROTDIR}/gdas.${PDY}/${cyc}
#export COMINgfs=${ROTDIR}/gfs.${PDY}/${cyc}
generate_com COM_OBS
if [[ ! -d "${COM_OBS}" ]]; then mkdir -p "${COM_OBS}"; fi

export CRES=$(echo ${CASE} | cut -c2-)
export LATB=$((CRES*2))
Expand Down
104 changes: 64 additions & 40 deletions jobs/rocoto/prep.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,32 +26,49 @@ status=$?
###############################################################
# Set script and dependency variables
export OPREFIX="${CDUMP}.t${cyc}z."
export COMOUT="${ROTDIR}/${CDUMP}.${PDY}/${cyc}/atmos"
generate_com -rx COM_OBS COM_OBSDMP

DATE_PREV=$(${NDATE} -${assim_freq} "${PDY}${cyc}")
PDY_PREV=$(echo $DATE_PREV | cut -c1-8)
cyc_PREV=$(echo $DATE_PREV | cut -c9-10)
GDUMP="gdas"
# shellcheck disable=SC2030
COM_OBS_PREV=$({
PDY=${PDY_PREV}
cyc=${cyc_PREV}
CDUMP=${GDUMP}
echo "${COM_OBS_TMPL}" | envsubst
})
declare -rx COM_OBS_PREV

COM_OBSDMP_PREV=$({
PDY=${PDY_PREV}
cyc=${cyc_PREV}
CDUMP=${GDUMP}
echo "${COM_OBSDMP_TMPL}" | envsubst
})
declare -rx COM_OBSDMP_PREV
# shellcheck disable=

export MAKE_PREPBUFR=${MAKE_PREPBUFR:-"YES"}
[[ ! -d ${COMOUT} ]] && mkdir -p ${COMOUT}
[[ ! -d ${COMIN_OBS} ]] && mkdir -p ${COMIN_OBS}
if [[ ! -d "${COM_OBS}" ]]; then mkdir -p "${COM_OBS}"; fi

###############################################################
# If ROTDIR_DUMP=YES, copy dump files to rotdir
if [[ ${ROTDIR_DUMP} = "YES" ]]; then
${HOMEgfs}/ush/getdump.sh "${CDATE}" "${CDUMP}" "${DMPDIR}/${CDUMP}${DUMP_SUFFIX}.${PDY}/${cyc}/atmos" "${COMIN_OBS}"
if [ $ROTDIR_DUMP = "YES" ]; then
"${HOMEgfs}/ush/getdump.sh" "${PDY}${cyc}" "${CDUMP}" "${COM_OBSDMP}" "${COM_OBS}"
status=$?
[[ ${status} -ne 0 ]] && exit ${status}

# Ensure previous cycle gdas dumps are available (used by cycle & downstream)
GDATE=$(${NDATE} -${assim_freq} ${CDATE})
gPDY=$(echo ${GDATE} | cut -c1-8)
gcyc=$(echo ${GDATE} | cut -c9-10)
GDUMP=gdas
gCOMOBS="${ROTDIR}/${GDUMP}.${gPDY}/${gcyc}/obs"
if [[ ! -s ${gCOMOBS}/${GDUMP}.t${gcyc}z.updated.status.tm00.bufr_d ]]; then
${HOMEgfs}/ush/getdump.sh "${GDATE}" "${GDUMP}" "${DMPDIR}/${GDUMP}${DUMP_SUFFIX}.${gPDY}/${gcyc}/atmos" "${gCOMOBS}"
# Ensure previous cycle gdas dumps are available (used by cycle & downstream)
if [[ ! -s "${COM_OBS_PREV}/${GDUMP}.t${cyc_PREV}z.updated.status.tm00.bufr_d" ]]; then
"${HOMEgfs}/ush/getdump.sh" "${DATE_PREV}" "${GDUMP}" "${COM_OBSDMP_PREV}" "${COM_OBS_PREV}"
status=$?
[[ ${status} -ne 0 ]] && exit ${status}
fi
# exception handling to ensure no dead link
[[ $(find ${COMIN_OBS} -xtype l | wc -l) -ge 1 ]] && exit 9
[[ $(find ${gCOMOBS} -xtype l | wc -l) -ge 1 ]] && exit 9
[[ $(find ${COM_OBS} -xtype l | wc -l) -ge 1 ]] && exit 9
[[ $(find ${COM_OBS_PREV} -xtype l | wc -l) -ge 1 ]] && exit 9
fi


Expand All @@ -75,40 +92,38 @@ if [[ ${PROCESS_TROPCY} = "YES" ]]; then
fi
fi

[[ ${ROTDIR_DUMP} = "YES" ]] && rm ${COMOUT}${CDUMP}.t${cyc}z.syndata.tcvitals.tm00
if [[ ${ROTDIR_DUMP} = "YES" ]]; then rm "${COM_OBS}/${CDUMP}.t${cyc}z.syndata.tcvitals.tm00"; fi

${HOMEgfs}/jobs/JGLOBAL_ATMOS_TROPCY_QC_RELOC
status=$?
[[ ${status} -ne 0 ]] && exit ${status}

else
[[ ${ROTDIR_DUMP} = "NO" ]] && cp ${DMPDIR}/${CDUMP}${DUMP_SUFFIX}.${PDY}/${cyc}/atmos/${CDUMP}.t${cyc}z.syndata.tcvitals.tm00 ${COMOUT}/
if [[ ${ROTDIR_DUMP} = "NO" ]]; then cp "${COM_OBSDMP}/${CDUMP}.t${cyc}z.syndata.tcvitals.tm00" "${COM_OBS}/"; fi
fi

# Will modify the new location later in the next PR to address issue 1198
if [[ ${ROTDIR_DUMP} = "YES" ]]; then
mv ${COMIN_OBS}/*syndata.tcvitals.tm00 ${COMOUT}
mv ${COMIN_OBS}/*snogrb_t1534.3072.1536 ${COMOUT}
mv ${COMIN_OBS}/*seaice.5min.blend.grb ${COMOUT}
fi

###############################################################
# Generate prepbufr files from dumps or copy from OPS
if [[ ${MAKE_PREPBUFR} = "YES" ]]; then
if [[ ${ROTDIR_DUMP} = "YES" ]]; then
rm -f ${COMIN_OBS}/${OPREFIX}prepbufr
rm -f ${COMIN_OBS}/${OPREFIX}prepbufr.acft_profiles
rm -f ${COMIN_OBS}/${OPREFIX}nsstbufr
if [ $MAKE_PREPBUFR = "YES" ]; then
if [ $ROTDIR_DUMP = "YES" ]; then
rm -f "${COM_OBS}/${OPREFIX}prepbufr"
rm -f "${COM_OBS}/${OPREFIX}prepbufr.acft_profiles"
rm -f "${COM_OBS}/${OPREFIX}nsstbufr"
fi

export job="j${CDUMP}_prep_${cyc}"
export DATAROOT="${RUNDIR}/${CDATE}/${CDUMP}/prepbufr"
export COMIN=${COMIN_OBS}
export COMINgdas=${COMINgdas:-${ROTDIR}/gdas.${PDY}/${cyc}/atmos}
export COMINgfs=${COMINgfs:-${ROTDIR}/gfs.${PDY}/${cyc}/atmos}
export COMOUT=${COMIN_OBS}
if [[ ${ROTDIR_DUMP} = "NO" ]]; then
COMIN_OBS=${COMIN_OBS:-${DMPDIR}/${CDUMP}${DUMP_SUFFIX}.${PDY}/${cyc}/atmos}
export COMIN=${COM_OBS}
export COMOUT=${COM_OBS}
COMINgdas=$(RUN="gdas"; echo "${COM_ATMOS_HISTORY_TMPL}" | envsubst)
declare -rx COMINgdas
COMINgfs=$(RUN="gfs"; echo "${COM_ATMOS_HISTORY_TMPL}" | envsubst)
declare -rx COMINgfs
if [ $ROTDIR_DUMP = "NO" ]; then
export COMSP=${COMSP:-"${COM_OBSDMP}/${CDUMP}.t${cyc}z."}
else
export COMSP=${COMSP:-"${COM_OBS}/${CDUMP}.t${cyc}z."}
fi
export COMSP=${COMSP:-${COMIN_OBS}/${CDUMP}.t${cyc}z.}

Expand All @@ -117,20 +132,29 @@ if [[ ${MAKE_PREPBUFR} = "YES" ]]; then
export MAKE_NSSTBUFR="NO"
fi

${HOMEobsproc}/jobs/JOBSPROC_GLOBAL_PREP
export DEBUG_LEVEL="4"
export LOUD="YES"
export HOMEprepobs="${BASE_GIT}/prepobs/v1.0.1"
export EXECprepobs=${HOMEprepobs}/exec
export FIXprepobs=${HOMEprepobs}/fix
export SCRIPTprepobs=${HOMEprepobs}/script
export USHprepobs=${HOMEprepobs}/ush


$HOMEobsproc/jobs/JOBSPROC_GLOBAL_PREP
status=$?
[[ ${status} -ne 0 ]] && exit ${status}

# If creating NSSTBUFR was disabled, copy from DMPDIR if appropriate.
if [[ ${MAKE_NSSTBUFR:-"NO"} = "NO" ]]; then
[[ ${DONST} = "YES" ]] && ${NCP} ${DMPDIR}/${CDUMP}${DUMP_SUFFIX}.${PDY}/${cyc}/atmos/${OPREFIX}nsstbufr ${COMIN_OBS}/${OPREFIX}nsstbufr
if [[ $DONST = "YES" ]]; then ${NCP} "${COM_OBSDMP}/${OPREFIX}nsstbufr" "${COM_OBS}/${OPREFIX}nsstbufr"; fi
fi

else
if [[ ${ROTDIR_DUMP} = "NO" ]]; then
${NCP} ${DMPDIR}/${CDUMP}${DUMP_SUFFIX}.${PDY}/${cyc}/atmos/${OPREFIX}prepbufr ${COMIN_OBS}/${OPREFIX}prepbufr
${NCP} ${DMPDIR}/${CDUMP}${DUMP_SUFFIX}.${PDY}/${cyc}/atmos/${OPREFIX}prepbufr.acft_profiles ${COMIN_OBS}/${OPREFIX}prepbufr.acft_profiles
[[ ${DONST} = "YES" ]] && ${NCP} ${DMPDIR}/${CDUMP}${DUMP_SUFFIX}.${PDY}/${cyc}/atmos/${OPREFIX}nsstbufr ${COMIN_OBS}/${OPREFIX}nsstbufr
if [ $ROTDIR_DUMP = "NO" ]; then
${NCP} "${COM_OBSDMP}/${OPREFIX}prepbufr" "${COM_OBS}/${OPREFIX}prepbufr"
${NCP} "${COM_OBSDMP}/${OPREFIX}prepbufr.acft_profiles" "${COM_OBS}/${OPREFIX}prepbufr.acft_profiles"
if [[ $DONST = "YES" ]]; then ${NCP} "${COM_OBSDMP}/${OPREFIX}nsstbufr" "${COM_OBS}/${OPREFIX}nsstbufr"; fi
fi
fi

Expand Down
41 changes: 20 additions & 21 deletions scripts/exglobal_atmos_tropcy_qc_reloc.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,6 @@ cd $DATA

cat break > $pgmout

export COMSP=$COMOUT/${RUN}.${cycle}.

tmhr=$(echo $tmmark|cut -c3-4)
cdate10=$( ${NDATE:?} -$tmhr $PDY$cyc)

Expand Down Expand Up @@ -60,7 +58,7 @@ if [ "$PROCESS_TROPCY" = 'YES' ]; then
fi


cd $COMOUT
cd "${COM_OBS}" || exit 1
pwd
ls -ltr *syndata*
cd $ARCHSYND
Expand All @@ -79,10 +77,10 @@ else
# don't want to wipe out these files)
#

[ ! -s ${COMSP}syndata.tcvitals.$tmmark ] && \
cp /dev/null ${COMSP}syndata.tcvitals.$tmmark
[ ! -s ${COMSP}jtwc-fnoc.tcvitals.$tmmark ] && \
cp /dev/null ${COMSP}jtwc-fnoc.tcvitals.$tmmark
[ ! -s "${COM_OBS}/${RUN}.t${cyc}z.syndata.tcvitals.${tmmark}" ] && \
cp "/dev/null" "${COM_OBS}/${RUN}.t${cyc}z.syndata.tcvitals.${tmmark}"
[ ! -s "${COM_OBS}/${RUN}.t${cyc}z.jtwc-fnoc.tcvitals.${tmmark}" ] && \
cp "/dev/null" "${COM_OBS}/${RUN}.t${cyc}z.jtwc-fnoc.tcvitals.${tmmark}"

# endif loop $PROCESS_TROPCY
fi
Expand Down Expand Up @@ -110,34 +108,35 @@ if [ "$DO_RELOCATE" = 'YES' ]; then
[ $RUN = gfs -o $RUN = gdas -o $NET = cfs ] && qual_last=""

if [ $BKGFREQ -eq 1 ]; then
[ -s sgm3prep ] && cp sgm3prep ${COMSP}sgm3prep${qual_last}
[ -s sgm2prep ] && cp sgm2prep ${COMSP}sgm2prep${qual_last}
[ -s sgm1prep ] && cp sgm1prep ${COMSP}sgm1prep${qual_last}
[ -s sgesprep ] && cp sgesprep ${COMSP}sgesprep${qual_last}
[ -s sgp1prep ] && cp sgp1prep ${COMSP}sgp1prep${qual_last}
[ -s sgp2prep ] && cp sgp2prep ${COMSP}sgp2prep${qual_last}
[ -s sgp3prep ] && cp sgp3prep ${COMSP}sgp3prep${qual_last}
if [[ -s sgm3prep ]]; then cp "sgm3prep" "${COM_OBS}/${RUN}.t${cyc}z.sgm3prep${qual_last}"; fi
if [[ -s sgm2prep ]]; then cp "sgm2prep" "${COM_OBS}/${RUN}.t${cyc}z.sgm2prep${qual_last}"; fi
if [[ -s sgm1prep ]]; then cp "sgm1prep" "${COM_OBS}/${RUN}.t${cyc}z.sgm1prep${qual_last}"; fi
if [[ -s sgesprep ]]; then cp "sgesprep" "${COM_OBS}/${RUN}.t${cyc}z.sgesprep${qual_last}"; fi
if [[ -s sgp1prep ]]; then cp "sgp1prep" "${COM_OBS}/${RUN}.t${cyc}z.sgp1prep${qual_last}"; fi
if [[ -s sgp2prep ]]; then cp "sgp2prep" "${COM_OBS}/${RUN}.t${cyc}z.sgp2prep${qual_last}"; fi
if [[ -s sgp3prep ]]; then cp "sgp3prep" "${COM_OBS}/${RUN}.t${cyc}z.sgp3prep${qual_last}"; fi
elif [ $BKGFREQ -eq 3 ]; then
[ -s sgm3prep ] && cp sgm3prep ${COMSP}sgm3prep${qual_last}
[ -s sgesprep ] && cp sgesprep ${COMSP}sgesprep${qual_last}
[ -s sgp3prep ] && cp sgp3prep ${COMSP}sgp3prep${qual_last}
if [[ -s sgm3prep ]]; then cp "sgm3prep" "${COM_OBS}/${RUN}.t${cyc}z.sgm3prep${qual_last}"; fi
if [[ -s sgesprep ]]; then cp "sgesprep" "${COM_OBS}/${RUN}.t${cyc}z.sgesprep${qual_last}"; fi
if [[ -s sgp3prep ]]; then cp "sgp3prep" "${COM_OBS}/${RUN}.t${cyc}z.sgp3prep${qual_last}"; fi
fi

# The existence of ${COMSP}tropcy_relocation_status.$tmmark file will tell the
# The existence of ${COM_OBS}/${RUN}.t${cyc}z.tropcy_relocation_status.$tmmark file will tell the
# subsequent PREP processing that RELOCATION processing occurred, if this file
# does not already exist at this point, echo "RECORDS PROCESSED" into it to
# further tell PREP processing that records were processed by relocation and
# the global sigma guess was modified by tropical cyclone relocation
# Note: If ${COMSP}tropcy_relocation_status.$tmmark already exists at this
# Note: If ${COM_OBS}/${RUN}.t${cyc}z.tropcy_relocation_status.$tmmark already exists at this
# point it means that it contains the string "NO RECORDS to process"
# and was created by the child script tropcy_relocate.sh because records
# were not processed by relocation and the global sigma guess was NOT
# modified by tropical cyclone relocation (because no tcvitals records
# were found in the relocation step)
# ----------------------------------------------------------------------------

[ ! -s ${COMSP}tropcy_relocation_status.$tmmark ] && \
echo "RECORDS PROCESSED" > ${COMSP}tropcy_relocation_status.$tmmark
if [[ ! -s "${COM_OBS}/${RUN}.t${cyc}z.tropcy_relocation_status.${tmmark}" ]]; then
echo "RECORDS PROCESSED" > "${COM_OBS}/${RUN}.t${cyc}z.tropcy_relocation_status.${tmmark}"
fi

# endif loop $DO_RELOCATE
fi
Expand Down
32 changes: 18 additions & 14 deletions ush/syndat_qctropcy.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -113,17 +113,19 @@ positional parameter 1"
set_trace
echo $msg >> $pgmout

# Copy null files into "${COMSP}syndata.tcvitals.$tmmark" and
# "${COMSP}jtwc-fnoc.tcvitals.$tmmark" so later ftp attempts will find and
# Copy null files into "${COM_OBS}/${RUN}.${cycle}.syndata.tcvitals.$tmmark" and
# "${COM_OBS}/${RUN}.${cycle}.jtwc-fnoc.tcvitals.$tmmark" so later ftp attempts will find and
# copy the zero-length file and avoid wasting time with multiple attempts
# to remote machine(s)
# (Note: Only do so if files don't already exist)

if [ $SENDCOM = YES ]; then
[ ! -s ${COMSP}syndata.tcvitals.$tmmark ] && \
cp /dev/null ${COMSP}syndata.tcvitals.$tmmark
[ ! -s ${COMSP}jtwc-fnoc.tcvitals.$tmmark ] && \
cp /dev/null ${COMSP}jtwc-fnoc.tcvitals.$tmmark
if [[ ! -s "${COM_OBS}/${RUN}.${cycle}.syndata.tcvitals.${tmmark}" ]]; then
cp "/dev/null" "${COM_OBS}/${RUN}.${cycle}.syndata.tcvitals.${tmmark}"
fi
if [[ ! -s "${COM_OBS}/${RUN}.${cycle}.jtwc-fnoc.tcvitals.${tmmark}" ]]; then
cp "/dev/null" "${COM_OBS}/${RUN}.${cycle}.jtwc-fnoc.tcvitals.${tmmark}"
fi
fi

exit
Expand Down Expand Up @@ -289,16 +291,18 @@ if [ "$errqct" -gt '0' ];then
echo $msg >> $pgmout

# In the event of a ERROR in PROGRAM SYNDAT_QCTROPCY, copy null files into
# "${COMSP}syndata.tcvitals.$tmmark" and "${COMSP}jtwc-fnoc.tcvitals.$tmmark"
# "${COM_OBS}/${RUN}.${cycle}.syndata.tcvitals.$tmmark" and "${COM_OBS}/${RUN}.${cycle}.jtwc-fnoc.tcvitals.$tmmark"
# so later ftp attempts will find and copy the zero-length file and avoid
# wasting time with multiple attempts to remote machine(s)
# (Note: Only do so if files don't already exist)

if [ $SENDCOM = YES ]; then
[ ! -s ${COMSP}syndata.tcvitals.$tmmark ] && \
cp /dev/null ${COMSP}syndata.tcvitals.$tmmark
[ ! -s ${COMSP}jtwc-fnoc.tcvitals.$tmmark ] && \
cp /dev/null ${COMSP}jtwc-fnoc.tcvitals.$tmmark
if [[ ! -s "${COM_OBS}/${RUN}.${cycle}.syndata.tcvitals.${tmmark}" ]]; then
cp "/dev/null" "${COM_OBS}/${RUN}.${cycle}.syndata.tcvitals.${tmmark}"
fi
if [[ ! -s ${COM_OBS}/${RUN}.${cycle}.jtwc-fnoc.tcvitals.${tmmark} ]]; then
cp "/dev/null" "${COM_OBS}/${RUN}.${cycle}.jtwc-fnoc.tcvitals.${tmmark}"
fi
fi

exit
Expand Down Expand Up @@ -375,15 +379,15 @@ fi


# This is the file that connects to the later RELOCATE and/or PREP scripts
[ $SENDCOM = YES ] && cp current ${COMSP}syndata.tcvitals.$tmmark
[ $SENDCOM = YES ] && cp current "${COM_OBS}/${RUN}.${cycle}.syndata.tcvitals.${tmmark}"

# Create the DBNet alert
if [ $SENDDBN = "YES" ]
then
$DBNROOT/bin/dbn_alert MODEL GDAS_TCVITALS $job ${COMSP}syndata.tcvitals.$tmmark
"${DBNROOT}/bin/dbn_alert" "MODEL" "GDAS_TCVITALS" "${job}" "${COM_OBS}/${RUN}.${cycle}.syndata.tcvitals.${tmmark}"
fi

# Write JTWC/FNOC Tcvitals to /com path since not saved anywhere else
[ $SENDCOM = YES ] && cp fnoc "${COMSP}jtwc-fnoc.tcvitals.${tmmark}"
[ $SENDCOM = YES ] && cp fnoc "${COM_OBS}/${RUN}.${cycle}.jtwc-fnoc.tcvitals.${tmmark}"

exit
Loading

0 comments on commit 881f237

Please sign in to comment.