From 6f2db6ccfa3b660a2707bb284755bcdb3ce0e929 Mon Sep 17 00:00:00 2001 From: "Kate.Friedman" Date: Wed, 28 Sep 2022 19:45:59 +0000 Subject: [PATCH 1/9] Retire config.prepbufr - The contents of config.prepbufr are only setting GESROOT; which is no longer used in the GFS in operations. - Don't need to set GESROOT so don't need config.prepbufr. - Delete config.prepbufr. - Remove "prepbufr" from config list in prep.sh. Refs #1033 --- jobs/rocoto/prep.sh | 2 +- parm/config/config.prepbufr | 19 ------------------- 2 files changed, 1 insertion(+), 20 deletions(-) delete mode 100755 parm/config/config.prepbufr diff --git a/jobs/rocoto/prep.sh b/jobs/rocoto/prep.sh index 7d22adc7aa..23edf2de42 100755 --- a/jobs/rocoto/prep.sh +++ b/jobs/rocoto/prep.sh @@ -10,7 +10,7 @@ status=$? ############################################################### # Source relevant configs -configs="base prep prepbufr" +configs="base prep" for config in $configs; do . $EXPDIR/config.${config} status=$? diff --git a/parm/config/config.prepbufr b/parm/config/config.prepbufr deleted file mode 100755 index 2d6ececc5b..0000000000 --- a/parm/config/config.prepbufr +++ /dev/null @@ -1,19 +0,0 @@ -#! /usr/bin/env bash - -########## config.prepbufr ########## -# PREPBUFR specific configuration - -echo "BEGIN: config.prepbufr" - -# Get task specific resources -. $EXPDIR/config.resources prepbufr - -# Set variables - -if [ $machine = "HERA" ]; then - export GESROOT=/scratch1/NCEPDEV/rstprod -elif [ $machine = "ORION" ]; then - export GESROOT=/dev/null -fi - -echo "END: config.prepbufr" From 97dba45ca2fc9d4f563bc5a6bc42f5a39412d895 Mon Sep 17 00:00:00 2001 From: "Kate.Friedman" Date: Wed, 28 Sep 2022 19:49:44 +0000 Subject: [PATCH 2/9] Update primary obsproc JJOB call in prep.sh The updated obsproc package renamed the primary global JJOB script and the HOMEobsproc variable is the only home path for the package, so update prep.sh to now call: $HOMEobsproc/jobs/JOBSPROC_GLOBAL_PREP instead of the old: $HOMEobsproc_network/jobs/JGLOBAL_PREP Refs #1033 --- jobs/rocoto/prep.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jobs/rocoto/prep.sh b/jobs/rocoto/prep.sh index 23edf2de42..f33a8bc9f6 100755 --- a/jobs/rocoto/prep.sh +++ b/jobs/rocoto/prep.sh @@ -109,7 +109,7 @@ if [ $DO_MAKEPREPBUFR = "YES" ]; then export MAKE_NSSTBUFR="NO" fi - $HOMEobsproc_network/jobs/JGLOBAL_PREP + $HOMEobsproc/jobs/JOBSPROC_GLOBAL_PREP status=$? [[ $status -ne 0 ]] && exit $status From 7fbd16651de0bb0968a94300d0fe0122aaaed45e Mon Sep 17 00:00:00 2001 From: "Kate.Friedman" Date: Wed, 28 Sep 2022 19:51:41 +0000 Subject: [PATCH 3/9] Add prepobs module load to R&D module_base modulefiles - Prepend MODULEPATH with local install of prepobs package on R&Ds. - Load prepobs/1.0.1 module from local install. The prepobs package is now a module that is loaded at runtime. Refs #1033 --- modulefiles/module_base.hera.lua | 3 +++ modulefiles/module_base.orion.lua | 3 +++ 2 files changed, 6 insertions(+) diff --git a/modulefiles/module_base.hera.lua b/modulefiles/module_base.hera.lua index 36cb672eb0..14c523c518 100644 --- a/modulefiles/module_base.hera.lua +++ b/modulefiles/module_base.hera.lua @@ -46,6 +46,9 @@ load(pathJoin("cdo", "1.9.5")) load(pathJoin("R", "3.5.0")) +prepend_path("MODULEPATH", pathJoin("/scratch1/NCEPDEV/global/glopara/git/prepobs/v1.0.1/modulefiles")) +load(pathJoin("prepobs", "1.0.1")) + -- Temporary until official hpc-stack is updated prepend_path("MODULEPATH", "/scratch2/NCEPDEV/ensemble/save/Walter.Kolczynski/hpc-stack/modulefiles/stack") load(pathJoin("hpc", "1.2.0")) diff --git a/modulefiles/module_base.orion.lua b/modulefiles/module_base.orion.lua index 22a4764419..bdb85d7430 100644 --- a/modulefiles/module_base.orion.lua +++ b/modulefiles/module_base.orion.lua @@ -47,6 +47,9 @@ load(pathJoin("slurm", "19.05.3-2")) load(pathJoin("cdo", "1.9.5")) +prepend_path("MODULEPATH", pathJoin("/work/noaa/global/glopara/git/prepobs/v1.0.1/modulefiles")) +load(pathJoin("prepobs", "1.0.1")) + -- Temporary until official hpc-stack is updated prepend_path("MODULEPATH", "/work2/noaa/global/wkolczyn/save/hpc-stack/modulefiles/stack") load(pathJoin("hpc", "1.2.0")) From 44c0a9f6e292b6b9e1f5af67cfc0f043b176ca66 Mon Sep 17 00:00:00 2001 From: "Kate.Friedman" Date: Wed, 28 Sep 2022 19:53:14 +0000 Subject: [PATCH 4/9] Update HOMEobsproc paths in config.base - Consolidate HOMEobsproc_prep, HOMEobsproc_network, and HOMEobsproc_global into a single HOMEobsproc path variable. - Set HOMEobsproc to local install location and current version 1.0.2 in config.base.emc.dyn. - Set HOMEobsproc to operational package location and current version 1.0.2 in config.base.nco.static. Refs #1033 --- parm/config/config.base.emc.dyn | 4 +--- parm/config/config.base.nco.static | 4 +--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/parm/config/config.base.emc.dyn b/parm/config/config.base.emc.dyn index 5063fa2ade..44487bb720 100755 --- a/parm/config/config.base.emc.dyn +++ b/parm/config/config.base.emc.dyn @@ -76,9 +76,7 @@ export MODE="@MODE@" # cycled/forecast-only export FIXgsi="$HOMEgfs/fix/fix_gsi" export HOMEfv3gfs="$HOMEgfs/sorc/fv3gfs.fd" export HOMEpost="$HOMEgfs" -export HOMEobsproc_prep="$BASE_GIT/obsproc/obsproc_prep.v5.5.0_hpc-stack" -export HOMEobsproc_network="$BASE_GIT/obsproc/obsproc_global.v3.4.2_hpc-stack" -export HOMEobsproc_global=$HOMEobsproc_network +export HOMEobsproc="$BASE_GIT/obsproc/v1.0.2" # CONVENIENT utility scripts and other environment parameters export NCP="/bin/cp -p" diff --git a/parm/config/config.base.nco.static b/parm/config/config.base.nco.static index a94f0be863..76784041c8 100755 --- a/parm/config/config.base.nco.static +++ b/parm/config/config.base.nco.static @@ -65,9 +65,7 @@ export REALTIME="YES" export FIXgsi="$HOMEgfs/fix/fix_gsi" export HOMEfv3gfs="$HOMEgfs/sorc/fv3gfs.fd" export HOMEpost="$HOMEgfs" -export HOMEobsproc_prep="$NWPROD/obsproc_prep.v5.5.0" -export HOMEobsproc_network="$NWPROD/obsproc_global.v3.4.2" -export HOMEobsproc_global=$HOMEobsproc_network +export HOMEobsproc="/lfs/h1/ops/prod/packages/obsproc.v1.0.2" # CONVENIENT utility scripts and other environment parameters export NCP="/bin/cp -p" From 428b438647e7e3fdf734de63e67a695fee508cef Mon Sep 17 00:00:00 2001 From: "kate.friedman" Date: Fri, 30 Sep 2022 09:16:36 -0500 Subject: [PATCH 5/9] Add cdate10 setting to config.prep The cdate10 variable is ${PDY}${cyc} and is used by the obsproc package to set CDATE10/CDATEMM. Refs #1033 --- parm/config/config.prep | 1 + 1 file changed, 1 insertion(+) diff --git a/parm/config/config.prep b/parm/config/config.prep index c4fe7c513e..4f5fce7333 100755 --- a/parm/config/config.prep +++ b/parm/config/config.prep @@ -9,6 +9,7 @@ echo "BEGIN: config.prep" . $EXPDIR/config.resources prep export DO_MAKEPREPBUFR="YES" # if NO, will copy prepbufr from globaldump +export cdate10=${PDY}${cyc} # Relocation and syndata QC export PROCESS_TROPCY=${PROCESS_TROPCY:-NO} From 684c426da9254cfe514bcdd73dd15787109726d5 Mon Sep 17 00:00:00 2001 From: "kate.friedman" Date: Fri, 30 Sep 2022 09:17:23 -0500 Subject: [PATCH 6/9] Add launcher_PREP to HERA and ORION env files Refs #1033 --- env/HERA.env | 1 + env/ORION.env | 1 + 2 files changed, 2 insertions(+) diff --git a/env/HERA.env b/env/HERA.env index 781cb9d415..954f25a322 100755 --- a/env/HERA.env +++ b/env/HERA.env @@ -43,6 +43,7 @@ if [ $step = "prep" -o $step = "prepbufr" ]; then export POE="NO" export BACK="NO" export sys_tp="HERA" + export launcher_PREP="srun" elif [ $step = "waveinit" -o $step = "waveprep" -o $step = "wavepostsbs" -o $step = "wavepostbndpnt" -o $step = "wavepostbndpntbll" -o $step = "wavepostpnt" ]; then diff --git a/env/ORION.env b/env/ORION.env index 63d978d2e5..04bae19e0e 100755 --- a/env/ORION.env +++ b/env/ORION.env @@ -42,6 +42,7 @@ if [ $step = "prep" -o $step = "prepbufr" ]; then export POE="NO" export BACK=${BACK:-"YES"} export sys_tp="ORION" + export launcher_PREP="srun" elif [ $step = "waveinit" -o $step = "waveprep" -o $step = "wavepostsbs" -o $step = "wavepostbndpnt" -o $step = "wavepostpnt" ]; then From 3f8c53bc3a84d7baa053034ff85f73ab363f813a Mon Sep 17 00:00:00 2001 From: "Kate.Friedman" Date: Mon, 3 Oct 2022 19:42:58 +0000 Subject: [PATCH 7/9] Retire prepbufr step from config.resources Refs #1033 --- parm/config/config.resources | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/parm/config/config.resources b/parm/config/config.resources index b723b57916..0162379ba6 100755 --- a/parm/config/config.resources +++ b/parm/config/config.resources @@ -41,12 +41,12 @@ elif [[ "$machine" = "ORION" ]]; then export npe_node_max=40 fi -if [ $step = "prep" -o $step = "prepbufr" ]; then - eval "export wtime_$step='00:45:00'" - eval "export npe_$step=4" - eval "export npe_node_$step=2" - eval "export nth_$step=1" - eval "export memory_$step=40G" +if [ $step = "prep" ]; then + export wtime_prep='00:45:00' + export npe_prep=4 + export npe_node_prep=2 + export nth_prep=1 + export memory_prep="40G" elif [ $step = "aerosol_init" ]; then export wtime_aerosol_init="00:05:00" From 24279f15d47d4855bd8e9f89e1aac4dec4a44fd7 Mon Sep 17 00:00:00 2001 From: "kate.friedman" Date: Tue, 4 Oct 2022 12:56:19 -0500 Subject: [PATCH 8/9] Add needed COMINtcvital path to config.prep - The prep job needs to know COMINtcvital so it can find the syndata.tcvitals file from the global dump archive in developer testing; uses compath.py in ops. - Add COMPONENT setting for use in COMINtcvital setting. Refs #1033 --- parm/config/config.prep | 2 ++ 1 file changed, 2 insertions(+) diff --git a/parm/config/config.prep b/parm/config/config.prep index de36fe2809..5889cd3aaa 100755 --- a/parm/config/config.prep +++ b/parm/config/config.prep @@ -18,6 +18,8 @@ export DO_RELOCATE="NO" export TROPCYQCRELOSH="$HOMEgfs/scripts/exglobal_atmos_tropcy_qc_reloc.sh" export SENDCOM=YES +export COMPONENT=${COMPONENT:-atmos} +export COMINtcvital=${COMINtcvital:-${DMPDIR}/${CDUMP}.${PDY}/${cyc}/${COMPONENT}} export COMINsyn=${COMINsyn:-$(compath.py ${envir}/com/gfs/${gfs_ver})/syndat} export HOMERELO=$HOMEgfs From cdeedc534057c20dca60190457395959db62140b Mon Sep 17 00:00:00 2001 From: "kate.friedman" Date: Fri, 7 Oct 2022 09:44:16 -0500 Subject: [PATCH 9/9] Remove COMPONENT use from config.prep - Remove COMPONENT setting. - Remove use of COMPONENT in setting COMINtcvital and hardcode "atmos" in path Refs #1033 --- parm/config/config.prep | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/parm/config/config.prep b/parm/config/config.prep index 5889cd3aaa..a300d544c8 100755 --- a/parm/config/config.prep +++ b/parm/config/config.prep @@ -18,8 +18,7 @@ export DO_RELOCATE="NO" export TROPCYQCRELOSH="$HOMEgfs/scripts/exglobal_atmos_tropcy_qc_reloc.sh" export SENDCOM=YES -export COMPONENT=${COMPONENT:-atmos} -export COMINtcvital=${COMINtcvital:-${DMPDIR}/${CDUMP}.${PDY}/${cyc}/${COMPONENT}} +export COMINtcvital=${COMINtcvital:-${DMPDIR}/${CDUMP}.${PDY}/${cyc}/atmos} export COMINsyn=${COMINsyn:-$(compath.py ${envir}/com/gfs/${gfs_ver})/syndat} export HOMERELO=$HOMEgfs