diff --git a/.gitignore b/.gitignore index 9bf41e1bac..6f5a7da9c5 100644 --- a/.gitignore +++ b/.gitignore @@ -82,6 +82,7 @@ sorc/gldas.fd sorc/gsi_enkf.fd sorc/gsi.fd sorc/enkf.fd +sorc/gdas.cd sorc/gsi_utils.fd sorc/gsi_monitor.fd sorc/ufs_utils.fd @@ -145,6 +146,12 @@ jobs/JGFS_ATMOS_WAFS_BLENDING_0P25 jobs/JGFS_ATMOS_WAFS_GCIP jobs/JGFS_ATMOS_WAFS_GRIB2 jobs/JGFS_ATMOS_WAFS_GRIB2_0P25 +jobs/JGDAS_GLOBAL_ATMOS_ANALYSIS_POST +jobs/JGDAS_GLOBAL_ATMOS_ANALYSIS_PREP +jobs/JGDAS_GLOBAL_ATMOS_ANALYSIS_RUN +jobs/JGDAS_GLOBAL_ATMOS_ENSANAL_POST +jobs/JGDAS_GLOBAL_ATMOS_ENSANAL_PREP +jobs/JGDAS_GLOBAL_ATMOS_ENSANAL_RUN # scripts symlinks scripts/exemcsfc_global_sfc_prep.sh scripts/exgdas_atmos_gldas.sh @@ -188,6 +195,7 @@ ush/radmon_verf_angle.sh ush/radmon_verf_bcoef.sh ush/radmon_verf_bcor.sh ush/radmon_verf_time.sh +ush/ufsda ush/wafs_blending.sh ush/wafs_grib2.regrid.sh ush/wafs_intdsk.sh diff --git a/env/HERA.env b/env/HERA.env index ef780a8a49..f5c0a5efcd 100755 --- a/env/HERA.env +++ b/env/HERA.env @@ -4,6 +4,7 @@ if [ $# -ne 1 ]; then echo "Must specify an input argument to set runtime environment variables!" echo "argument can be any one of the following:" + echo "atmanalrun atmensanalrun" echo "anal sfcanl fcst post vrfy metp" echo "eobs eupd ecen efcs epos" echo "postsnd awips gempak" @@ -50,6 +51,30 @@ elif [ $step = "waveinit" -o $step = "waveprep" -o $step = "wavepostsbs" -o $ste export wavempexec=${launcher} export wave_mpmd=${mpmd} +elif [ $step = "atmanalrun" ]; then + + export CFP_MP=${CFP_MP:-"YES"} + export USE_CFP=${USE_CFP:-"YES"} + export APRUNCFP="$launcher -n \$ncmd --multi-prog" + + nth_max=$(($npe_node_max / $npe_node_atmanalrun)) + + export NTHREADS_ATMANAL=${nth_atmanalrun:-$nth_max} + [[ $NTHREADS_ATMANAL -gt $nth_max ]] && export NTHREADS_ATMANAL=$nth_max + export APRUN_ATMANAL="$launcher -n $npe_atmanalrun" + +elif [ $step = "atmensanalrun" ]; then + + export CFP_MP=${CFP_MP:-"YES"} + export USE_CFP=${USE_CFP:-"YES"} + export APRUNCFP="$launcher -n \$ncmd --multi-prog" + + nth_max=$(($npe_node_max / $npe_node_atmensanalrun)) + + export NTHREADS_ATMENSANAL=${nth_atmensanalrun:-$nth_max} + [[ $NTHREADS_ATMENSANAL -gt $nth_max ]] && export NTHREADS_ATMENSANAL=$nth_max + export APRUN_ATMENSANAL="$launcher -n $npe_atmensanalrun" + elif [ $step = "anal" ]; then export MKL_NUM_THREADS=4 diff --git a/env/JET.env b/env/JET.env index 8d74aba5ad..21321b23c9 100755 --- a/env/JET.env +++ b/env/JET.env @@ -4,6 +4,7 @@ if [ $# -ne 1 ]; then echo "Must specify an input argument to set runtime environment variables!" echo "argument can be any one of the following:" + echo "atmanalrun atmensanalrun" echo "anal sfcanl fcst post vrfy metp" echo "eobs eupd ecen efcs epos" echo "postsnd awips gempak" @@ -44,6 +45,22 @@ elif [ $step = "waveinit" -o $step = "waveprep" -o $step = "wavepostsbs" -o $ste export wavempexec=${launcher} export wave_mpmd=${mpmd} +elif [ $step = "atmanalrun" ]; then + + nth_max=$(($npe_node_max / $npe_node_atmanalrun)) + + export NTHREADS_ATMANAL=${nth_atmanalrun:-$nth_max} + [[ $NTHREADS_ATMANAL -gt $nth_max ]] && export NTHREADS_ATMANAL=$nth_max + export APRUN_ATMANAL="$launcher $npe_atmanalrun" + +elif [ $step = "atmensanalrun" ]; then + + nth_max=$(($npe_node_max / $npe_node_atmensanalrun)) + + export NTHREADS_ATMENSANAL=${nth_atmensanalrun:-$nth_max} + [[ $NTHREADS_ATMENSANAL -gt $nth_max ]] && export NTHREADS_ATMENSANAL=$nth_max + export APRUN_ATMENSANAL="$launcher $npe_atmensanalrun" + elif [ $step = "anal" ]; then nth_max=$(($npe_node_max / $npe_node_anal)) diff --git a/env/ORION.env b/env/ORION.env index bef649c43f..f3b58d987c 100755 --- a/env/ORION.env +++ b/env/ORION.env @@ -4,6 +4,7 @@ if [ $# -ne 1 ]; then echo "Must specify an input argument to set runtime environment variables!" echo "argument can be any one of the following:" + echo "atmanalrun atmensanalrun" echo "anal sfcanl fcst post vrfy metp" echo "eobs eupd ecen efcs epos" echo "postsnd awips gempak" @@ -49,6 +50,30 @@ elif [ $step = "waveinit" -o $step = "waveprep" -o $step = "wavepostsbs" -o $ste export wavempexec=${launcher} export wave_mpmd=${mpmd} +elif [ $step = "atmanalrun" ]; then + + export CFP_MP=${CFP_MP:-"YES"} + export USE_CFP=${USE_CFP:-"YES"} + export APRUNCFP="$launcher -n \$ncmd --multi-prog" + + nth_max=$(($npe_node_max / $npe_node_atmanalrun)) + + export NTHREADS_ATMANAL=${nth_atmanalrun:-$nth_max} + [[ $NTHREADS_ATMANAL -gt $nth_max ]] && export NTHREADS_ATMANAL=$nth_max + export APRUN_ATMANAL="$launcher -n $npe_atmanalrun" + +elif [ $step = "atmensanalrun" ]; then + + export CFP_MP=${CFP_MP:-"YES"} + export USE_CFP=${USE_CFP:-"YES"} + export APRUNCFP="$launcher -n \$ncmd --multi-prog" + + nth_max=$(($npe_node_max / $npe_node_atmensanalrun)) + + export NTHREADS_ATMENSANAL=${nth_atmensanalrun:-$nth_max} + [[ $NTHREADS_ATMENSANAL -gt $nth_max ]] && export NTHREADS_ATMENSANAL=$nth_max + export APRUN_ATMENSANAL="$launcher -n $npe_atmensanalrun" + elif [ $step = "anal" ]; then export MKL_NUM_THREADS=4 diff --git a/jobs/rocoto/atmanalpost.sh b/jobs/rocoto/atmanalpost.sh new file mode 100755 index 0000000000..90a9b9bace --- /dev/null +++ b/jobs/rocoto/atmanalpost.sh @@ -0,0 +1,13 @@ +#!/bin/bash -x + +############################################################### +# Source FV3GFS workflow modules +. $HOMEgfs/ush/load_fv3gfs_modules.sh +status=$? +[[ $status -ne 0 ]] && exit $status + +############################################################### +# Execute the JJOB +$HOMEgfs/jobs/JGDAS_GLOBAL_ATMOS_ANALYSIS_POST +status=$? +exit $status diff --git a/jobs/rocoto/atmanalprep.sh b/jobs/rocoto/atmanalprep.sh new file mode 100755 index 0000000000..e4b76c8407 --- /dev/null +++ b/jobs/rocoto/atmanalprep.sh @@ -0,0 +1,13 @@ +#!/bin/bash -x + +############################################################### +# Source FV3GFS workflow modules +. $HOMEgfs/ush/load_fv3gfs_modules.sh +status=$? +[[ $status -ne 0 ]] && exit $status + +############################################################### +# Execute the JJOB +$HOMEgfs/jobs/JGDAS_GLOBAL_ATMOS_ANALYSIS_PREP +status=$? +exit $status diff --git a/jobs/rocoto/atmanalrun.sh b/jobs/rocoto/atmanalrun.sh new file mode 100755 index 0000000000..cebe478b7e --- /dev/null +++ b/jobs/rocoto/atmanalrun.sh @@ -0,0 +1,13 @@ +#!/bin/bash -x + +############################################################### +# Source FV3GFS workflow modules +. $HOMEgfs/ush/load_fv3gfs_modules.sh +status=$? +[[ $status -ne 0 ]] && exit $status + +############################################################### +# Execute the JJOB +$HOMEgfs/jobs/JGDAS_GLOBAL_ATMOS_ANALYSIS_RUN +status=$? +exit $status diff --git a/jobs/rocoto/atmensanalpost.sh b/jobs/rocoto/atmensanalpost.sh new file mode 100755 index 0000000000..ea6e490f20 --- /dev/null +++ b/jobs/rocoto/atmensanalpost.sh @@ -0,0 +1,13 @@ +#!/bin/bash -x + +############################################################### +# Source FV3GFS workflow modules +. $HOMEgfs/ush/load_fv3gfs_modules.sh +status=$? +[[ $status -ne 0 ]] && exit $status + +############################################################### +# Execute the JJOB +$HOMEgfs/jobs/JGDAS_GLOBAL_ATMOS_ENSANAL_POST +status=$? +exit $status diff --git a/jobs/rocoto/atmensanalprep.sh b/jobs/rocoto/atmensanalprep.sh new file mode 100755 index 0000000000..5ed434c6bf --- /dev/null +++ b/jobs/rocoto/atmensanalprep.sh @@ -0,0 +1,13 @@ +#!/bin/bash -x + +############################################################### +# Source FV3GFS workflow modules +. $HOMEgfs/ush/load_fv3gfs_modules.sh +status=$? +[[ $status -ne 0 ]] && exit $status + +############################################################### +# Execute the JJOB +$HOMEgfs/jobs/JGDAS_GLOBAL_ATMOS_ENSANAL_PREP +status=$? +exit $status diff --git a/jobs/rocoto/atmensanalrun.sh b/jobs/rocoto/atmensanalrun.sh new file mode 100755 index 0000000000..ddb3bb1432 --- /dev/null +++ b/jobs/rocoto/atmensanalrun.sh @@ -0,0 +1,13 @@ +#!/bin/bash -x + +############################################################### +# Source FV3GFS workflow modules +. $HOMEgfs/ush/load_fv3gfs_modules.sh +status=$? +[[ $status -ne 0 ]] && exit $status + +############################################################### +# Execute the JJOB +$HOMEgfs/jobs/JGDAS_GLOBAL_ATMOS_ENSANAL_RUN +status=$? +exit $status diff --git a/parm/config/config.atmanal b/parm/config/config.atmanal new file mode 100755 index 0000000000..9c11e9e593 --- /dev/null +++ b/parm/config/config.atmanal @@ -0,0 +1,24 @@ +#!/bin/bash -x + +########## config.atmanal ########## +# configuration common to all atm analysis tasks + +echo "BEGIN: config.atmanal" + +export OBS_YAML_DIR=$HOMEgfs/sorc/gdas.cd/parm/atm/obs/config/ +export OBS_LIST=$HOMEgfs/sorc/gdas.cd/parm/atm/obs/lists/gdas_prototype.yaml +export ATMVARYAML=$HOMEgfs/sorc/gdas.cd/parm/atm/variational/3dvar_dripcg.yaml +export BERROR_YAML=$HOMEgfs/sorc/gdas.cd/parm/atm/berror/hybvar_bump.yaml +export FV3JEDI_FIX=$HOMEgfs/fix/fix_jedi +export R2D2_OBS_DB='ufsda_test' +export R2D2_OBS_DUMP='oper_gdas' +export R2D2_OBS_SRC='ncdiag' +export R2D2_BC_SRC='gsi' +export R2D2_BC_DUMP='oper_gdas' +export R2D2_ARCH_DB='local' +export INTERP_METHOD='barycentric' + +export io_layout_x=1 +export io_layout_y=1 + +echo "END: config.atmanal" diff --git a/parm/config/config.atmanalpost b/parm/config/config.atmanalpost new file mode 100755 index 0000000000..fd5f3bbbcc --- /dev/null +++ b/parm/config/config.atmanalpost @@ -0,0 +1,10 @@ +#!/bin/bash -x + +########## config.atmanalpost ########## +# Post Atm Analysis specific + +echo "BEGIN: config.atmanalpost" + +# Get task specific resources +. $EXPDIR/config.resources atmanalpost +echo "END: config.atmanalpost" diff --git a/parm/config/config.atmanalprep b/parm/config/config.atmanalprep new file mode 100755 index 0000000000..0014520f5f --- /dev/null +++ b/parm/config/config.atmanalprep @@ -0,0 +1,10 @@ +#!/bin/bash -x + +########## config.atmanalprep ########## +# Pre Atm Analysis specific + +echo "BEGIN: config.atmanalprep" + +# Get task specific resources +. $EXPDIR/config.resources atmanalprep +echo "END: config.atmanalprep" diff --git a/parm/config/config.atmanalrun b/parm/config/config.atmanalrun new file mode 100755 index 0000000000..5aaac6a01d --- /dev/null +++ b/parm/config/config.atmanalrun @@ -0,0 +1,14 @@ +#!/bin/bash -x + +########## config.atmanalrun ########## +# Atm Analysis specific + +echo "BEGIN: config.atmanalrun" + +# Get task specific resources +. $EXPDIR/config.resources atmanalrun + +# Task specific variables +export JEDIVAREXE=$HOMEgfs/exec/fv3jedi_var.x + +echo "END: config.atmanalrun" diff --git a/parm/config/config.atmensanal b/parm/config/config.atmensanal new file mode 100755 index 0000000000..4f8e244b2e --- /dev/null +++ b/parm/config/config.atmensanal @@ -0,0 +1,25 @@ +#!/bin/bash -x + +########## config.atmensanal ########## +# configuration common to all atm atmensanal analysis tasks + +echo "BEGIN: config.atmensanal" + +export OBS_YAML_DIR=$HOMEgfs/sorc/gdas.cd/parm/atm/obs/config/ +export OBS_LIST=$HOMEgfs/sorc/gdas.cd/parm/atm/obs/lists/lgetkf_prototype.yaml +export BERROR_YAML=$HOMEgfs/sorc/gdas.cd/parm/atm/berror/hybvar_bump.yaml +export ATMENSYAML=$HOMEgfs/sorc/gdas.cd/parm/atm/lgetkf/lgetkf.yaml +export FV3JEDI_FIX=$HOMEgfs/fix/fix_jedi +export R2D2_OBS_DB='ufsda_test' +export R2D2_OBS_DUMP='oper_gdas' +export R2D2_OBS_SRC='ncdiag' +export R2D2_BC_SRC='gsi' +##export R2D2_BC_DUMP='oper_gdas' +export R2D2_BC_DUMP='prgdasens' +export R2D2_ARCH_DB='local' +export INTERP_METHOD='barycentric' + +export io_layout_x=1 # hardwired to 1,1 in yamltools.py +export io_layout_y=1 + +echo "END: config.atmensanal" diff --git a/parm/config/config.atmensanalpost b/parm/config/config.atmensanalpost new file mode 100755 index 0000000000..f79ee5b507 --- /dev/null +++ b/parm/config/config.atmensanalpost @@ -0,0 +1,10 @@ +#!/bin/bash -x + +########## config.atmensanalpost ########## +# Post Atm Analysis specific + +echo "BEGIN: config.atmensanalpost" + +# Get task specific resources +. $EXPDIR/config.resources atmensanalpost +echo "END: config.atmensanalpost" diff --git a/parm/config/config.atmensanalprep b/parm/config/config.atmensanalprep new file mode 100755 index 0000000000..b719b9ac6c --- /dev/null +++ b/parm/config/config.atmensanalprep @@ -0,0 +1,10 @@ +#!/bin/bash -x + +########## config.atmensanalprep ########## +# Pre Atm Analysis specific + +echo "BEGIN: config.atmensanalprep" + +# Get task specific resources +. $EXPDIR/config.resources atmensanalprep +echo "END: config.atmensanalprep" diff --git a/parm/config/config.atmensanalrun b/parm/config/config.atmensanalrun new file mode 100755 index 0000000000..aeb59d1805 --- /dev/null +++ b/parm/config/config.atmensanalrun @@ -0,0 +1,14 @@ +#!/bin/bash -x + +########## config.atmensanalrun ########## +# Atm LETKFs specific + +echo "BEGIN: config.atmensanalrun" + +# Get task specific resources +. $EXPDIR/config.resources atmensanalrun + +# Task specific variables +export JEDIENSEXE=$HOMEgfs/exec/fv3jedi_letkf.x + +echo "END: config.atmensanalrun" diff --git a/parm/config/config.base.emc.dyn b/parm/config/config.base.emc.dyn index c48260abdd..722818b22a 100755 --- a/parm/config/config.base.emc.dyn +++ b/parm/config/config.base.emc.dyn @@ -320,6 +320,10 @@ fi export imp_physics=8 # Shared parameters +# DA engine +export DO_JEDIVAR="NO" +export DO_JEDIENS="NO" + # Hybrid related export DOHYBVAR="YES" export NMEM_ENKF=@NMEM_ENKF@ diff --git a/parm/config/config.base.nco.static b/parm/config/config.base.nco.static index 4980ecf752..48db9dd020 100755 --- a/parm/config/config.base.nco.static +++ b/parm/config/config.base.nco.static @@ -215,6 +215,10 @@ export DOBNDPNT_WAVE="YES" export imp_physics=8 # Shared parameters +# DA engine +export DO_JEDIVAR="NO" +export DO_JEDIENS="NO" + # Hybrid related export DOHYBVAR="YES" export NMEM_ENKF="80" diff --git a/parm/config/config.resources b/parm/config/config.resources index 47f9ee5bca..f9b3e4132a 100755 --- a/parm/config/config.resources +++ b/parm/config/config.resources @@ -9,6 +9,8 @@ if [ $# -ne 1 ]; then echo "Must specify an input task argument to set resource variables!" echo "argument can be any one of the following:" echo "getic init coupled_ic aerosol_init" + echo "atmanalprep atmanalrun atmanalpost" + echo "atmensanalprep atmensanalrun atmensanalpost" echo "anal sfcanl analcalc analdiag gldas fcst post vrfy metp arch echgres" echo "eobs ediag eomg eupd ecen esfc efcs epos earc" echo "init_chem mom6ic ocnpost" @@ -136,6 +138,35 @@ elif [ $step = "waveawipsgridded" ]; then export npe_node_waveawipsgridded=$(echo "$npe_node_max / $nth_waveawipsgridded" | bc) export NTASKS=${npe_waveawipsgridded} +elif [ $step = "atmanalprep" ]; then + + export wtime_atmanalprep="00:10:00" + export npe_atmanalprep=1 + export nth_atmanalprep=1 + export npe_node_atmanalprep=$(echo "$npe_node_max / $nth_atmanalprep" | bc) + export memory_atmanalprep="3072M" + +elif [ $step = "atmanalrun" ]; then + + # make below case dependent later + export layout_x=1 + export layout_y=1 + + export wtime_atmanalrun="00:30:00" + export npe_atmanalrun=$(echo "$layout_x * $layout_y * 6" | bc) + export npe_atmanalrun_gfs=$(echo "$layout_x * $layout_y * 6" | bc) + export nth_atmanalrun=1 + export nth_atmanalrun_gfs=$nth_atmanalrun + export native_atmanalrun="--exclusive" + export npe_node_atmanalrun=$(echo "$npe_node_max / $nth_atmanalrun" | bc) + +elif [ $step = "atmanalpost" ]; then + + export wtime_atmanalpost="00:30:00" + export npe_atmanalpost=$npe_node_max + export nth_atmanalpost=1 + export npe_node_atmanalpost=$(echo "$npe_node_max / $nth_atmanalpost" | bc) + elif [ $step = "anal" ]; then export wtime_anal="01:00:00" @@ -426,6 +457,34 @@ elif [ $step = "coupled_ic" ]; then export npe_node_coupled_ic=1 export nth_coupled_ic=1 +elif [ $step = "atmensanalprep" ]; then + + export wtime_atmensanalprep="00:10:00" + export npe_atmensanalprep=1 + export nth_atmensanalprep=1 + export npe_node_atmensanalprep=$(echo "$npe_node_max / $nth_atmensanalprep" | bc) + +elif [ $step = "atmensanalrun" ]; then + + # make below case dependent later + export layout_x=2 + export layout_y=3 + + export wtime_atmensanalrun="00:30:00" + export npe_atmensanalrun=$(echo "$layout_x * $layout_y * 6" | bc) + export npe_atmensanalrun_gfs=$(echo "$layout_x * $layout_y * 6" | bc) + export nth_atmensanalrun=1 + export nth_atmensanalrun_gfs=$nth_atmensanalrun + export native_atmensanalrun="--exclusive" + export npe_node_atmensanalrun=$(echo "$npe_node_max / $nth_atmensanalrun" | bc) + +elif [ $step = "atmensanalpost" ]; then + + export wtime_atmensanalpost="00:30:00" + export npe_atmensanalpost=$npe_node_max + export nth_atmensanalpost=1 + export npe_node_atmensanalpost=$(echo "$npe_node_max / $nth_atmensanalpost" | bc) + elif [ $step = "eobs" -o $step = "eomg" ]; then export wtime_eobs="00:45:00" diff --git a/sorc/build_all.sh b/sorc/build_all.sh index 956cc1400a..10f8b630c9 100755 --- a/sorc/build_all.sh +++ b/sorc/build_all.sh @@ -147,6 +147,20 @@ $Build_gsi_utils && { ((err+=$rc)) } +#------------------------------------ +# build gdas +#------------------------------------ +$Build_gdas && { + echo " .... Building GDASApp .... " + ./build_gdas.sh > $logs_dir/build_gdas.log 2>&1 + rc=$? + if [[ $rc -ne 0 ]] ; then + echo "Fatal error in building GDAS." + echo "The log file is in $logs_dir/build_gdas.log" + fi + ((err+=$rc)) +} + #------------------------------------ # build gsi monitor #------------------------------------ diff --git a/sorc/build_gdas.sh b/sorc/build_gdas.sh new file mode 100755 index 0000000000..f9238c9ab0 --- /dev/null +++ b/sorc/build_gdas.sh @@ -0,0 +1,26 @@ +#! /usr/bin/env bash +set -eux + +source ./machine-setup.sh > /dev/null 2>&1 +cwd=$(pwd) + +export BUILD_TARGET=$target + +# use more build jobs if on NOAA HPC +build_jobs=4 +case "${target}" in + hera|orion) + build_jobs=10 + ;; +esac + +# Check final exec folder exists +if [ ! -d "../exec" ]; then + mkdir ../exec +fi + +cd gdas.cd +BUILD_JOBS=$build_jobs ./build.sh -t $BUILD_TARGET + +exit + diff --git a/sorc/checkout.sh b/sorc/checkout.sh index 978d932ba7..9e1caf9022 100755 --- a/sorc/checkout.sh +++ b/sorc/checkout.sh @@ -134,6 +134,7 @@ checkout "ufs_model.fd" "https://github.com/ufs-community/ufs-weather-model" checkout "gsi_enkf.fd" "https://github.com/NOAA-EMC/GSI.git" "67f5ab4" ; errs=$((errs + $?)) checkout "gsi_utils.fd" "https://github.com/NOAA-EMC/GSI-Utils.git" "322cc7b" ; errs=$((errs + $?)) checkout "gsi_monitor.fd" "https://github.com/NOAA-EMC/GSI-Monitor.git" "acf8870" ; errs=$((errs + $?)) +checkout "gdas.cd" "https://github.com/NOAA-EMC/GDASApp.git" "5952c9d" ; errs=$((errs + $?)) checkout "gldas.fd" "https://github.com/NOAA-EMC/GLDAS.git" "fd8ba62" ; errs=$((errs + $?)) checkout "ufs_utils.fd" "https://github.com/ufs-community/UFS_UTILS.git" "a2b0817" ; errs=$((errs + $?)) checkout "verif-global.fd" "https://github.com/NOAA-EMC/EMC_verif-global.git" "c267780" ; errs=$((errs + $?)) diff --git a/sorc/link_workflow.sh b/sorc/link_workflow.sh index a34643232a..22e9d99901 100755 --- a/sorc/link_workflow.sh +++ b/sorc/link_workflow.sh @@ -66,7 +66,8 @@ for dir in fix_aer \ fix_cpl \ fix_wave \ fix_reg2grb2 \ - fix_ugwd + fix_ugwd \ + fix_jedi do if [ -d $dir ]; then [[ $RUN_ENVIR = nco ]] && chmod -R 755 $dir @@ -141,6 +142,20 @@ cd ${pwd}/../fix ||exit 8 $LINK ../sorc/gsi_enkf.fd/fix fix_gsi +#------------------------------ +#--add GDASApp files +#------------------------------ +cd ${pwd}/../jobs ||exit 8 + $LINK ../sorc/gdas.cd/jobs/JGDAS_GLOBAL_ATMOS_ANALYSIS_PREP . + $LINK ../sorc/gdas.cd/jobs/JGDAS_GLOBAL_ATMOS_ANALYSIS_RUN . + $LINK ../sorc/gdas.cd/jobs/JGDAS_GLOBAL_ATMOS_ANALYSIS_POST . + $LINK ../sorc/gdas.cd/jobs/JGDAS_GLOBAL_ATMOS_ENSANAL_PREP . + $LINK ../sorc/gdas.cd/jobs/JGDAS_GLOBAL_ATMOS_ENSANAL_RUN . + $LINK ../sorc/gdas.cd/jobs/JGDAS_GLOBAL_ATMOS_ENSANAL_POST . +cd ${pwd}/../ush ||exit 8 + $LINK ../sorc/gdas.cd/ush/ufsda . + + #------------------------------ #--add DA Monitor file (NOTE: ensure to use correct version) #------------------------------ @@ -254,6 +269,15 @@ for gldasexe in gdas2gldas gldas2gdas gldas_forcing gldas_model gldas_post $LINK ../sorc/gldas.fd/exec/$gldasexe . done +# GDASApp +for gdasexe in fv3jedi_addincrement.x fv3jedi_diffstates.x fv3jedi_ensvariance.x fv3jedi_hofx.x \ + fv3jedi_var.x fv3jedi_convertincrement.x fv3jedi_dirac.x fv3jedi_error_covariance_training.x \ + fv3jedi_letkf.x fv3jedi_convertstate.x fv3jedi_eda.x fv3jedi_forecast.x fv3jedi_plot_field.x \ + fv3jedi_data_checker.py fv3jedi_enshofx.x fv3jedi_hofx_nomodel.x fv3jedi_testdata_downloader.py; do + [[ -s $gdasexe ]] && rm -f $gdasexe + $LINK ../sorc/gdas.cd/build/bin/$gdasexe . +done + #------------------------------ #--link source code directories #------------------------------ diff --git a/workflow/applications.py b/workflow/applications.py index c53f3635ea..8256cd9699 100644 --- a/workflow/applications.py +++ b/workflow/applications.py @@ -105,6 +105,8 @@ def __init__(self, configuration: Configuration) -> None: self.do_wafs = _base.get('DO_WAFS', False) self.do_vrfy = _base.get('DO_VRFY', True) self.do_metp = _base.get('DO_METP', False) + self.do_jedivar = _base.get('DO_JEDIVAR', False) + self.do_jediens = _base.get('DO_JEDIENS', False) self.do_hpssarch = _base.get('HPSSARCH', False) @@ -170,15 +172,25 @@ def _cycled_configs(self): Returns the config_files that are involved in the cycled app """ - configs = ['prep', - 'anal', 'sfcanl', 'analdiag', 'analcalc', - 'fcst', 'post', 'vrfy', 'arch'] + configs = ['prep'] + if self.do_jedivar: + configs += ['atmanalprep', 'atmanalrun', 'atmanalpost'] + else: + configs += ['anal', 'analdiag'] + + configs += ['sfcanl', 'analcalc', 'fcst', 'post', 'vrfy', 'arch'] + + if self.do_gldas: configs += ['gldas'] if self.do_hybvar: - configs += ['eobs', 'eomg', 'ediag', 'eupd', 'ecen', 'esfc', 'efcs', 'echgres', 'epos', 'earc'] + if self.do_jediens: + configs += ['atmensanalprep', 'atmensanalrun', 'atmensanalpost'] + else: + configs += ['eobs', 'eomg', 'ediag', 'eupd'] + configs += ['ecen', 'esfc', 'efcs', 'echgres', 'epos', 'earc'] if self.do_metp: configs += ['metp'] @@ -319,10 +331,17 @@ def _get_cycled_task_names(self): This is the place where that order is set. """ - gdas_gfs_common_tasks_before_fcst = ['prep', 'anal', 'sfcanl', 'analcalc'] + gdas_gfs_common_tasks_before_fcst = ['prep'] gdas_gfs_common_tasks_after_fcst = ['post', 'vrfy'] gdas_gfs_common_cleanup_tasks = ['arch'] + if self.do_jedivar: + gdas_gfs_common_tasks_before_fcst += ['atmanalprep', 'atmanalrun', 'atmanalpost'] + else: + gdas_gfs_common_tasks_before_fcst += ['anal'] + + gdas_gfs_common_tasks_before_fcst += ['sfcanl', 'analcalc'] + gldas_tasks = ['gldas'] wave_prep_tasks = ['waveinit', 'waveprep'] wave_bndpnt_tasks = ['wavepostbndpnt', 'wavepostbndpntbll'] @@ -331,12 +350,17 @@ def _get_cycled_task_names(self): hybrid_gdas_or_gfs_tasks = [] hybrid_gdas_tasks = [] if self.do_hybvar: - hybrid_gdas_or_gfs_tasks += ['eobs', 'eupd', 'echgres'] - hybrid_gdas_or_gfs_tasks += ['ediag'] if self.lobsdiag_forenkf else ['eomg'] + if self.do_jediens: + hybrid_gdas_or_gfs_tasks += ['atmensanalprep', 'atmensanalrun', 'atmensanalpost', 'echgres'] + else: + hybrid_gdas_or_gfs_tasks += ['eobs', 'eupd', 'echgres'] + hybrid_gdas_or_gfs_tasks += ['ediag'] if self.lobsdiag_forenkf else ['eomg'] hybrid_gdas_tasks += ['ecen', 'esfc', 'efcs', 'epos', 'earc'] # Collect all "gdas" cycle tasks - gdas_tasks = gdas_gfs_common_tasks_before_fcst + ['analdiag'] + gdas_tasks = gdas_gfs_common_tasks_before_fcst.copy() + if not self.do_jedivar: + gdas_tasks += ['analdiag'] if self.do_gldas: gdas_tasks += gldas_tasks diff --git a/workflow/rocoto/workflow_tasks.py b/workflow/rocoto/workflow_tasks.py index 8e356a2a52..ea90ee37aa 100644 --- a/workflow/rocoto/workflow_tasks.py +++ b/workflow/rocoto/workflow_tasks.py @@ -12,8 +12,10 @@ class Tasks: SERVICE_TASKS = ['arch', 'earc', 'getic'] VALID_TASKS = ['aerosol_init', 'coupled_ic', 'getic', 'init', 'prep', 'anal', 'sfcanl', 'analcalc', 'analdiag', 'gldas', 'arch', + 'atmanalprep', 'atmanalrun', 'atmanalpost', 'earc', 'ecen', 'echgres', 'ediag', 'efcs', 'eobs', 'eomg', 'epos', 'esfc', 'eupd', + 'atmensanalprep', 'atmensanalrun', 'atmensanalpost', 'fcst', 'post', 'ocnpost', 'vrfy', 'metp', 'postsnd', 'awips', 'gempak', 'wafs', 'wafsblending', 'wafsblending0p25', @@ -336,7 +338,10 @@ def anal(self): def sfcanl(self): deps = [] - dep_dict = {'type': 'task', 'name': f'{self.cdump}anal'} + if self.app_config.do_jedivar: + dep_dict = {'type': 'task', 'name': f'{self.cdump}atmanalrun'} + else: + dep_dict = {'type': 'task', 'name': f'{self.cdump}anal'} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep=deps) @@ -348,7 +353,10 @@ def sfcanl(self): def analcalc(self): deps = [] - dep_dict = {'type': 'task', 'name': f'{self.cdump}anal'} + if self.app_config.do_jedivar: + dep_dict = {'type': 'task', 'name': f'{self.cdump}atmanalrun'} + else: + dep_dict = {'type': 'task', 'name': f'{self.cdump}anal'} deps.append(rocoto.add_dependency(dep_dict)) dep_dict = {'type': 'task', 'name': f'{self.cdump}sfcanl'} deps.append(rocoto.add_dependency(dep_dict)) @@ -376,6 +384,65 @@ def analdiag(self): return task + def atmanalprep(self): + + suffix = self._base["SUFFIX"] + dump_suffix = self._base["DUMP_SUFFIX"] + gfs_cyc = self._base["gfs_cyc"] + dmpdir = self._base["DMPDIR"] + do_gfs_enkf = True if self.app_config.do_hybvar and 'gfs' in self.app_config.eupd_cdumps else False + + deps = [] + dep_dict = {'type': 'metatask', 'name': 'gdaspost', 'offset': '-06:00:00'} + deps.append(rocoto.add_dependency(dep_dict)) + data = f'&ROTDIR;/gdas.@Y@m@d/@H/atmos/gdas.t@Hz.atmf009{suffix}' + dep_dict = {'type': 'data', 'data': data, 'offset': '-06:00:00'} + deps.append(rocoto.add_dependency(dep_dict)) + data = f'{dmpdir}/{self.cdump}{dump_suffix}.@Y@m@d/@H/{self.cdump}.t@Hz.updated.status.tm00.bufr_d' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) + + cycledef = self.cdump + if self.cdump in ['gfs'] and do_gfs_enkf and gfs_cyc != 4: + cycledef = 'gdas' + + resources = self.get_resource('atmanalprep') + task = create_wf_task('atmanalprep', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies, + cycledef=cycledef) + return task + + def atmanalrun(self): + + deps = [] + dep_dict = {'type': 'task', 'name': f'{self.cdump}atmanalprep'} + deps.append(rocoto.add_dependency(dep_dict)) + if self.app_config.do_hybvar: + dep_dict = {'type': 'metatask', 'name': 'gdasepmn', 'offset': '-06:00:00'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) + else: + dependencies = rocoto.create_dependency(dep=deps) + + resources = self.get_resource('atmanalrun') + task = create_wf_task('atmanalrun', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + + return task + + def atmanalpost(self): + + deps = [] + dep_dict = {'type': 'task', 'name': f'{self.cdump}atmanalrun'} + deps.append(rocoto.add_dependency(dep_dict)) + dep_dict = {'type': 'cycleexist', 'offset': '-06:00:00'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) + + resources = self.get_resource('atmanalpost') + task = create_wf_task('atmanalpost', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + + return task + def gldas(self): deps = [] @@ -880,6 +947,61 @@ def eupd(self): return task + def atmensanalprep(self): + + suffix = self._base["SUFFIX"] + dump_suffix = self._base["DUMP_SUFFIX"] + gfs_cyc = self._base["gfs_cyc"] + dmpdir = self._base["DMPDIR"] + do_gfs_enkf = True if self.app_config.do_hybvar and 'gfs' in self.app_config.eupd_cdumps else False + + deps = [] + dep_dict = {'type': 'metatask', 'name': 'gdaspost', 'offset': '-06:00:00'} + deps.append(rocoto.add_dependency(dep_dict)) + data = f'&ROTDIR;/gdas.@Y@m@d/@H/atmos/gdas.t@Hz.atmf009{suffix}' + dep_dict = {'type': 'data', 'data': data, 'offset': '-06:00:00'} + deps.append(rocoto.add_dependency(dep_dict)) + data = f'{dmpdir}/{self.cdump}{dump_suffix}.@Y@m@d/@H/{self.cdump}.t@Hz.updated.status.tm00.bufr_d' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) + + cycledef = self.cdump + if self.cdump in ['gfs'] and do_gfs_enkf and gfs_cyc != 4: + cycledef = 'gdas' + + resources = self.get_resource('atmensanalprep') + task = create_wf_task('atmensanalprep', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies, + cycledef=cycledef) + + return task + + def atmensanalrun(self): + + deps = [] + dep_dict = {'type': 'task', 'name': f'{self.cdump}atmensanalprep'} + deps.append(rocoto.add_dependency(dep_dict)) + dep_dict = {'type': 'metatask', 'name': 'gdasepmn', 'offset': '-06:00:00'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) + + resources = self.get_resource('atmensanalrun') + task = create_wf_task('atmensanalrun', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + + return task + + def atmensanalpost(self): + + deps = [] + dep_dict = {'type': 'task', 'name': f'{self.cdump}atmensanalrun'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep=deps) + + resources = self.get_resource('atmensanalpost') + task = create_wf_task('atmensanalpost', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + + return task + def ecen(self): self._is_this_a_gdas_task(self.cdump, 'ecen') @@ -912,7 +1034,10 @@ def _get_ecengroups(): deps = [] dep_dict = {'type': 'task', 'name': f'{self.cdump}analcalc'} deps.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'task', 'name': f'{eupd_cdump}eupd'} + if self.app_config.do_jediens: + dep_dict = {'type': 'task', 'name': f'{eupd_cdump}atmensanalrun'} + else: + dep_dict = {'type': 'task', 'name': f'{eupd_cdump}eupd'} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) @@ -940,7 +1065,10 @@ def esfc(self): deps = [] dep_dict = {'type': 'task', 'name': f'{self.cdump}analcalc'} deps.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'task', 'name': f'{eupd_cdump}eupd'} + if self.app_config.do_jediens: + dep_dict = {'type': 'task', 'name': f'{eupd_cdump}atmensanalrun'} + else: + dep_dict = {'type': 'task', 'name': f'{eupd_cdump}eupd'} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps)