Merge pull request #807 from NOAA-EMC/dev_v16

Sync merge dev_v16 updates into GFSv16.3.0 release branch

Externals.cfg

@@ -36,7 +36,7 @@ protocol = git
 required = True
 
 [EMC_verif-global]
-tag = verif_global_v1.11.0
+tag = verif_global_v2.9.0
 local_path = sorc/verif-global.fd
 repo_url = https://github.com/NOAA-EMC/EMC_verif-global.git
 protocol = git

docs/Release_Notes.gfs.v16.2.0.md

@@ -15,7 +15,7 @@ The NOAA VLab and both the NOAA-EMC and NCAR organization spaces on GitHub.com a
 cd $PACKAGEROOT
 mkdir gfs.v16.2.0
 cd gfs.v16.2.0
-git clone -b EMC-v16.2.0.6 https://github.com/NOAA-EMC/global-workflow.git .
+git clone -b EMC-v16.2.0.7 https://github.com/NOAA-EMC/global-workflow.git .
 cd sorc
 ./checkout.sh -o
 ```
@@ -118,7 +118,7 @@ All components updated their codes to build on WCOSS2:
 FIX CHANGES
 -----------
 
-* No changes from GFS v16.1.7
+* No changes from GFS v16.1.8
 
 PARM/CONFIG CHANGES
 -------------------
@@ -354,7 +354,7 @@ CHANGES TO RESOURCES AND FILE SIZES
 -----------------------------------
 
 * File sizes
-  * No change to GFSv16.1.7.
+  * No change to GFSv16.1.8.
 * Resource changes to meet operational time windows:
   * See updated Ecflow scripts for adjusted compute resources for WCOSS2.
   * Pre-hand-off development testing results:
@@ -379,21 +379,21 @@ DISSEMINATION INFORMATION
 -------------------------
 
 * Where should this output be sent?
-  * No change from GFS v16.1.7
+  * No change from GFS v16.1.8
 * Who are the users?
-  * No change from GFS v16.1.7
+  * No change from GFS v16.1.8
 * Which output files should be transferred from PROD WCOSS to DEV WCOSS?
-  * No change from GFS v16.1.7
+  * No change from GFS v16.1.8
 * Directory changes
-  * No change from GFS v16.1.7
+  * No change from GFS v16.1.8
 * File changes
-  * No change from GFS v16.1.7
+  * No change from GFS v16.1.8
 
 HPSS ARCHIVE
 ------------
 
-* No change from GFS v16.1.7
+* No change from GFS v16.1.8
 
 JOB DEPENDENCIES AND FLOW DIAGRAM
 ---------------------------------
-* No change from GFS v16.1.7
+* No change from GFS v16.1.8

ecf/defs/GFSApp.def

@@ -0,0 +1,66 @@
+#### totality only
+extern /task_limit:TOTALITY
+
+suite fcstonly
+  inlimit /task_limit:TOTALITY
+  edit PSLOT 'ecffo_c192'
+  edit PDY '20200906'
+  edit EDATE '20200907'
+  edit ECF_INCLUDE '/scratch1/NCEPDEV/global/Lin.Gan/git/global-workflow-develop/ecf/include'
+  edit ENVIR 'para'
+  edit MACHINE_SITE 'development'
+  edit CYC '00'
+  edit QUEUE 'batch'
+  edit QUEUE_SERVICE 'service'
+  edit PROJENVIR 'DEV'
+  edit HERA_ACCOUNT 'fv3-cpu'
+  edit EMC_USER 'Lin.Gan'
+  edit gfs_ver 'v16.2.0'
+  edit PACKAGEHOME '/scratch1/NCEPDEV/global/%EMC_USER%/para/packages/gfs.%gfs_ver%'
+  edit ECF_FILES '%PACKAGEHOME%/ecf/scripts/gfs'
+  edit PROJ 'GFS'
+  family GFSApp
+    edit NET 'gfs'
+    family gfs
+      edit RUN 'gfs'
+      family atmos
+        task jgfs_getic
+        task jgfs_forecast
+          trigger ./atmos/jgfs_getic == complete 
+        family post
+          task jgfs_atmos_post_manager
+            trigger ../../jgfs_forecast == active
+            event 1 release_post000
+            event 2 release_post001
+            event 3 release_post002
+            event 4 release_post003
+          task jgfs_atmos_post_f000
+            trigger ./jgfs_atmos_post_manager:release_post000
+            edit FHRGRP '001'
+            edit FHRLST 'f000'
+            edit FHR 'f000'
+            edit HR '000'
+          task jgfs_atmos_post_f001
+            trigger ./jgfs_atmos_post_manager:release_post001
+            edit FHRGRP '002'
+            edit FHRLST 'f001'
+            edit FHR 'f001'
+            edit HR '001'
+          task jgfs_atmos_post_f002
+            trigger ./jgfs_atmos_post_manager:release_post002
+            edit FHRGRP '003'
+            edit FHRLST 'f002'
+            edit FHR 'f002'
+            edit HR '002'
+          task jgfs_atmos_post_f003
+            trigger ./jgfs_atmos_post_manager:release_post003
+            edit FHRGRP '004'
+            edit FHRLST 'f003'
+            edit FHR 'f003'
+            edit HR '003'
+        endfamily # post
+      endfamily # atmos
+    endfamily # gfs
+  endfamily # GFSApp
+endsuite
+

ecf/include/envir-p1.h

@@ -15,3 +15,17 @@ fi
 export DBNROOT=$SIPHONROOT
 
 if [[ ! " prod para test " =~ " ${envir} " && " ops.prod ops.para " =~ " $(whoami) " ]]; then err_exit "ENVIR must be prod, para, or test [envir-p1.h]"; fi
+
+#### Developer Overwrite
+####  For script level COM path assignment - compath.py gfs/v16.2
+####  Result in path: ${PTMP}/${USER}/${PSLOT}/para/com/gfs/v16.2
+PTMP=/lfs/h2/emc/ptmp
+PSLOT=ecfops
+export COMROOT=${PTMP}/${USER}/${PSLOT}/para/com
+export COMPATH=${PTMP}/${USER}/${PSLOT}/para/com/gfs
+export ROTDIR="$(compath.py gfs/${gfs_ver})"
+
+if [ -n "%PDY:%" ]; then
+  export PDY=${PDY:-%PDY:%}
+  export CDATE=${PDY}%CYC:%
+fi

ecf/scripts/enkfgdas/analysis/create/jenkfgdas_select_obs.ecf

@@ -5,7 +5,7 @@
 #PBS -A %PROJ%-%PROJENVIR%
 #PBS -l walltime=00:10:00
 #PBS -l select=12:mpiprocs=40:ompthreads=3:ncpus=120
-#PBS -l place=vscatter:excl
+#PBS -l place=vscatter:exclhost
 #PBS -l debug=true
 
 model=gfs

ecf/scripts/enkfgdas/analysis/create/jenkfgdas_update.ecf

@@ -5,7 +5,7 @@
 #PBS -A %PROJ%-%PROJENVIR%
 #PBS -l walltime=00:30:00
 #PBS -l select=35:mpiprocs=9:ompthreads=14:ncpus=126
-#PBS -l place=vscatter:excl
+#PBS -l place=vscatter:exclhost
 #PBS -l debug=true
 
 model=gfs

ecf/scripts/enkfgdas/analysis/recenter/ecen/jenkfgdas_ecen.ecf

@@ -5,7 +5,7 @@
 #PBS -A %PROJ%-%PROJENVIR%
 #PBS -l walltime=00:10:00
 #PBS -l select=3:mpiprocs=32:ompthreads=4:ncpus=128
-#PBS -l place=vscatter:excl
+#PBS -l place=vscatter:exclhost
 #PBS -l debug=true
 
 model=gfs

ecf/scripts/enkfgdas/forecast/jenkfgdas_fcst.ecf

@@ -5,7 +5,7 @@
 #PBS -A %PROJ%-%PROJENVIR%
 #PBS -l walltime=00:40:00
 #PBS -l select=4:mpiprocs=128:ompthreads=1:ncpus=128
-#PBS -l place=vscatter:excl
+#PBS -l place=vscatter:exclhost
 #PBS -l debug=true
 
 model=gfs

ecf/scripts/enkfgdas/post/jenkfgdas_post_master.ecf

@@ -5,7 +5,7 @@
 #PBS -A %PROJ%-%PROJENVIR%
 #PBS -l walltime=00:15:00
 #PBS -l select=3:mpiprocs=32:ompthreads=4:ncpus=128
-#PBS -l place=vscatter:excl
+#PBS -l place=vscatter:exclhost
 #PBS -l debug=true
 
 model=gfs

ecf/scripts/gdas/atmos/analysis/jgdas_atmos_analysis.ecf

@@ -5,7 +5,7 @@
 #PBS -A %PROJ%-%PROJENVIR%
 #PBS -l walltime=00:50:00
 #PBS -l select=52:mpiprocs=15:ompthreads=8:ncpus=120
-#PBS -l place=vscatter:excl
+#PBS -l place=vscatter:exclhost
 #PBS -l debug=true
 
 export model=gfs

ecf/scripts/gdas/atmos/analysis/jgdas_atmos_analysis_calc.ecf

@@ -5,8 +5,7 @@
 #PBS -A %PROJ%-%PROJENVIR%
 #PBS -l walltime=00:10:00
 #PBS -l select=1:mpiprocs=128:ompthreads=1:ncpus=128
-#PBS -l place=vscatter:excl
-#PBS -l hyper=true
+#PBS -l place=vscatter:exclhost
 #PBS -l debug=true
 
 model=gfs

ecf/scripts/gdas/atmos/gempak/jgdas_atmos_gempak.ecf

@@ -4,7 +4,7 @@
 #PBS -q %QUEUE%
 #PBS -A %PROJ%-%PROJENVIR%
 #PBS -l walltime=00:15:00
-#PBS -l select=1:ncpus=2:mpiprocs=2:mem=1GB
+#PBS -l select=1:ncpus=2:mpiprocs=2:mem=4GB
 #PBS -l place=vscatter
 #PBS -l debug=true
 

ecf/scripts/gdas/atmos/init/jgdas_atmos_gldas.ecf

@@ -5,7 +5,7 @@
 #PBS -A %PROJ%-%PROJENVIR%
 #PBS -l walltime=00:20:00
 #PBS -l select=1:mpiprocs=112:ompthreads=1:ncpus=112
-#PBS -l place=vscatter:excl
+#PBS -l place=vscatter:exclhost
 #PBS -l debug=true
 
 model=gfs

ecf/scripts/gdas/atmos/post/jgdas_atmos_post_master.ecf

@@ -4,8 +4,8 @@
 #PBS -q %QUEUE%
 #PBS -A %PROJ%-%PROJENVIR%
 #PBS -l walltime=00:12:00
-#PBS -l select=1:mpiprocs=112:ompthreads=1:ncpus=112
-#PBS -l place=vscatter:excl
+#PBS -l select=1:mpiprocs=126:ompthreads=1:ncpus=126
+#PBS -l place=vscatter:exclhost
 #PBS -l debug=true
 
 model=gfs

ecf/scripts/gdas/jgdas_forecast.ecf

@@ -5,7 +5,7 @@
 #PBS -A %PROJ%-%PROJENVIR%
 #PBS -l walltime=01:00:00
 #PBS -l select=27:mpiprocs=32:ompthreads=3:ncpus=96
-#PBS -l place=vscatter:excl
+#PBS -l place=vscatter:exclhost
 #PBS -l debug=true
 
 model=gfs

ecf/scripts/gfs/atmos/analysis/jgfs_atmos_analysis.ecf

@@ -5,7 +5,7 @@
 #PBS -A %PROJ%-%PROJENVIR%
 #PBS -l walltime=00:40:00
 #PBS -l select=55:mpiprocs=15:ompthreads=8:ncpus=120
-#PBS -l place=vscatter:excl
+#PBS -l place=vscatter:exclhost
 #PBS -l debug=true
 
 export model=gfs

ecf/scripts/gfs/atmos/analysis/jgfs_atmos_analysis_calc.ecf

@@ -5,7 +5,7 @@
 #PBS -A %PROJ%-%PROJENVIR%
 #PBS -l walltime=00:10:00
 #PBS -l select=1:mpiprocs=128:ompthreads=1:ncpus=128
-#PBS -l place=vscatter:excl
+#PBS -l place=vscatter:exclhost
 #PBS -l debug=true
 
 model=gfs

ecf/scripts/gfs/atmos/post/jgfs_atmos_post_master.ecf

@@ -5,7 +5,7 @@
 #PBS -A %PROJ%-%PROJENVIR%
 #PBS -l walltime=00:20:00
 #PBS -l select=1:mpiprocs=126:ompthreads=1:ncpus=126
-#PBS -l place=vscatter:excl
+#PBS -l place=vscatter:exclhost
 #PBS -l debug=true
 
 model=gfs

ecf/scripts/gfs/atmos/post_processing/awips_20km_1p0/jgfs_atmos_awips_master.ecf

@@ -4,7 +4,7 @@
 #PBS -q %QUEUE%
 #PBS -A %PROJ%-%PROJENVIR%
 #PBS -l walltime=00:10:00
-#PBS -l select=1:ncpus=1:mem=1GB
+#PBS -l select=1:ncpus=1:mem=3GB
 #PBS -l place=vscatter
 #PBS -l debug=true
 

ecf/scripts/gfs/atmos/post_processing/awips_g2/jgfs_atmos_awips_g2_master.ecf

@@ -4,7 +4,7 @@
 #PBS -q %QUEUE%
 #PBS -A %PROJ%-%PROJENVIR%
 #PBS -l walltime=00:05:00
-#PBS -l select=1:ncpus=1:mem=1GB
+#PBS -l select=1:ncpus=1:mem=3GB
 #PBS -l place=vscatter
 #PBS -l debug=true
 

ecf/scripts/gfs/atmos/post_processing/bufr_sounding/jgfs_atmos_postsnd.ecf

@@ -4,8 +4,8 @@
 #PBS -q %QUEUE%
 #PBS -A %PROJ%-%PROJENVIR%
 #PBS -l walltime=02:00:00
-#PBS -l select=2:mpiprocs=20:ompthreads=1:ncpus=20
-#PBS -l place=vscatter:excl
+#PBS -l select=2:mpiprocs=20:ompthreads=6:ncpus=120
+#PBS -l place=vscatter:exclhost
 #PBS -l debug=true
 
 model=gfs

ecf/scripts/gfs/atmos/post_processing/bulletins/jgfs_atmos_fbwind.ecf

@@ -4,7 +4,7 @@
 #PBS -q %QUEUE%
 #PBS -A %PROJ%-%PROJENVIR%
 #PBS -l walltime=00:05:00
-#PBS -l select=1:ncpus=1:mem=1GB
+#PBS -l select=1:ncpus=1:mem=2GB
 #PBS -l place=vscatter
 #PBS -l debug=true
 

ecf/scripts/gfs/jgfs_forecast.ecf

@@ -5,7 +5,7 @@
 #PBS -A %PROJ%-%PROJENVIR%
 #PBS -l walltime=02:30:00
 #PBS -l select=112:mpiprocs=24:ompthreads=5:ncpus=120
-#PBS -l place=vscatter:excl
+#PBS -l place=vscatter:exclhost
 #PBS -l debug=true
 
 model=gfs

ecf/scripts/gfs/wave/post/jgfs_wave_post_bndpnt.ecf

@@ -5,7 +5,7 @@
 #PBS -A %PROJ%-%PROJENVIR%
 #PBS -l walltime=01:00:00
 #PBS -l select=3:ncpus=80:ompthreads=1
-#PBS -l place=vscatter:excl
+#PBS -l place=vscatter:exclhost
 #PBS -l debug=true
 
 model=gfs

ecf/scripts/gfs/wave/post/jgfs_wave_post_bndpntbll.ecf

@@ -5,7 +5,7 @@
 #PBS -A %PROJ%-%PROJENVIR%
 #PBS -l walltime=01:00:00
 #PBS -l select=4:ncpus=112:ompthreads=1
-#PBS -l place=vscatter:excl
+#PBS -l place=vscatter:exclhost
 #PBS -l debug=true
 
 model=gfs

ecf/scripts/gfs/wave/post/jgfs_wave_postpnt.ecf

@@ -5,7 +5,7 @@
 #PBS -A %PROJ%-%PROJENVIR%
 #PBS -l walltime=01:30:00
 #PBS -l select=4:ncpus=50:ompthreads=1
-#PBS -l place=vscatter:excl
+#PBS -l place=vscatter:exclhost
 #PBS -l debug=true
 
 model=gfs

env/HERA.env

@@ -37,9 +37,7 @@ if [ $step = "prep" -o $step = "prepbufr" ]; then
     export POE="NO"
     export BACK="NO"
     export sys_tp="HERA"
-
-    module unload netcdfp/4.7.4
-    module load netcdf/4.7.0
+    export launcher_PREP="srun"
 
 elif [ $step = "waveinit" -o $step = "waveprep" -o $step = "wavepostsbs" -o $step = "wavepostbndpnt" -o $step = "wavepostbndpntbll" -o $step = "wavepostpnt" ]; then
     export mpmd="--multi-prog"

env/ORION.env

@@ -41,6 +41,7 @@ if [ $step = "prep" -o $step = "prepbufr" ]; then
     export POE="NO"
     export BACK=${BACK:-"YES"}
     export sys_tp="ORION"
+    export launcher_PREP="srun"
 
 elif [ $step = "waveinit" -o $step = "waveprep" -o $step = "wavepostsbs" -o $step = "wavepostbndpnt" -o $step = "wavepostpnt" ]; then
 
@@ -81,6 +82,8 @@ elif [ $step = "anal" ]; then
 
 elif [ $step = "gldas" ]; then
 
+    export USE_CFP="NO"
+
     nth_max=$(($npe_node_max / $npe_node_gldas))
 
     export NTHREADS_GLDAS=${nth_gldas:-$nth_max}

env/WCOSS2.env

@@ -27,6 +27,7 @@ if [ $step = "prep" -o $step = "prepbufr" ]; then
     export POE=${POE:-"YES"}
     export BACK=${BACK:-"off"}
     export sys_tp="wcoss2"
+    export launcher_PREP="mpiexec"
 
 elif [ $step = "waveinit" -o $step = "waveprep" -o $step = "wavepostsbs" -o $step = "wavepostbndpnt" -o $step = "wavepostbndpntbll"  -o $step = "wavepostpnt" ]; then
 
@@ -182,7 +183,7 @@ elif [ $step = "fv3ic" ]; then
 elif [ $step = "postsnd" ]; then
 
     export NTHREADS_POSTSND=${nth_postsnd:-1}
-    export APRUN_POSTSND="$launcher -n $npe_postsnd --depth=6 --cpu-bind depth"
+    export APRUN_POSTSND="$launcher -n $npe_postsnd --depth=$NTHREADS_POSTSND --cpu-bind depth"
     export NTHREADS_POSTSNDCFP=${nth_postsndcfp:-1}
     export APRUN_POSTSNDCFP="$launcher -np $npe_postsndcfp $mpmd"