Merge remote-tracking branch 'upstream/develop' into feature/use_jcb_atm

* upstream/develop:
  Add rocoto `sh` tag, script to check netcdf file and apply this to check ocean output (NOAA-EMC#2484)
  Revert file name changes in off-line post (NOAA-EMC#2499)
  Add mean/spread for atmos grib2 (NOAA-EMC#2482)
  Upadte and tested CI Bash for WCOSS2 (NOAA-EMC#2481)

ci/cases/pr/C48_S2SWA_gefs.yaml

@@ -17,3 +17,5 @@ arguments:
   edate: 2021032312
   yaml: {{ HOMEgfs }}/ci/cases/yamls/gefs_ci_defaults.yaml
 
+skip_ci_on_hosts:
+  - wcoss2

ci/cases/pr/C48mx500_3DVarAOWCDA.yaml

@@ -18,6 +18,7 @@ arguments:
   yaml: {{ HOMEgfs }}/ci/cases/yamls/soca_gfs_defaults_ci.yaml
 
 skip_ci_on_hosts:
+  - wcoss2
   - orion
   - hera
   - hercules

ci/cases/pr/C96C48_ufs_hybatmDA.yaml

@@ -21,3 +21,5 @@ skip_ci_on_hosts:
   - hera
   - orion
   - hercules
+  - wcoss2
+

ci/cases/pr/C96_atmaerosnowDA.yaml

@@ -19,3 +19,4 @@ arguments:
 skip_ci_on_hosts:
   - orion
   - hercules
+  - wcoss2

ci/platforms/config.wcoss2

@@ -0,0 +1,8 @@
+#!/usr/bin/bash
+
+export GFS_CI_ROOT=/lfs/h2/emc/global/noscrub/globalworkflow.ci/GFS_CI_ROOT
+export ICSDIR_ROOT=/lfs/h2/emc/global/noscrub/emc.global/data/ICSDIR
+export STMP="/lfs/h2/emc/stmp/${USER}"
+export SLURM_ACCOUNT=GFS-DEV
+export max_concurrent_cases=5
+export max_concurrent_pr=4

ci/scripts/check_ci.sh

@@ -13,16 +13,15 @@ scriptname=$(basename "${BASH_SOURCE[0]}")
 echo "Begin ${scriptname} at $(date -u)" || true
 export PS4='+ $(basename ${BASH_SOURCE})[${LINENO}]'
 
-GH=${HOME}/bin/gh
-REPO_URL="https://github.com/NOAA-EMC/global-workflow.git"
+REPO_URL=${REPO_URL:-"git@github.com:NOAA-EMC/global-workflow.git"}
 
 #########################################################################
 #  Set up runtime environment varibles for accounts on supproted machines
 #########################################################################
 
 source "${HOMEgfs}/ush/detect_machine.sh"
 case ${MACHINE_ID} in
-  hera | orion | hercules)
+  hera | orion | hercules | wcoss2)
    echo "Running Automated Testing on ${MACHINE_ID}"
    source "${HOMEgfs}/ci/platforms/config.${MACHINE_ID}"
    ;;
@@ -38,7 +37,18 @@ source "${HOMEgfs}/ci/scripts/utils/ci_utils.sh"
 module use "${HOMEgfs}/modulefiles"
 module load "module_gwsetup.${MACHINE_ID}"
 module list
+# Load machine specific modules for ci (only wcoss2 is current)
+if [[ "${MACHINE_ID}" == "wcoss2" ]]; then
+  module load "module_gwci.${MACHINE_ID}"
+fi
 set -x
+if ! command -v gh > /dev/null; then
+   GH="${HOME}/bin/gh"
+else
+   GH=$(command -v gh)
+fi
+export GH
+
 rocotostat=$(command -v rocotostat)
 if [[ -z ${rocotostat+x} ]]; then
   echo "rocotostat not found on system"
@@ -141,10 +151,7 @@ for pr in ${pr_list}; do
       sed -i "1 i\`\`\`" "${output_ci}"
       "${GH}" pr comment "${pr}" --repo "${REPO_URL}" --body-file "${output_ci}"
       "${HOMEgfs}/ci/scripts/pr_list_database.py" --remove_pr "${pr}" --dbfile "${pr_list_dbfile}"
-      for kill_cases in "${pr_dir}/RUNTESTS/"*; do
-         pslot=$(basename "${kill_cases}")
-         cancel_slurm_jobs "${pslot}"
-      done
+      cancel_all_batch_jobs "${pr_dir}/RUNTESTS/"
       break
     fi
     if [[ "${num_done}" -eq  "${num_cycles}" ]]; then

ci/scripts/clone-build_ci.sh

@@ -74,7 +74,7 @@ set +e
 source "${HOMEgfs}/ush/module-setup.sh"
 export BUILD_JOBS=8
 rm -rf log.build
-./build_all.sh -gu  >> log.build 2>&1
+./build_all.sh -g  >> log.build 2>&1
 build_status=$?
 
 DATE=$(date +'%D %r')

ci/scripts/driver.sh

@@ -14,11 +14,7 @@ set -eux
 # It then is ready to run a suite of regression tests with various configurations
 #######################################################################################
 
-#################################################################
-# TODO using static build for GitHub CLI until fixed in HPC-Stack
-#################################################################
-export GH=${HOME}/bin/gh
-export REPO_URL=${REPO_URL:-"https://github.com/NOAA-EMC/global-workflow.git"}
+export REPO_URL=${REPO_URL:-"git@github.com:NOAA-EMC/global-workflow.git"}
 
 ################################################################
 # Setup the reletive paths to scripts and PS4 for better logging
@@ -34,7 +30,7 @@ export PS4='+ $(basename ${BASH_SOURCE})[${LINENO}]'
 
 source "${ROOT_DIR}/ush/detect_machine.sh"
 case ${MACHINE_ID} in
-  hera | orion | hercules)
+  hera | orion | hercules | wcoss2)
     echo "Running Automated Testing on ${MACHINE_ID}"
     source "${ROOT_DIR}/ci/platforms/config.${MACHINE_ID}"
     ;;
@@ -54,8 +50,18 @@ source "${ROOT_DIR}/ci/scripts/utils/ci_utils.sh"
 source "${ROOT_DIR}/ush/module-setup.sh"
 module use "${ROOT_DIR}/modulefiles"
 module load "module_gwsetup.${MACHINE_ID}"
+# Load machine specific modules for ci (only wcoss2 is current)
+if [[ "${MACHINE_ID}" == "wcoss2" ]]; then
+  module load "module_gwci.${MACHINE_ID}"
+fi
 set -x
 unset HOMEgfs
+if ! command -v gh > /dev/null; then
+   GH="${HOME}/bin/gh"
+else
+   GH=$(command -v gh)
+fi
+export GH
 
 ############################################################
 # query repo and get list of open PRs with tags {machine}-CI
@@ -113,7 +119,7 @@ for pr in ${pr_list}; do
     else
       for case in ${experiments}; do
         case_name=$(basename "${case}")
-        cancel_slurm_jobs "${case_name}"
+        cancel_batch_jobs "${case_name}"
         {
           echo "Canceled all jobs for experiment ${case_name} in PR:${pr} on ${MACHINE_ID^}"
         } >> "${output_ci_single}"

ci/scripts/driver_weekly.sh

@@ -38,7 +38,7 @@ export PS4='+ $(basename ${BASH_SOURCE[0]})[${LINENO}]'
 
 source "${ROOT_DIR}/ush/detect_machine.sh"
 case ${MACHINE_ID} in
-  hera | orion | hercules)
+  hera | orion | hercules | wcoss2)
     echo "Running Automated Testing on ${MACHINE_ID}"
     source "${ROOT_DIR}/ci/platforms/config.${MACHINE_ID}"
     ;;

ci/scripts/run_ci.sh

@@ -20,7 +20,7 @@ export PS4='+ $(basename ${BASH_SOURCE})[${LINENO}]'
 
 source "${HOMEgfs}/ush/detect_machine.sh"
 case ${MACHINE_ID} in
-  hera | orion | hercules)
+  hera | orion | hercules | wcoss2)
    echo "Running Automated Testing on ${MACHINE_ID}"
    source "${HOMEgfs}/ci/platforms/config.${MACHINE_ID}"
    ;;

ci/scripts/utils/ci_utils.sh

@@ -23,7 +23,7 @@ function cancel_batch_jobs() {
 
   scheduler=$(determine_scheduler)
 
-  if [[ "${schduler}" == "torque" ]]; then
+  if [[ "${scheduler}" == "torque" ]]; then
     job_ids=$(qstat -u "${USER}" | awk '{print $1}') || true
 
     for job_id in ${job_ids}; do

jobs/JGLOBAL_ATMOS_ENSSTAT

@@ -0,0 +1,48 @@
+#! /usr/bin/env bash
+
+#
+# Caculate the mean, spread, and other probabilistic fields.
+#
+
+source "${HOMEgfs}/ush/preamble.sh"
+source "${HOMEgfs}/ush/jjob_header.sh" -e "atmos_ensstat" -c "base atmos_ensstat"
+
+
+##############################################
+# Begin JOB SPECIFIC work
+##############################################
+
+# Construct COM variables from templates
+# Input directories loop over members, so this is done downstream
+
+for grid in '0p25' '0p50' '1p00'; do
+  prod_dir="COMOUT_ATMOS_GRIB_${grid}"
+  MEMDIR="ensstat" GRID=${grid} YMD=${PDY} HH=${cyc} declare_from_tmpl -rx "${prod_dir}:COM_ATMOS_GRIB_GRID_TMPL"
+  if [[ ! -d "${!prod_dir}" ]]; then mkdir -m 775 -p "${!prod_dir}"; fi
+done
+
+###############################################################
+# Run exglobal script
+"${SCRgfs}/exglobal_atmos_ensstat.sh"
+status=$?
+(( status != 0 )) && exit "${status}"
+
+##############################################
+# End JOB SPECIFIC work
+##############################################
+
+##############################################
+# Final processing
+##############################################
+if [[ -e "${pgmout}" ]]; then
+  cat "${pgmout}"
+fi
+
+##########################################
+# Remove the Temporary working directory
+##########################################
+cd "${DATAROOT}" || exit 1
+[[ "${KEEPDATA:-NO}" = "NO" ]] && rm -rf "${DATA}"
+
+
+exit 0

jobs/rocoto/atmos_ensstat.sh

@@ -0,0 +1,35 @@
+#! /usr/bin/env bash
+
+source "${HOMEgfs}/ush/preamble.sh"
+
+###############################################################
+## atmosphere products driver script
+## FHRLST : forecast hour list to post-process (e.g. -f001, f000, f000_f001_f002, ...)
+###############################################################
+
+# Source FV3GFS workflow modules
+. "${HOMEgfs}/ush/load_fv3gfs_modules.sh"
+status=$?
+if (( status != 0 )); then exit "${status}"; fi
+
+export job="atmos_ensstat"
+export jobid="${job}.$$"
+
+###############################################################
+# shellcheck disable=SC2153,SC2001
+IFS='_' read -ra fhrs <<< "${FHRLST//f}" # strip off the 'f's and convert to array
+
+#---------------------------------------------------------------
+# Execute the JJOB
+for fhr in "${fhrs[@]}"; do
+    # The analysis fhr is -001.  Performing math on negative, leading 0 integers is tricky.
+    # The negative needs to be in front of "10#", so do some regex magic to make it happen.
+    fhr="10#${fhr}"
+    fhr=${fhr//10\#-/-10\#}
+    export FORECAST_HOUR=$(( fhr ))
+    "${HOMEgfs}/jobs/JGLOBAL_ATMOS_ENSSTAT"
+    status=$?
+    if (( status != 0 )); then exit "${status}"; fi
+done
+
+exit 0

modulefiles/module_gwci.wcoss2.lua

@@ -0,0 +1,8 @@
+help([[
+Load environment to run GFS workflow ci scripts on WCOSS2
+]])
+
+prepend_path("MODULEPATH", "/apps/ops/para/nco/modulefiles/core")
+load(pathJoin("gh","2.28.0"))
+
+whatis("Description: GFS run setup ci environment")

parm/config/gefs/config.atmos_ensstat

@@ -0,0 +1,11 @@
+#! /usr/bin/env bash
+
+########## config.atmos_ensstat ##########
+# atmosphere grib2 enstat specific
+
+echo "BEGIN: config.atmos_ensstat"
+
+# Get task specific resources
+. "${EXPDIR}/config.resources" atmos_ensstat
+
+echo "END: config.atmos_ensstat"

parm/config/gefs/config.resources

@@ -7,23 +7,6 @@
 if (( $# != 1 )); then
 
     echo "Must specify an input task argument to set resource variables!"
-    echo "argument can be any one of the following:"
-    echo "stage_ic aerosol_init"
-    echo "prep prepsnowobs prepatmiodaobs"
-    echo "atmanlinit atmanlrun atmanlfinal"
-    echo "atmensanlinit atmensanlrun atmensanlfinal"
-    echo "snowanl"
-    echo "aeroanlinit aeroanlrun aeroanlfinal"
-    echo "anal sfcanl analcalc analdiag fcst echgres"
-    echo "upp atmos_products"
-    echo "tracker genesis genesis_fsu"
-    echo "verfozn verfrad vminmon fit2obs metp arch cleanup"
-    echo "eobs ediag eomg eupd ecen esfc efcs epos earc"
-    echo "init_chem mom6ic"
-    echo "waveinit waveprep wavepostsbs wavepostbndpnt wavepostbndpntbll wavepostpnt"
-    echo "wavegempak waveawipsbulls waveawipsgridded"
-    echo "postsnd awips gempak npoess"
-    echo "ocnanalprep prepoceanobs ocnanalbmat ocnanalrun ocnanalchkpt ocnanalpost ocnanalvrfy"
     exit 1
 
 fi
@@ -232,6 +215,18 @@ case ${step} in
     export is_exclusive=True
     ;;
 
+  "atmos_ensstat")
+    export wtime_atmos_ensstat="00:30:00"
+    export npe_atmos_ensstat=6
+    export nth_atmos_ensstat=1
+    export npe_node_atmos_ensstat="${npe_atmos_ensstat}"
+    export wtime_atmos_ensstat_gfs="${wtime_atmos_ensstat}"
+    export npe_atmos_ensstat_gfs="${npe_atmos_ensstat}"
+    export nth_atmos_ensstat_gfs="${nth_atmos_ensstat}"
+    export npe_node_atmos_ensstat_gfs="${npe_node_atmos_ensstat}"
+    export is_exclusive=True
+    ;;
+
   "oceanice_products")
     export wtime_oceanice_products="00:15:00"
     export npe_oceanice_products=1

parm/post/upp.yaml

@@ -40,10 +40,10 @@ forecast:
       - ["{{ COM_ATMOS_HISTORY }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.sfcf{{ '%03d' % forecast_hour }}.nc", "{{ DATA }}/{{ flux_filename }}"]
   data_out:
     copy:
-      - ["{{ DATA }}/GFSPRS.GrbF{{ '%02d' % forecast_hour }}", "{{ COM_ATMOS_MASTER }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.masterf{{ '%03d' % forecast_hour }}.grb2"]
-      - ["{{ DATA }}/GFSFLX.GrbF{{ '%02d' % forecast_hour }}", "{{ COM_ATMOS_MASTER }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.sfluxf{{ '%03d' % forecast_hour }}.grb2"]
-      - ["{{ DATA }}/GFSPRS.GrbF{{ '%02d' % forecast_hour }}.idx", "{{ COM_ATMOS_MASTER }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.masterf{{ '%03d' % forecast_hour }}.grb2.idx"]
-      - ["{{ DATA }}/GFSFLX.GrbF{{ '%02d' % forecast_hour }}.idx", "{{ COM_ATMOS_MASTER }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.sfluxf{{ '%03d' % forecast_hour }}.grb2.idx"]
+      - ["{{ DATA }}/GFSPRS.GrbF{{ '%02d' % forecast_hour }}", "{{ COM_ATMOS_MASTER }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.master.grb2f{{ '%03d' % forecast_hour }}"]
+      - ["{{ DATA }}/GFSFLX.GrbF{{ '%02d' % forecast_hour }}", "{{ COM_ATMOS_MASTER }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.sfluxgrbf{{ '%03d' % forecast_hour }}.grib2"]
+      - ["{{ DATA }}/GFSPRS.GrbF{{ '%02d' % forecast_hour }}.idx", "{{ COM_ATMOS_MASTER }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.master.grb2if{{ '%03d' % forecast_hour }}"]
+      - ["{{ DATA }}/GFSFLX.GrbF{{ '%02d' % forecast_hour }}.idx", "{{ COM_ATMOS_MASTER }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.sfluxgrbf{{ '%03d' % forecast_hour }}.grib2.idx"]
 
 goes:
   config:
@@ -86,5 +86,5 @@ goes:
       - ["{{ COM_ATMOS_HISTORY }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.sfcf{{ '%03d' % forecast_hour }}.nc", "{{ DATA }}/{{ flux_filename }}"]
   data_out:
     copy:
-      - ["{{ DATA }}/GFSPRS.GrbF{{ '%02d' % forecast_hour }}", "{{ COM_ATMOS_MASTER }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.goesmasterf{{ '%03d' % forecast_hour }}.grb2"]
-      - ["{{ DATA }}/GFSPRS.GrbF{{ '%02d' % forecast_hour }}.idx", "{{ COM_ATMOS_MASTER }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.goesmasterf{{ '%03d' % forecast_hour }}.grb2.idx"]
+      - ["{{ DATA }}/GFSPRS.GrbF{{ '%02d' % forecast_hour }}", "{{ COM_ATMOS_MASTER }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.special.grb2f{{ '%03d' % forecast_hour }}"]
+      - ["{{ DATA }}/GFSPRS.GrbF{{ '%02d' % forecast_hour }}.idx", "{{ COM_ATMOS_MASTER }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.special.grb2if{{ '%03d' % forecast_hour }}"]

scripts/exglobal_atmos_ensstat.sh

@@ -0,0 +1,19 @@
+#! /usr/bin/env bash
+
+source "${HOMEgfs}/ush/preamble.sh"
+
+fhr3=$(printf "%03d" "${FORECAST_HOUR}")
+
+if [[ -a mpmd_script ]]; then rm -Rf mpmd_script; fi
+
+{
+    for grid in '0p25' '0p50' '1p00'; do
+        echo "${USHgfs}/atmos_ensstat.sh ${grid} ${fhr3}"
+        # echo "${USHgfs}/atmos_ensstat.sh ${grid} ${fhr3} b"
+    done
+} > mpmd_script
+
+"${USHgfs}/run_mpmd.sh" mpmd_script
+err=$?
+
+exit "${err}"

sorc/link_workflow.sh

@@ -279,7 +279,8 @@ cd "${HOMEgfs}/exec" || exit 1
 
 for utilexe in fbwndgfs.x gaussian_sfcanl.x gfs_bufr.x supvit.x syndat_getjtbul.x \
   syndat_maksynrc.x syndat_qctropcy.x tocsbufr.x overgridid.x rdbfmsua.x \
-  mkgfsawps.x enkf_chgres_recenter_nc.x tave.x vint.x ocnicepost.x webtitle.x
+  mkgfsawps.x enkf_chgres_recenter_nc.x tave.x vint.x ocnicepost.x webtitle.x \
+  ensadd.x ensppf.x ensstat.x wave_stat.x
 do
   [[ -s "${utilexe}" ]] && rm -f "${utilexe}"
   ${LINK_OR_COPY} "${HOMEgfs}/sorc/gfs_utils.fd/install/bin/${utilexe}" .