Merge branch 'NOAA-EMC:develop' into develop

.gitignore

@@ -166,6 +166,14 @@ scripts/exemcsfc_global_sfc_prep.sh
 scripts/exgdas_global_marine_analysis_ecen.py
 scripts/exglobal_prep_ocean_obs.py
 # ush symlinks
+ush/bufr2ioda_insitu_profile_argo.py
+ush/bufr2ioda_insitu_profile_bathy.py
+ush/bufr2ioda_insitu_profile_glider.py
+ush/bufr2ioda_insitu_profile_marinemammal.py
+ush/bufr2ioda_insitu_profile_tesac.py
+ush/bufr2ioda_insitu_profile_xbtctd.py
+ush/bufr2ioda_insitu_surface_altkob.py
+ush/bufr2ioda_insitu_surface_trkob.py
 ush/chgres_cube.sh
 ush/emcsfc_ice_blend.sh
 ush/emcsfc_snow.sh

.gitmodules

@@ -26,7 +26,3 @@
 [submodule "sorc/gsi_monitor.fd"]
   path = sorc/gsi_monitor.fd
   url = https://github.com/NOAA-EMC/GSI-Monitor.git
-[submodule "sorc/jcb"]
-  path = sorc/jcb
-  url = https://github.com/noaa-emc/jcb
-  fetchRecurseSubmodules = false

ci/cases/yamls/gfs_extended_ci.yaml

@@ -10,3 +10,4 @@ base:
   DO_NPOESS: "YES"
   DO_GENESIS_FSU: "NO"
   FHMAX_GFS: 384
+  FHMAX_HF_GFS: 120

env/HERA.env

@@ -102,6 +102,13 @@ elif [[ "${step}" = "atmanlfv3inc" ]]; then
     [[ ${NTHREADS_ATMANLFV3INC} -gt ${nth_max} ]] && export NTHREADS_ATMANLFV3INC=${nth_max}
     export APRUN_ATMANLFV3INC="${launcher} -n ${npe_atmanlfv3inc} --cpus-per-task=${NTHREADS_ATMANLFV3INC}"
 
+elif [[ "${step}" = "prepobsaero" ]]; then
+
+    nth_max=$((npe_node_max / npe_node_prepobsaero))
+
+    export NTHREADS_PREPOBSAERO=${nth_prepobsaero:-1}
+    export APRUN_PREPOBSAERO="${launcher} -n ${npe_prepobsaero} --cpus-per-task=${NTHREADS_PREPOBSAERO}"
+
 elif [[ "${step}" = "snowanl" ]]; then
 
     nth_max=$((npe_node_max / npe_node_snowanl))

env/HERCULES.env

@@ -99,6 +99,12 @@ case ${step} in
     [[ ${NTHREADS_AEROANL} -gt ${nth_max} ]] && export NTHREADS_AEROANL=${nth_max}
     export APRUN_AEROANL="${launcher} -n ${npe_aeroanlrun} --cpus-per-task=${NTHREADS_AEROANL}"
  ;;
+ "prepobsaero")
+    nth_max=$((npe_node_max / npe_node_prepobsaero))
+
+    export NTHREADS_PREPOBSAERO=${nth_prepobsaero:-1}
+    export APRUN_PREPOBSAERO="${launcher} -n ${npe_prepobsaero} --cpus-per-task=${NTHREADS_PREPOBSAERO}"
+;;
  "snowanl")
 
     nth_max=$((npe_node_max / npe_node_snowanl))

env/JET.env

@@ -82,6 +82,13 @@ elif [[ "${step}" = "aeroanlrun" ]]; then
     [[ ${NTHREADS_AEROANL} -gt ${nth_max} ]] && export NTHREADS_AEROANL=${nth_max}
     export APRUN_AEROANL="${launcher} -n ${npe_aeroanlrun}"
 
+elif [[ "${step}" = "prepobsaero" ]]; then
+
+    nth_max=$((npe_node_max / npe_node_prepobsaero))
+
+    export NTHREADS_PREPOBSAERO=${nth_prepobsaero:-1}
+    export APRUN_PREPOBSAERO="${launcher} -n ${npe_prepobsaero} --cpus-per-task=${NTHREADS_PREPOBSAERO}"
+
 elif [[ "${step}" = "snowanl" ]]; then
 
     nth_max=$((npe_node_max / npe_node_snowanl))

env/ORION.env

@@ -90,6 +90,13 @@ elif [[ "${step}" = "aeroanlrun" ]]; then
     [[ ${NTHREADS_AEROANL} -gt ${nth_max} ]] && export NTHREADS_AEROANL=${nth_max}
     export APRUN_AEROANL="${launcher} -n ${npe_aeroanlrun} --cpus-per-task=${NTHREADS_AEROANL}"
 
+elif [[ "${step}" = "prepobsaero" ]]; then
+
+    nth_max=$((npe_node_max / npe_node_prepobsaero))
+
+    export NTHREADS_PREPOBSAERO=${nth_prepobsaero:-1}
+    export APRUN_PREPOBSAERO="${launcher} -n ${npe_prepobsaero} --cpus-per-task=${NTHREADS_PREPOBSAERO}"
+
 elif [[ "${step}" = "snowanl" ]]; then
 
     nth_max=$((npe_node_max / npe_node_snowanl))

env/S4.env

@@ -82,6 +82,13 @@ elif [[ "${step}" = "aeroanlrun" ]]; then
     [[ ${NTHREADS_AEROANL} -gt ${nth_max} ]] && export NTHREADS_AEROANL=${nth_max}
     export APRUN_AEROANL="${launcher} -n ${npe_aeroanlrun}"
 
+elif [[ "${step}" = "prepobsaero" ]]; then
+
+    nth_max=$((npe_node_max / npe_node_prepobsaero))
+
+    export NTHREADS_PREPOBSAERO=${nth_prepobsaero:-1}
+    export APRUN_PREPOBSAERO="${launcher} -n ${npe_prepobsaero} --cpus-per-task=${NTHREADS_PREPOBSAERO}"
+
 elif [[ "${step}" = "snowanl" ]]; then
 
     nth_max=$((npe_node_max / npe_node_snowanl))

env/WCOSS2.env

@@ -76,6 +76,13 @@ elif [[ "${step}" = "aeroanlrun" ]]; then
     [[ ${NTHREADS_AEROANL} -gt ${nth_max} ]] && export NTHREADS_AEROANL=${nth_max}
     export APRUN_AEROANL="${launcher} -n ${npe_aeroanlrun}"
 
+elif [[ "${step}" = "prepobsaero" ]]; then
+
+    nth_max=$((npe_node_max / npe_node_prepaeroobs))
+
+    export NTHREADS_PREPOBSAERO=${nth_prepobsaero:-1}
+    export APRUN_PREPOBSAERO="${launcher} -n ${npe_prepobsaero} --ppn ${npe_node_prepobsaero}--cpu-bind depth --depth=${NTHREADS_PREPOBSAERO}"
+
 elif [[ "${step}" = "snowanl" ]]; then
 
     nth_max=$((npe_node_max / npe_node_snowanl))

jobs/JGDAS_ENKF_ARCHIVE

@@ -11,8 +11,8 @@ export CDUMP=${RUN/enkf}
 
 YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COM_TOP
 MEMDIR="ensstat" YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \
-  COM_ATMOS_ANALYSIS_ENSSTAT:COM_ATMOS_ANALYSIS_TMPL \
-  COM_ATMOS_HISTORY_ENSSTAT:COM_ATMOS_HISTORY_TMPL
+  COMIN_ATMOS_ANALYSIS_ENSSTAT:COM_ATMOS_ANALYSIS_TMPL \
+  COMIN_ATMOS_HISTORY_ENSSTAT:COM_ATMOS_HISTORY_TMPL
 
 ###############################################################
 # Run archive script

jobs/JGLOBAL_ARCHIVE

@@ -9,21 +9,45 @@ source "${HOMEgfs}/ush/jjob_header.sh" -e "arch" -c "base arch"
 ##############################################
 export CDUMP=${RUN/enkf}
 
-YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COM_ATMOS_ANALYSIS COM_ATMOS_BUFR COM_ATMOS_GEMPAK \
-  COM_ATMOS_GENESIS COM_ATMOS_HISTORY COM_ATMOS_INPUT COM_ATMOS_MASTER COM_ATMOS_RESTART \
-  COM_ATMOS_TRACK COM_ATMOS_WMO \
-  COM_CHEM_HISTORY COM_CHEM_ANALYSIS\
-  COM_MED_RESTART \
-  COM_SNOW_ANALYSIS \
-  COM_ICE_HISTORY COM_ICE_INPUT COM_ICE_RESTART COM_ICE_GRIB \
-  COM_OBS COM_TOP \
-  COM_OCEAN_HISTORY COM_OCEAN_RESTART COM_OCEAN_GRIB COM_OCEAN_NETCDF \
-  COM_OCEAN_ANALYSIS \
-  COM_WAVE_GRID COM_WAVE_HISTORY COM_WAVE_STATION COM_WAVE_RESTART \
-  COM_ATMOS_OZNMON COM_ATMOS_RADMON COM_ATMOS_MINMON COM_CONF
+YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \
+	COMIN_ATMOS_ANALYSIS:COM_ATMOS_ANALYSIS_TMPL \
+	COMIN_ATMOS_BUFR:COM_ATMOS_BUFR_TMPL \
+	COMIN_ATMOS_GEMPAK:COM_ATMOS_GEMPAK_TMPL \
+	COMIN_ATMOS_GENESIS:COM_ATMOS_GENESIS_TMPL \
+	COMIN_ATMOS_HISTORY:COM_ATMOS_HISTORY_TMPL \
+	COMIN_ATMOS_INPUT:COM_ATMOS_INPUT_TMPL \
+	COMIN_ATMOS_MASTER:COM_ATMOS_MASTER_TMPL \
+	COMIN_ATMOS_RESTART:COM_ATMOS_RESTART_TMPL \
+	COMIN_ATMOS_TRACK:COM_ATMOS_TRACK_TMPL \
+	COMIN_ATMOS_WMO:COM_ATMOS_WMO_TMPL \
+	COMIN_CHEM_HISTORY:COM_CHEM_HISTORY_TMPL \
+	COMIN_CHEM_ANALYSIS:COM_CHEM_ANALYSIS_TMPL \
+	COMIN_MED_RESTART:COM_MED_RESTART_TMPL \
+	COMIN_SNOW_ANALYSIS:COM_SNOW_ANALYSIS_TMPL \
+	COMIN_ICE_HISTORY:COM_ICE_HISTORY_TMPL \
+	COMIN_ICE_INPUT:COM_ICE_INPUT_TMPL \
+	COMIN_ICE_RESTART:COM_ICE_RESTART_TMPL \
+	COMIN_ICE_GRIB:COM_ICE_GRIB_TMPL \
+	COMIN_OBS:COM_OBS_TMPL \
+	COMIN_TOP:COM_TOP_TMPL \
+	COMIN_OCEAN_HISTORY:COM_OCEAN_HISTORY_TMPL \
+	COMIN_OCEAN_RESTART:COM_OCEAN_RESTART_TMPL \
+	COMIN_OCEAN_GRIB:COM_OCEAN_GRIB_TMPL \
+	COMIN_OCEAN_NETCDF:COM_OCEAN_NETCDF_TMPL \
+	COMIN_OCEAN_ANALYSIS:COM_OCEAN_ANALYSIS_TMPL \
+	COMIN_WAVE_GRID:COM_WAVE_GRID_TMPL \
+	COMIN_WAVE_HISTORY:COM_WAVE_HISTORY_TMPL \
+	COMIN_WAVE_STATION:COM_WAVE_STATION_TMPL \
+	COMIN_WAVE_RESTART:COM_WAVE_RESTART_TMPL \
+	COMIN_ATMOS_OZNMON:COM_ATMOS_OZNMON_TMPL \
+	COMIN_ATMOS_RADMON:COM_ATMOS_RADMON_TMPL \
+	COMIN_ATMOS_MINMON:COM_ATMOS_MINMON_TMPL \
+	COMIN_CONF:COM_CONF_TMPL \
+	COMOUT_ATMOS_TRACK:COM_ATMOS_TRACK_TMPL
 
 for grid in "0p25" "0p50" "1p00"; do
-  YMD=${PDY} HH=${cyc} GRID=${grid} declare_from_tmpl -rx "COM_ATMOS_GRIB_${grid}:COM_ATMOS_GRIB_GRID_TMPL"
+    YMD=${PDY} HH=${cyc} GRID=${grid} declare_from_tmpl -rx \
+       "COMIN_ATMOS_GRIB_${grid}:COM_ATMOS_GRIB_GRID_TMPL"
 done
 
 ###############################################################

jobs/JGLOBAL_PREP_OBS_AERO

@@ -0,0 +1,43 @@
+#! /usr/bin/env bash
+
+source "${HOMEgfs}/ush/preamble.sh"
+source "${HOMEgfs}/ush/jjob_header.sh" -e "prepobsaero" -c "base prepobsaero"
+
+##############################################
+# Set variables used in the script
+##############################################
+
+export COMIN_OBS="${DATA}"
+YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COMOUT_OBS:COM_OBS_TMPL
+
+##############################################
+# Begin JOB SPECIFIC work
+##############################################
+
+###############################################################
+# Run relevant script
+
+EXSCRIPT=${GDASPREPAEROOBSPY:-${SCRgfs}/exglobal_prep_obs_aero.py}
+${EXSCRIPT}
+status=$?
+[[ ${status} -ne 0 ]] && exit "${status}"
+
+
+##############################################
+# End JOB SPECIFIC work
+##############################################
+
+##############################################
+# Final processing
+##############################################
+if [[ -e "${pgmout}" ]] ; then
+  cat "${pgmout}"
+fi
+
+##########################################
+# Remove the Temporary working directory
+##########################################
+cd "${DATAROOT}" || exit
+[[ "${KEEPDATA}" = "NO" ]] && rm -rf "${DATA}"
+
+exit 0

jobs/rocoto/atmos_ensstat.sh

@@ -15,21 +15,11 @@ if (( status != 0 )); then exit "${status}"; fi
 export job="atmos_ensstat"
 export jobid="${job}.$$"
 
-###############################################################
-# shellcheck disable=SC2153,SC2001
-IFS='_' read -ra fhrs <<< "${FHRLST//f}" # strip off the 'f's and convert to array
+export FORECAST_HOUR=$(( 10#${FHR3} ))
 
-#---------------------------------------------------------------
+###############################################################
 # Execute the JJOB
-for fhr in "${fhrs[@]}"; do
-    # The analysis fhr is -001.  Performing math on negative, leading 0 integers is tricky.
-    # The negative needs to be in front of "10#", so do some regex magic to make it happen.
-    fhr="10#${fhr}"
-    fhr=${fhr//10\#-/-10\#}
-    export FORECAST_HOUR=$(( fhr ))
-    "${HOMEgfs}/jobs/JGLOBAL_ATMOS_ENSSTAT"
-    status=$?
-    if (( status != 0 )); then exit "${status}"; fi
-done
+###############################################################
+"${HOMEgfs}/jobs/JGLOBAL_ATMOS_ENSSTAT"
 
-exit 0
+exit $?

jobs/rocoto/atmos_products.sh

@@ -15,21 +15,13 @@ if (( status != 0 )); then exit "${status}"; fi
 export job="atmos_products"
 export jobid="${job}.$$"
 
-###############################################################
-# shellcheck disable=SC2153,SC2001
-IFS='_' read -ra fhrs <<< "${FHRLST//f}" # strip off the 'f's and convert to array
+# Negatation needs to be before the base
+fhr3_base="10#${FHR3}"
+export FORECAST_HOUR=$(( ${fhr3_base/10#-/-10#} ))
 
-#---------------------------------------------------------------
+###############################################################
 # Execute the JJOB
-for fhr in "${fhrs[@]}"; do
-    # The analysis fhr is -001.  Performing math on negative, leading 0 integers is tricky.
-    # The negative needs to be in front of "10#", so do some regex magic to make it happen.
-    fhr="10#${fhr}"
-    fhr=${fhr//10\#-/-10\#}
-    export FORECAST_HOUR=$(( fhr ))
-    "${HOMEgfs}/jobs/JGLOBAL_ATMOS_PRODUCTS"
-    status=$?
-    if (( status != 0 )); then exit "${status}"; fi
-done
+###############################################################
+"${HOMEgfs}/jobs/JGLOBAL_ATMOS_PRODUCTS"
 
-exit 0
+exit $?

jobs/rocoto/oceanice_products.sh

@@ -21,17 +21,11 @@ export PYTHONPATH
 export job="oceanice_products"
 export jobid="${job}.$$"
 
-###############################################################
-# shellcheck disable=SC2153,SC2001
-IFS='_' read -ra fhrs <<< "${FHRLST//f}" # strip off the 'f's and convert to array
+export FORECAST_HOUR=$(( 10#${FHR3} ))
 
-#---------------------------------------------------------------
+###############################################################
 # Execute the JJOB
-for fhr in "${fhrs[@]}"; do
-    export FORECAST_HOUR=$(( 10#${fhr} ))
-    "${HOMEgfs}/jobs/JGLOBAL_OCEANICE_PRODUCTS"
-    status=$?
-    if (( status != 0 )); then exit "${status}"; fi
-done
-
-exit 0
+###############################################################
+"${HOMEgfs}/jobs/JGLOBAL_OCEANICE_PRODUCTS"
+
+exit $?

jobs/rocoto/prepobsaero.sh

@@ -0,0 +1,24 @@
+#! /usr/bin/env bash
+
+source "${HOMEgfs}/ush/preamble.sh"
+
+###############################################################
+# Source UFSDA workflow modules
+. "${HOMEgfs}/ush/load_ufsda_modules.sh"
+status=$?
+[[ ${status} -ne 0 ]] && exit "${status}"
+
+export job="prepobsaero"
+export jobid="${job}.$$"
+
+###############################################################
+# setup python path for workflow utilities and tasks
+wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src"
+PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}"
+export PYTHONPATH
+
+###############################################################
+# Execute the JJOB
+"${HOMEgfs}/jobs/JGLOBAL_PREP_OBS_AERO"
+status=$?
+exit "${status}"

jobs/rocoto/upp.sh

@@ -44,16 +44,11 @@ export PYTHONPATH
 export job="upp"
 export jobid="${job}.$$"
 
-###############################################################
-# shellcheck disable=SC2153,SC2001
-IFS='_' read -ra fhrs <<< "${FHRLST//f}" # strip off the 'f's convert to array
+export FORECAST_HOUR=$(( 10#${FHR3} ))
 
+###############################################################
 # Execute the JJOB
-for fhr in "${fhrs[@]}"; do
-    export FORECAST_HOUR=$(( 10#${fhr} ))
-    "${HOMEgfs}/jobs/JGLOBAL_ATMOS_UPP"
-    status=$?
-    if (( status != 0 )); then exit "${status}"; fi
-done
-
-exit 0
+###############################################################
+"${HOMEgfs}/jobs/JGLOBAL_ATMOS_UPP"
+
+exit $?