Merge remote-tracking branch 'upstream/develop' into feature/use_jcb_atm

* upstream/develop:
  Update fbwind for COM refactor (NOAA-EMC#2479)
  Update parm files for atm product (NOAA-EMC#2486)
  Fix paths in prepocnobs task (NOAA-EMC#2459)
  Add oceanalecn to workflow generator (NOAA-EMC#2409)
  Add support for CADS IR cloud detection scheme in the GSI (NOAA-EMC#2478)
  Improve error messaging to resolve bugzilla (NOAA-EMC#2468)
  Check the DO_JEDISNOWDA condition before adding snow DA analysis to the list (NOAA-EMC#2471)

.gitignore

@@ -159,16 +159,19 @@ sorc/ocnicepost.fd
 # jobs symlinks
 # scripts symlinks
 scripts/exemcsfc_global_sfc_prep.sh
+scripts/exgdas_global_marine_analysis_ecen.py
+scripts/exglobal_prep_ocean_obs.py
 # ush symlinks
 ush/chgres_cube.sh
 ush/emcsfc_ice_blend.sh
 ush/emcsfc_snow.sh
-ush/exglobal_prep_ocean_obs.py
 ush/fix_precip.sh
 ush/fv3gfs_driver_grid.sh
 ush/fv3gfs_filter_topo.sh
 ush/fv3gfs_make_grid.sh
 ush/fv3gfs_make_orog.sh
+ush/gen_bufr2ioda_json.py
+ush/gen_bufr2ioda_yaml.py
 ush/global_chgres.sh
 ush/global_chgres_driver.sh
 ush/global_cycle.sh

jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_ECEN

@@ -5,6 +5,15 @@ source "${HOMEgfs}/ush/jjob_header.sh" -e "ocnanalecen" -c "base ocnanal ocnanal
 ##############################################
 # Set variables used in the script
 ##############################################
+# Ignore possible spelling error (nothing is misspelled)
+# shellcheck disable=SC2153
+GDATE=$(date --utc +%Y%m%d%H -d "${PDY} ${cyc} - ${assim_freq} hours")
+export gPDY=${GDATE:0:8}
+export gcyc=${GDATE:8:2}
+
+YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \
+   COM_OCEAN_HISTORY_PREV:COM_OCEAN_HISTORY_TMPL \
+   COM_ICE_HISTORY_PREV:COM_ICE_HISTORY_TMPL
 
 ##############################################
 # Begin JOB SPECIFIC work

jobs/JGFS_ATMOS_FBWIND

@@ -17,33 +17,34 @@ export COMPONENT="atmos"
 ##############################################
 # Define COM directories
 ##############################################
-export COMIN=${COMIN:-$(compath.py ${envir}/${NET}/${gfs_ver})/${RUN}.${PDY}/${cyc}/${COMPONENT}}
-export COMOUT=${COMOUT:-$(compath.py -o ${NET}/${gfs_ver}/${RUN}.${PDY})/${cyc}/${COMPONENT}}
-export COMOUTwmo=${COMOUTwmo:-${COMOUT}/wmo}
+
+GRID="0p25" YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COMIN_ATMOS_GRIB_0p25:COM_ATMOS_GRIB_GRID_TMPL
+YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COMOUT:COM_ATMOS_WMO_TMPL
+if [[ ! -d "${COMOUT}" ]]; then
+  mkdir -m 775 -p "${COMOUT}"
+fi
 
 export SENDDBN=${SENDDBN:-NO}
 export DBNROOT=${DBNROOT:-${UTILROOT}/fakedbn}
 
-mkdir -m 775 -p ${COMOUT} ${COMOUTwmo}
-
 
 ########################################################
 # Execute the script.
-${SCRgfs}/exgfs_atmos_fbwind.sh
+"${SCRgfs}/exgfs_atmos_fbwind.sh"
 export err=$?;err_chk
 ########################################################
 
 ############################################
 # print exec I/O output
 ############################################
-if [ -e "${pgmout}" ] ; then
-  cat ${pgmout}
+if [[ -e "${pgmout}" ]] ; then
+  cat "${pgmout}"
 fi
 
 ###################################
 # Remove temp directories
 ###################################
-if [ "${KEEPDATA}" != "YES" ] ; then
-  rm -rf ${DATA}
+if [[ "${KEEPDATA}" != "YES" ]] ; then
+  rm -rf "${DATA}"
 fi
 

parm/config/gfs/config.base

@@ -326,7 +326,7 @@ export imp_physics=8
 export DO_JEDIATMVAR="@DO_JEDIATMVAR@"
 export DO_JEDIATMENS="@DO_JEDIATMENS@"
 export DO_JEDIOCNVAR="@DO_JEDIOCNVAR@"
-export DO_JEDILANDDA="@DO_JEDILANDDA@"
+export DO_JEDISNOWDA="@DO_JEDISNOWDA@"
 export DO_MERGENSST="@DO_MERGENSST@"
 
 # Hybrid related

parm/config/gfs/config.prepoceanobs

@@ -7,15 +7,19 @@ echo "BEGIN: config.prepoceanobs"
 export OCNOBS2IODAEXEC=${HOMEgfs}/sorc/gdas.cd/build/bin/gdas_obsprovider2ioda.x
 
 export SOCA_INPUT_FIX_DIR=@SOCA_INPUT_FIX_DIR@
-export OBS_YAML_DIR=${HOMEgfs}/sorc/gdas.cd/parm/soca/obs/config
+
+export OBS_YAML_DIR="${PARMgfs}/gdas/soca/obs/config"
 export OBSPREP_YAML=@OBSPREP_YAML@
 export OBS_LIST=@SOCA_OBS_LIST@
-[[ -n "${OBS_LIST}" ]] || export OBS_LIST=${HOMEgfs}/sorc/gdas.cd/parm/soca/obs/obs_list.yaml
 export OBS_YAML=${OBS_LIST}
 
 # ocean analysis needs own dmpdir until standard dmpdir has full ocean obs
 export DMPDIR=@DMPDIR@
 
+# For BUFR2IODA json and python scripts
+export JSON_TMPL_DIR="${PARMgfs}/gdas/ioda/bufr2ioda"
+export BUFR2IODA_PY_DIR="${USHgfs}"
+
 # Get task specific resources
 . "${EXPDIR}/config.resources" prepoceanobs
 echo "END: config.prepoceanobs"

parm/product/gfs.anl.paramlist.a.txt

@@ -152,7 +152,6 @@ HLCY:3000-0 m above ground
 ICAHT:max wind
 ICAHT:tropopause
 ICETK:surface
-ICETMP:surface
 ICMR:50 mb
 ICMR:1000 mb
 ICMR:100 mb

parm/product/gfs.f000.paramlist.a.txt

@@ -166,7 +166,6 @@ ICAHT:max wind
 ICAHT:tropopause
 ICEC:surface
 ICETK:surface
-ICETMP:surface
 ICMR:50 mb
 ICMR:1000 mb
 ICMR:100 mb
@@ -697,4 +696,3 @@ WEASD:surface
 WILT:surface
 SNOD:surface
 VIS:surface
-ICEG:10 m above mean sea level

parm/product/gfs.fFFF.paramlist.a.txt

@@ -45,7 +45,6 @@ ACPCP:surface
 ALBDO:surface
 APCP:surface
 APTMP:2 m above ground
-var discipline=0 master_table=2 parmcat=0 parm=21:2 m above ground
 AOTK:entire atmosphere
 CAPE:180-0 mb above ground
 CAPE:255-0 mb above ground
@@ -177,7 +176,6 @@ ICAHT:max wind
 ICAHT:tropopause
 ICEC:surface
 ICETK:surface
-ICETMP:surface
 ICSEV:100 mb
 ICSEV:150 mb
 ICSEV:200 mb
@@ -758,4 +756,3 @@ WEASD:surface
 WILT:surface
 SNOD:surface
 VIS:surface
-ICEG:10 m above mean sea level

scripts/exgfs_atmos_fbwind.sh

@@ -14,15 +14,15 @@
 # echo "         Nov 2019 - B Vuong  Removed WINTEMV bulletin (retired)"
 #####################################################################
 
-source "${USHgfs}/preamble.sh"
+source "${HOMEgfs}/ush/preamble.sh"
 
-cd $DATA
+cd "${DATA}" || exit 2
 
 ######################
 # Set up Here Files.
 ######################
 
-job_name=$(echo $job|sed 's/[jpt]gfs/gfs/')
+outfile_name="${COMOUT}/${RUN}.atmos.t${cyc}z.fbwind.pacific.ascii"
 
 set +x
 echo " "
@@ -34,20 +34,17 @@ echo " "
 set_trace
 
 export pgm=bulls_fbwndgfs
-. prep_step
-
-for fhr in 006 012 024
-do
-
-  cp $COMIN/gfs.${cycle}.pgrb2.0p25.f${fhr}   tmp_pgrb2_0p25${fhr} 
-  cp $COMIN/gfs.${cycle}.pgrb2b.0p25.f${fhr}  tmp_pgrb2b_0p25${fhr} 
-  cat tmp_pgrb2_0p25${fhr} tmp_pgrb2b_0p25${fhr} > tmp0p25filef${fhr} 
-  $WGRIB2 tmp0p25filef${fhr} | grep  -F -f ${PARMgfs}/product/gfs_fbwnd_parmlist_g2 | $WGRIB2 -i -grib tmpfilef${fhr} tmp0p25filef${fhr}
-  $CNVGRIB -g21 tmpfilef${fhr} tmpfilef${fhr}.grib1
-  $GRBINDEX tmpfilef${fhr}.grib1 tmpfilef${fhr}.grib1i
-  mv tmpfilef${fhr}.grib1   gfs.t${cyc}z.grbf${fhr}_grb1
-  mv tmpfilef${fhr}.grib1i  gfs.t${cyc}z.grbif${fhr}_grb1
-
+source prep_step
+
+for fhr3 in 006 012 024; do
+  cp "${COMIN_ATMOS_GRIB_0p25}/gfs.${cycle}.pgrb2.0p25.f${fhr3}"   "tmp_pgrb2_0p25${fhr3}" 
+  cp "${COMIN_ATMOS_GRIB_0p25}/gfs.${cycle}.pgrb2b.0p25.f${fhr3}"  "tmp_pgrb2b_0p25${fhr3}"
+  cat "tmp_pgrb2_0p25${fhr3}" "tmp_pgrb2b_0p25${fhr3}" > "tmp0p25filef${fhr3}"
+  # shellcheck disable=SC2312
+  ${WGRIB2} "tmp0p25filef${fhr3}" | grep -F -f "${PARMgfs}/product/gfs_fbwnd_parmlist_g2" | \
+    ${WGRIB2} -i -grib "tmpfilef${fhr3}" "tmp0p25filef${fhr3}"
+  ${CNVGRIB} -g21 "tmpfilef${fhr3}" "gfs.t${cyc}z.grbf${fhr3}_grb1"
+  ${GRBINDEX} "gfs.t${cyc}z.grbf${fhr3}_grb1" "gfs.t${cyc}z.grbf${fhr3}_grb1.idx"
 done
 
 export FORT11="gfs.t${cyc}z.grbf006_grb1"
@@ -56,31 +53,21 @@ export FORT13="gfs.t${cyc}z.grbf024_grb1"
 
 #       GFS grib index files
 
-export FORT31="gfs.t${cyc}z.grbif006_grb1"
-export FORT32="gfs.t${cyc}z.grbif012_grb1"
-export FORT33="gfs.t${cyc}z.grbif024_grb1"
+export FORT31="gfs.t${cyc}z.grbf006_grb1.idx"
+export FORT32="gfs.t${cyc}z.grbf012_grb1.idx"
+export FORT33="gfs.t${cyc}z.grbf024_grb1.idx"
 
 #
 #   1280 byte transmission file
 #
 
 export FORT51="tran.fbwnd_pacific"
 
-startmsg
+cp "${PARMgfs}/product/fbwnd_pacific.stnlist" fbwnd_pacific.stnlist
 
-$EXECgfs/fbwndgfs < ${PARMgfs}/product/fbwnd_pacific.stnlist >> $pgmout 2> errfile
+"${EXECgfs}/fbwndgfs.x" < fbwnd_pacific.stnlist >> "${pgmout}" 2> errfile
 export err=$?; err_chk
 
-
-cp tran.fbwnd_pacific ${COMOUTwmo}/tran.fbwnd_pacific.$job_name
-
-if test "$SENDDBN" = 'YES'
-then
-#    make_ntc_bull.pl WMOBH NONE KWNO NONE tran.fbwnd_pacific ${COMOUTwmo}/tran.fbwnd_pacific.$job_name
-   ${USHgfs}/make_ntc_bull.pl WMOBH NONE KWNO NONE tran.fbwnd_pacific ${COMOUTwmo}/tran.fbwnd_pacific.$job_name
-fi
-
-#####################################################################
-
+"${USHgfs}/make_ntc_bull.pl" WMOBH NONE KWNO NONE tran.fbwnd_pacific "${outfile_name}"
 
 ############################### END OF SCRIPT #######################

scripts/exglobal_atmos_analysis.sh

@@ -380,6 +380,10 @@ ${NLN} ${SCANINFO}     scaninfo
 ${NLN} ${HYBENSINFO}   hybens_info
 ${NLN} ${OBERROR}      errtable
 
+${NLN} ${FIXgfs}/gsi/AIRS_CLDDET.NL   AIRS_CLDDET.NL
+${NLN} ${FIXgfs}/gsi/CRIS_CLDDET.NL   CRIS_CLDDET.NL
+${NLN} ${FIXgfs}/gsi/IASI_CLDDET.NL   IASI_CLDDET.NL
+
 #If using correlated error, link to the covariance files
 if [ ${USE_CORRELATED_OBERRS} == "YES" ];  then
   if grep -q "Rcov" ${ANAVINFO} ;  then
@@ -674,7 +678,7 @@ if [ ${DOHYBVAR} = "YES" ]; then
       SETUP="niter(1)=50,niter(2)=150,niter_no_qc(1)=25,niter_no_qc(2)=0,thin4d=.true.,ens_nstarthr=3,l4densvar=${l4densvar},lwrite4danl=${lwrite4danl},${SETUP}"
       JCOPTS="ljc4tlevs=.true.,${JCOPTS}"
       STRONGOPTS="tlnmc_option=3,${STRONGOPTS}"
-      OBSQC="c_varqc=0.04,${OBSQC}"
+      OBSQC="c_varqc=0.04,airs_cads=.false.,cris_cads=.false.,iasi_cads=.false.,${OBSQC}"
    fi
 fi
 

sorc/link_workflow.sh

@@ -153,6 +153,14 @@ done
 
 cd "${HOMEgfs}/scripts" || exit 8
 ${LINK_OR_COPY} "${HOMEgfs}/sorc/ufs_utils.fd/scripts/exemcsfc_global_sfc_prep.sh" .
+if [[ -d "${HOMEgfs}/sorc/gdas.cd" ]]; then
+  declare -a gdas_scripts=(exglobal_prep_ocean_obs.py \
+                           exgdas_global_marine_analysis_ecen.py \
+                           )
+  for gdas_script in "${gdas_scripts[@]}" ; do
+    ${LINK_OR_COPY} "${HOMEgfs}/sorc/gdas.cd/scripts/${gdas_script}" .
+  done
+fi
 cd "${HOMEgfs}/ush" || exit 8
 for file in emcsfc_ice_blend.sh global_cycle_driver.sh emcsfc_snow.sh global_cycle.sh; do
   ${LINK_OR_COPY} "${HOMEgfs}/sorc/ufs_utils.fd/ush/${file}" .
@@ -227,9 +235,10 @@ if [[ -d "${HOMEgfs}/sorc/gdas.cd/build" ]]; then
   cd "${HOMEgfs}/ush" || exit 1
   ${LINK_OR_COPY} "${HOMEgfs}/sorc/gdas.cd/ush/ufsda"                              .
   ${LINK_OR_COPY} "${HOMEgfs}/sorc/gdas.cd/ush/jediinc2fv3.py"                     .
+  ${LINK_OR_COPY} "${HOMEgfs}/sorc/gdas.cd/ush/ioda/bufr2ioda/gen_bufr2ioda_json.py"    .
+  ${LINK_OR_COPY} "${HOMEgfs}/sorc/gdas.cd/ush/ioda/bufr2ioda/gen_bufr2ioda_yaml.py"    .
   ${LINK_OR_COPY} "${HOMEgfs}/sorc/gdas.cd/ush/ioda/bufr2ioda/run_bufr2ioda.py"    .
   ${LINK_OR_COPY} "${HOMEgfs}/sorc/gdas.cd/build/bin/imsfv3_scf2ioda.py"           .
-  ${LINK_OR_COPY} "${HOMEgfs}/sorc/gdas.cd/scripts/exglobal_prep_ocean_obs.py"           .
 fi
 
 

ush/forecast_postdet.sh

@@ -64,7 +64,7 @@ EOF
             increment_file="${COM_ATMOS_ANALYSIS}/${RUN}.t${cyc}z.${PREFIX_ATMINC}atmi${incfhr}.nc"
           fi
           if [[ ! -f ${increment_file} ]]; then
-            echo "ERROR: DOIAU = ${DOIAU}, but missing increment file for fhr ${incfhr} at ${increment_file}"
+            echo "FATAL ERROR: DOIAU = ${DOIAU}, but missing increment file for fhr ${incfhr} at ${increment_file}"
             echo "Abort!"
             exit 1
           fi
@@ -148,7 +148,7 @@ EOF
 
   _suite_file="${HOMEgfs}/sorc/ufs_model.fd/FV3/ccpp/suites/suite_${CCPP_SUITE}.xml"
   if [[ ! -f ${_suite_file} ]]; then
-    echo "FATAL: CCPP Suite file ${_suite_file} does not exist!"
+    echo "FATAL ERROR: CCPP Suite file ${_suite_file} does not exist!"
     exit 2
   fi
 
@@ -569,7 +569,7 @@ WW3_postdet() {
       if [[ ${RERUN} = "NO" ]]; then
         echo "WARNING: NON-FATAL ERROR wave IC is missing, will start from rest"
       else
-        echo "ERROR: Wave IC is missing in RERUN, exiting."
+        echo "FATAL ERROR: Wave IC ${waverstfile} is missing in RERUN, exiting."
         exit 1
       fi
     else
@@ -592,7 +592,7 @@ WW3_postdet() {
   if [[ "${WW3ICEINP}" = "YES" ]]; then
     local wavicefile="${COM_WAVE_PREP}/${RUNwave}.${WAVEICE_FID}.${cycle}.ice"
     if [[ ! -f ${wavicefile} ]]; then
-      echo "ERROR: WW3ICEINP = ${WW3ICEINP}, but missing ice file"
+      echo "FATAL ERROR: WW3ICEINP = ${WW3ICEINP}, but missing ice file ${wavicefile}"
       echo "Abort!"
       exit 1
     fi
@@ -602,7 +602,7 @@ WW3_postdet() {
   if [[ "${WW3CURINP}" = "YES" ]]; then
     local wavcurfile="${COM_WAVE_PREP}/${RUNwave}.${WAVECUR_FID}.${cycle}.cur"
     if [[ ! -f ${wavcurfile} ]]; then
-      echo "ERROR: WW3CURINP = ${WW3CURINP}, but missing current file"
+      echo "FATAL ERROR: WW3CURINP = ${WW3CURINP}, but missing current file ${wavcurfile}"
       echo "Abort!"
       exit 1
     fi
@@ -697,7 +697,7 @@ MOM6_postdet() {
   # Link increment
   if [[ "${DO_JEDIOCNVAR:-NO}" = "YES" ]]; then
       if [[ ! -f "${COM_OCEAN_ANALYSIS}/${RUN}.t${cyc}z.ocninc.nc" ]]; then
-          echo "FATAL ERROR: Ocean increment not found, ABORT!"
+          echo "FATAL ERROR: Ocean increment ${COM_OCEAN_ANALYSIS}/${RUN}.t${cyc}z.ocninc.nc not found, ABORT!"
           exit 111
       fi
       ${NLN} "${COM_OCEAN_ANALYSIS}/${RUN}.t${cyc}z.ocninc.nc" "${DATA}/INPUT/mom6_increment.nc"

ush/hpssarch_gen.sh

@@ -506,12 +506,14 @@ if [[ ${type} == "gdas" ]]; then
 
     [[ -s "${COM_CONF}/${head}letkfoi.yaml" ]] && echo "${COM_CONF/${ROTDIR}\//}/${head}letkfoi.yaml"
 
-    echo "${COM_SNOW_ANALYSIS/${ROTDIR}\//}/*0000.sfc_data.tile1.nc"
-    echo "${COM_SNOW_ANALYSIS/${ROTDIR}\//}/*0000.sfc_data.tile2.nc"
-    echo "${COM_SNOW_ANALYSIS/${ROTDIR}\//}/*0000.sfc_data.tile3.nc"
-    echo "${COM_SNOW_ANALYSIS/${ROTDIR}\//}/*0000.sfc_data.tile4.nc"
-    echo "${COM_SNOW_ANALYSIS/${ROTDIR}\//}/*0000.sfc_data.tile5.nc"
-    echo "${COM_SNOW_ANALYSIS/${ROTDIR}\//}/*0000.sfc_data.tile6.nc"
+    if [[ "${DO_JEDISNOWDA:-}" == "YES" ]]; then
+      echo "${COM_SNOW_ANALYSIS/${ROTDIR}\//}/*0000.sfc_data.tile1.nc"
+      echo "${COM_SNOW_ANALYSIS/${ROTDIR}\//}/*0000.sfc_data.tile2.nc"
+      echo "${COM_SNOW_ANALYSIS/${ROTDIR}\//}/*0000.sfc_data.tile3.nc"
+      echo "${COM_SNOW_ANALYSIS/${ROTDIR}\//}/*0000.sfc_data.tile4.nc"
+      echo "${COM_SNOW_ANALYSIS/${ROTDIR}\//}/*0000.sfc_data.tile5.nc"
+      echo "${COM_SNOW_ANALYSIS/${ROTDIR}\//}/*0000.sfc_data.tile6.nc"
+    fi
   } >> "${DATA}/gdas_restarta.txt"
 
   #..................

workflow/applications/gfs_cycled.py

@@ -43,8 +43,10 @@ def _get_app_configs(self):
             configs += ['anal', 'analdiag']
 
         if self.do_jediocnvar:
-            configs += ['prepoceanobs', 'ocnanalprep', 'ocnanalbmat',
-                        'ocnanalrun', 'ocnanalchkpt', 'ocnanalpost']
+            configs += ['prepoceanobs', 'ocnanalprep', 'ocnanalbmat', 'ocnanalrun']
+            if self.do_hybvar:
+                configs += ['ocnanalecen']
+            configs += ['ocnanalchkpt', 'ocnanalpost']
             if self.do_vrfy_oceanda:
                 configs += ['ocnanalvrfy']
 
@@ -139,9 +141,10 @@ def get_task_names(self):
             gdas_gfs_common_tasks_before_fcst += ['anal']
 
         if self.do_jediocnvar:
-            gdas_gfs_common_tasks_before_fcst += ['prepoceanobs', 'ocnanalprep',
-                                                  'ocnanalbmat', 'ocnanalrun',
-                                                  'ocnanalchkpt', 'ocnanalpost']
+            gdas_gfs_common_tasks_before_fcst += ['prepoceanobs', 'ocnanalprep', 'ocnanalbmat', 'ocnanalrun']
+            if self.do_hybvar:
+                gdas_gfs_common_tasks_before_fcst += ['ocnanalecen']
+            gdas_gfs_common_tasks_before_fcst += ['ocnanalchkpt', 'ocnanalpost']
             if self.do_vrfy_oceanda:
                 gdas_gfs_common_tasks_before_fcst += ['ocnanalvrfy']