Add nwges directory (NOAA-EMC#176)

* Add NWGES space to save all boundary, initial, restart files.
Each cycle will search for nwges directory for boundary, initial and restart files.

jobs/JREGIONAL_MAKE_ICS

@@ -59,12 +59,24 @@ mkdir_vrfy -p "${ics_dir}"
 #
 #-----------------------------------------------------------------------
 #
+# Set the name of and create the directory in which the output from this
+# script will be saved for long time (if that directory doesn't already exist).
+# Usually ics_dir under tmpnwprd should be purged fairly soon after cycle done
+#
+#-----------------------------------------------------------------------
+#
+ics_nwges_dir="${NWGES_DIR}${SLASH_ENSMEM_SUBDIR}/ics"
+mkdir_vrfy -p "${ics_nwges_dir}"
+#
+#-----------------------------------------------------------------------
+#
 # Call the ex-script for this J-job and pass to it the necessary variables.
 #
 #-----------------------------------------------------------------------
 #
 $SCRIPTSDIR/exregional_make_ics.sh \
-  ics_dir="${ics_dir}" || \
+  ics_dir="${ics_dir}" \
+  ics_nwges_dir="${ics_nwges_dir}" || \
 print_err_msg_exit "\
 Call to ex-script corresponding to J-job \"${scrfunc_fn}\" failed."
 #

jobs/JREGIONAL_MAKE_LBCS

@@ -57,6 +57,18 @@ hour zero).
 lbcs_dir="${CYCLE_DIR}${SLASH_ENSMEM_SUBDIR}/lbcs"
 mkdir_vrfy -p "${lbcs_dir}"
 
+#
+#-----------------------------------------------------------------------
+#
+# Set the name of and create the directory in which the output from this
+# script will be saved for a long time (if it doesn't already exist).
+# Usually lbcs_dir under tmpnwprd should be purged fairly soon after cycle done
+#
+#-----------------------------------------------------------------------
+#
+lbcs_nwges_dir="${NWGES_DIR}${SLASH_ENSMEM_SUBDIR}/lbcs"
+mkdir_vrfy -p "${lbcs_nwges_dir}"
+
 #
 #-----------------------------------------------------------------------
 #
@@ -65,7 +77,8 @@ mkdir_vrfy -p "${lbcs_dir}"
 #-----------------------------------------------------------------------
 #
 $SCRIPTSDIR/exregional_make_lbcs.sh \
-  lbcs_dir="${lbcs_dir}" || \
+  lbcs_dir="${lbcs_dir}" \
+  lbcs_nwges_dir="${lbcs_nwges_dir}" || \
 print_err_msg_exit "\
 Call to ex-script corresponding to J-job \"${scrfunc_fn}\" failed."
 #

jobs/JREGIONAL_RUN_POST

@@ -64,11 +64,20 @@ on the output files corresponding to a specified forecast hour.
 #   spinup: this job is for spin-up cycles
 #   others(prod): this job is for product cycle
 #
+# NWGES_DIR is the directory in which the model restart file will be saved
+#
 CYCLE_TYPE=${CYCLE_TYPE:-prod}
 if [ ${CYCLE_TYPE} == "spinup" ]; then
   run_dir="${CYCLE_DIR}/fcst_fv3lam_spinup"
+  nwges_dir="${NWGES_DIR}/fcst_fv3lam_spinup"
 else
   run_dir="${CYCLE_DIR}/fcst_fv3lam${SLASH_ENSMEM_SUBDIR}"
+  nwges_dir="${NWGES_DIR}/fcst_fv3lam${SLASH_ENSMEM_SUBDIR}"
+fi
+
+if [ "${RUN_ENVIR}" = "nco" ]; then
+   mkdir_vrfy -p "${nwges_dir}/RESTART"
+   mkdir_vrfy -p "${nwges_dir}/INPUT"
 fi
 
 #
@@ -133,11 +142,13 @@ fi
 $SCRIPTSDIR/exregional_run_post.sh \
   cdate="${CDATE}" \
   run_dir="${run_dir}" \
+  nwges_dir="${nwges_dir}" \
   postprd_dir="${postprd_dir}" \
   comout="${COMOUT}" \
   fhr_dir="${fhr_dir}" \
   fhr="${fhr}" \
-  tmmark="${TMMARK}" || \
+  tmmark="${TMMARK}" \
+  cycle_type="${CYCLE_TYPE}" || \
 print_err_msg_exit "\
 Call to ex-script corresponding to J-job \"${scrfunc_fn}\" failed."
 #

scripts/exregional_make_ics.sh

@@ -57,6 +57,7 @@ This is the ex-script for the task that generates initial condition
 #
 valid_args=( \
 "ics_dir" \
+"ics_nwges_dir" \
 )
 process_args valid_args "$@"
 #
@@ -666,6 +667,17 @@ The following variables were being used:
   fvcom_exe_dir = \"${fvcom_exe_dir}\"
   fvcom_exe = \"${fvcom_exe}\""
 fi
+
+#
+#-----------------------------------------------------------------------
+#
+# copy results to nwges for longe time disk storage.
+#
+#-----------------------------------------------------------------------
+#
+
+cp_vrfy ${ics_dir}/*.nc ${ics_nwges_dir}/.
+
 #
 #-----------------------------------------------------------------------
 #

scripts/exregional_make_lbcs.sh

@@ -57,6 +57,7 @@ hour zero).
 #
 valid_args=( \
 "lbcs_dir" \
+"lbcs_nwges_dir" \
 )
 process_args valid_args "$@"
 #
@@ -539,6 +540,17 @@ located in the following directory:
   mv_vrfy gfs.bndy.nc ${lbcs_dir}/gfs_bndy.tile7.${fcst_hhh_FV3LAM}.nc
 
 done
+
+#
+#-----------------------------------------------------------------------
+#
+# copy results to nwges for longe time disk storage.
+#
+#-----------------------------------------------------------------------
+#
+
+cp_vrfy ${lbcs_dir}/gfs_bndy.tile7.*.nc ${lbcs_nwges_dir}/.
+
 #
 #-----------------------------------------------------------------------
 #

scripts/exregional_run_fcst.sh

@@ -125,7 +125,11 @@ case $MACHINE in
     ulimit -s unlimited
     ulimit -a
     APRUN="srun"
-    OMP_NUM_THREADS=4
+    if [${PREDEF_GRID_NAME} == "RRFS_NA_3km" ]; then
+      OMP_NUM_THREADS=4
+    else
+      OMP_NUM_THREADS=2
+    fi
     ;;
 
   "ODIN")

scripts/exregional_run_post.sh

@@ -58,11 +58,13 @@ the output files corresponding to a specified forecast hour.
 valid_args=( \
 "cdate" \
 "run_dir" \
+"nwges_dir" \
 "postprd_dir" \
 "comout" \
 "fhr_dir" \
 "fhr" \
 "tmmark" \
+"cycle_type" \
 )
 process_args valid_args "$@"
 #
@@ -240,6 +242,51 @@ ${phy_file}
  KPO=47,PO=1000.,975.,950.,925.,900.,875.,850.,825.,800.,775.,750.,725.,700.,675.,650.,625.,600.,575.,550.,525.,500.,475.,450.,425.,400.,375.,350.,325.,300.,275.,250.,225.,200.,175.,150.,125.,100.,70.,50.,30.,20.,10.,7.,5.,3.,2.,1.,
  /
 EOF
+
+# 
+#-----------------------------------------------------------------------
+#
+# Let save the restart files if needed before run post.
+# This part will copy or move restart files matching the forecast hour
+# this post will process to the nwges directory. The nwges is used to 
+# stage the restart files for a long time. 
+#-----------------------------------------------------------------------
+#
+filelist="coupler.res fv_core.res.nc fv_core.res.tile1.nc fv_srf_wnd.res.tile1.nc fv_tracer.res.tile1.nc phy_data.nc sfc_data.nc"
+restart_prefix=${post_yyyy}${post_mm}${post_dd}.${post_hh}0000
+if [ ! -r ${nwges_dir}/INPUT/gfs_ctrl.nc ]; then
+    cp_vrfy $run_dir/INPUT/gfs_ctrl.nc ${nwges_dir}/INPUT/gfs_ctrl.nc
+fi
+if [ -r "$run_dir/RESTART/${restart_prefix}.fv_core.res.tile1.nc" ]; then
+  for file in ${filelist}; do
+    cp_vrfy $run_dir/RESTART/${restart_prefix}.${file} ${nwges_dir}/RESTART/${restart_prefix}.${file}
+  done
+  echo " ${fhr} forecast from ${yyyymmdd}${hh} is ready " #> ${nwges_dir}/RESTART/restart_done_f${fhr}
+else
+
+  FCST_LEN_HRS_thiscycle=${FCST_LEN_HRS}
+  if [ ${cycle_type} == "spinup" ]; then
+    FCST_LEN_HRS_thiscycle=${FCST_LEN_HRS_SPINUP}
+  else
+    num_fhrs=( "${#FCST_LEN_HRS_CYCLES[@]}" )
+    ihh=`expr ${hh} + 0`
+    if [ ${num_fhrs} -gt ${ihh} ]; then
+       FCST_LEN_HRS_thiscycle=${FCST_LEN_HRS_CYCLES[${ihh}]}
+    fi
+  fi
+  print_info_msg "$VERBOSE" " The forecast length for cycle (\"${hh}\") is
+                 ( \"${FCST_LEN_HRS_thiscycle}\") "
+
+  if [ -r "$run_dir/RESTART/fv_core.res.tile1.nc" ] && [ ${fhr} -eq ${FCST_LEN_HRS_thiscycle} ] ; then
+    for file in ${filelist}; do
+      cp_vrfy $run_dir/RESTART/${file} ${nwges_dir}/RESTART/${restart_prefix}.${file}
+    done
+    echo " ${fhr} forecast from ${yyyymmdd}${hh} is ready " #> ${nwges_dir}/RESTART/restart_done_f${fhr}
+  else
+    echo "This forecast hour does not need to save restart: ${yyyymmdd}${hh}f${fhr}"
+  fi
+fi
+#
 #
 #-----------------------------------------------------------------------
 #

scripts/exregional_run_prepstart.sh

@@ -128,7 +128,7 @@ fi
 cd_vrfy ${modelinputdir}
 
 if [ ${BKTYPE} -eq 1 ] ; then  # cold start, use prepare cold strat initial files from ics
-    bkpath=${cycle_dir}/ics
+    bkpath=${lbcs_root}/$YYYYMMDD$HH/ics
     if [ -r "${bkpath}/gfs_data.tile7.halo0.nc" ]; then
       cp_vrfy ${bkpath}/gfs_bndy.tile7.000.nc gfs_bndy.tile7.000.nc        
       cp_vrfy ${bkpath}/gfs_ctrl.nc gfs_ctrl.nc        
@@ -158,9 +158,6 @@ if [ ${BKTYPE} -eq 1 ] ; then  # cold start, use prepare cold strat initial file
            n=$((n + ${DA_CYCLE_INTERV}))
            YYYYMMDDHHmInterv=$( date +%Y%m%d%H -d "${START_DATE} ${n} hours ago" )
            bkpath=${fg_root}/${YYYYMMDDHHmInterv}/${surface_file_dir_name}/RESTART  # cycling, use background from RESTART
-           if [ ${n} -eq ${FCST_LEN_HRS} ]; then
-             restart_prefix=""
-           fi
            print_info_msg "$VERBOSE" "Trying this path: ${bkpath}"
          fi
       done
@@ -213,12 +210,6 @@ else
       n=$((n + ${DA_CYCLE_INTERV}))
       YYYYMMDDHHmInterv=$( date +%Y%m%d%H -d "${START_DATE} ${n} hours ago" )
       bkpath=${fg_root}/${YYYYMMDDHHmInterv}/${fg_restart_dirname}/RESTART  # cycling, use background from RESTART
-      if [ ${n} -eq ${FCST_LEN_HRS_SPINUP} ] && [ ${cycle_type} == "spinup" ]; then
-        restart_prefix=""
-      fi
-      if [ ${n} -eq ${FCST_LEN_HRS} ] && [ ${cycle_type} == "prod" ] ; then
-        restart_prefix=""
-      fi
       print_info_msg "$VERBOSE" "Trying this path: ${bkpath}"
     fi
   done

ush/config.sh.3DRTMA_dev1

@@ -233,6 +233,7 @@ COMINgfs=""
 
 STMP="/lfs4/BMC/nrtrr/NCO_dirs/stmp"  # Path to directory STMP that mostly contains input files.
 PTMP="/lfs4/BMC/nrtrr/NCO_dirs/ptmp"  # Path to directory STMP that mostly contains input files.
+NWGES="/lfs4/BMC/nrtrr/NCO_dirs/nwges"  # Path to directory NWGES that save boundary, cold initial, restart files
 
 if [[ $DO_RETRO == "true" ]] ; then
   CLEAN_OLDPROD_HRS="240"

ush/config.sh.RRFS_AK_dev1

@@ -159,6 +159,7 @@ COMINgfs=""
 
 STMP="/lfs4/BMC/nrtrr/NCO_dirs/stmp"  # Path to directory STMP that mostly contains input files.
 PTMP="/lfs4/BMC/nrtrr/NCO_dirs/ptmp"  # Path to directory STMP that mostly contains input files.
+NWGES="/lfs4/BMC/nrtrr/NCO_dirs/nwges"  # Path to directory NWGES that save boundary, cold initial, restart files
 
 if [[ $DO_RETRO == "true" ]] ; then
   CLEAN_OLDPROD_HRS="240"