Add LOCALARCH option (#628)

Adds the option to archive output locally during archive jobs. This is intended for systems that do not have access to HPSS (e.g. Orion and soon S4), but can be used on any machine. To enable, the LOCALARCH setting in config.base should be set to "YES" (HPSSARCH must be "NO"). When enabled, the tarballs normally created on HPSS will instead be created in the local directory specified by $ATARDIR. Defaults have been added to setup_expt.py to point to a local ATARDIR and LOCALARCH (currently =NO). Fixes #624.
NOAA-EMC · Feb 11, 2022 · f8867d3 · f8867d3
1 parent 2950c7b
commit f8867d3
Show file tree

Hide file tree

Showing 6 changed files with 65 additions and 18 deletions.
diff --git a/jobs/rocoto/arch.sh b/jobs/rocoto/arch.sh
@@ -129,10 +129,18 @@ fi
 
 
 ###############################################################
-# Archive data to HPSS
-if [ $HPSSARCH = "YES" ]; then
+# Archive data either to HPSS or locally
+if [[ $HPSSARCH = "YES" || $LOCALARCH = "YES" ]]; then
 ###############################################################
 
+# --set the archiving command and create local directories, if necessary
+TARCMD="htar"
+if [[ $LOCALARCH = "YES" ]]; then
+   TARCMD="tar"
+   [ ! -d $ATARDIR/$CDATE ] && mkdir -p $ATARDIR/$CDATE
+   [ ! -d $ATARDIR/$CDATE_MOS -a -d $ROTDIR/gfsmos.$PDY_MOS -a $cyc -eq 18 ] && mkdir -p $ATARDIR/$CDATE_MOS
+fi
+
 #--determine when to save ICs for warm start and forecast-only runs 
 SAVEWARMICA="NO"
 SAVEWARMICB="NO"
@@ -208,10 +216,10 @@ if [ $CDUMP = "gfs" ]; then
 
     #--save mdl gfsmos output from all cycles in the 18Z archive directory
     if [ -d gfsmos.$PDY_MOS -a $cyc -eq 18 ]; then
-        htar -P -cvf $ATARDIR/$CDATE_MOS/gfsmos.tar ./gfsmos.$PDY_MOS
+        $TARCMD -P -cvf $ATARDIR/$CDATE_MOS/gfsmos.tar ./gfsmos.$PDY_MOS
         status=$?
         if [ $status -ne 0  -a $CDATE -ge $firstday ]; then
-            echo "HTAR $CDATE gfsmos.tar failed"
+            echo "$(echo $TARCMD | tr 'a-z' 'A-Z') $CDATE gfsmos.tar failed"
             exit $status
         fi
     fi
@@ -240,10 +248,10 @@ fi
 # Turn on extended globbing options
 shopt -s extglob
 for targrp in $targrp_list; do
-    htar -P -cvf $ATARDIR/$CDATE/${targrp}.tar $(cat $ARCH_LIST/${targrp}.txt)
+    $TARCMD -P -cvf $ATARDIR/$CDATE/${targrp}.tar $(cat $ARCH_LIST/${targrp}.txt)
     status=$?
     if [ $status -ne 0 -a $CDATE -ge $firstday ]; then
-        echo "HTAR $CDATE ${targrp}.tar failed"
+        echo "$(echo $TARCMD | tr 'a-z' 'A-Z') $CDATE ${targrp}.tar failed"
         exit $status
     fi
 done

diff --git a/jobs/rocoto/earc.sh b/jobs/rocoto/earc.sh
@@ -60,7 +60,14 @@ cd $ROTDIR
 ###################################################################
 # ENSGRP > 0 archives a group of ensemble members
 firstday=$($NDATE +24 $SDATE)
-if [[ $ENSGRP -gt 0 ]] && [[ $HPSSARCH = "YES" ]]; then
+if [[ $ENSGRP -gt 0 ]] && [[ $HPSSARCH = "YES" || $LOCALARCH = "YES" ]]; then
+
+#--set the archiving command and create local directories, if necessary
+   TARCMD="htar"
+   if [[ $LOCALARCH = "YES" ]]; then
+       TARCMD="tar"
+       [ ! -d $ATARDIR/$CDATE ] && mkdir -p $ATARDIR/$CDATE
+   fi
 
 #--determine when to save ICs for warm start
    SAVEWARMICA="NO"
@@ -84,27 +91,27 @@ if [[ $ENSGRP -gt 0 ]] && [[ $HPSSARCH = "YES" ]]; then
 
    if [ $CDATE -gt $SDATE ]; then # Don't run for first half cycle
 
-     htar -P -cvf $ATARDIR/$CDATE/enkf${CDUMP}_grp${ENSGRP}.tar $(cat $ARCH_LIST/enkf${CDUMP}_grp${n}.txt)
+     $TARCMD -P -cvf $ATARDIR/$CDATE/enkf${CDUMP}_grp${ENSGRP}.tar $(cat $ARCH_LIST/enkf${CDUMP}_grp${n}.txt)
      status=$?
      if [ $status -ne 0  -a $CDATE -ge $firstday ]; then
-         echo "HTAR $CDATE enkf${CDUMP}_grp${ENSGRP}.tar failed"
+         echo "$(echo $TARCMD | tr 'a-z' 'A-Z') $CDATE enkf${CDUMP}_grp${ENSGRP}.tar failed"
          exit $status
      fi
 
      if [ $SAVEWARMICA = "YES" -a $cyc -eq $EARCINC_CYC ]; then
-       htar -P -cvf $ATARDIR/$CDATE/enkf${CDUMP}_restarta_grp${ENSGRP}.tar $(cat $ARCH_LIST/enkf${CDUMP}_restarta_grp${n}.txt)
+       $TARCMD -P -cvf $ATARDIR/$CDATE/enkf${CDUMP}_restarta_grp${ENSGRP}.tar $(cat $ARCH_LIST/enkf${CDUMP}_restarta_grp${n}.txt)
        status=$?
        if [ $status -ne 0 ]; then
-           echo "HTAR $CDATE enkf${CDUMP}_restarta_grp${ENSGRP}.tar failed"
+           echo "$(echo $TARCMD | tr 'a-z' 'A-Z') $CDATE enkf${CDUMP}_restarta_grp${ENSGRP}.tar failed"
            exit $status
        fi
      fi
 
      if [ $SAVEWARMICB = "YES"  -a $cyc -eq $EARCICS_CYC ]; then
-       htar -P -cvf $ATARDIR/$CDATE/enkf${CDUMP}_restartb_grp${ENSGRP}.tar $(cat $ARCH_LIST/enkf${CDUMP}_restartb_grp${n}.txt)
+       $TARCMD -P -cvf $ATARDIR/$CDATE/enkf${CDUMP}_restartb_grp${ENSGRP}.tar $(cat $ARCH_LIST/enkf${CDUMP}_restartb_grp${n}.txt)
        status=$?
        if [ $status -ne 0 ]; then
-           echo "HTAR $CDATE enkf${CDUMP}_restartb_grp${ENSGRP}.tar failed"
+           echo "$(echo $TARCMD | tr 'a-z' 'A-Z') $CDATE enkf${CDUMP}_restartb_grp${ENSGRP}.tar failed"
            exit $status
        fi
      fi
@@ -118,12 +125,19 @@ fi
 # ENSGRP 0 archives ensemble means and copy data to online archive
 if [ $ENSGRP -eq 0 ]; then
 
-    if [ $HPSSARCH = "YES" ]; then
+    if [[ $HPSSARCH = "YES" || $LOCALARCH = "YES" ]]; then
+
+#--set the archiving command and create local directories, if necessary
+        TARCMD="htar"
+        if [[ $LOCALARCH = "YES" ]]; then
+            TARCMD="tar"
+            [ ! -d $ATARDIR/$CDATE ] && mkdir -p $ATARDIR/$CDATE
+        fi
 
-        htar -P -cvf $ATARDIR/$CDATE/enkf${CDUMP}.tar $(cat $ARCH_LIST/enkf${CDUMP}.txt)
+        $TARCMD -P -cvf $ATARDIR/$CDATE/enkf${CDUMP}.tar $(cat $ARCH_LIST/enkf${CDUMP}.txt)
         status=$?
         if [ $status -ne 0  -a $CDATE -ge $firstday ]; then
-            echo "HTAR $CDATE enkf${CDUMP}.tar failed"
+            echo "$(echo $TARCMD | tr 'a-z' 'A-Z') $CDATE enkf${CDUMP}.tar failed"
             exit $status
         fi
     fi

diff --git a/parm/config/config.base.emc.dyn b/parm/config/config.base.emc.dyn
@@ -115,7 +115,7 @@ export RUNDIR="$STMP/RUNDIRS/$PSLOT"
 export DATAROOT="$RUNDIR/$CDATE/$CDUMP"
 export ARCDIR="$NOSCRUB/archive/$PSLOT"
 export ICSDIR="@ICSDIR@"
-export ATARDIR="/NCEPDEV/$HPSS_PROJECT/1year/$USER/$machine/scratch/$PSLOT"
+export ATARDIR="@ATARDIR@"
 
 # Commonly defined parameters in JJOBS
 export envir=${envir:-"prod"}
@@ -350,6 +350,11 @@ export DO_VSDB="YES"         # Run VSDB package - set VSDB settings in config.vr
 
 # Archiving options
 export HPSSARCH="@HPSSARCH@"        # save data to HPSS archive
+export LOCALARCH="@LOCALARCH@"        # save data to local archive
+if [[ $HPSSARCH = "YES" ]] && [[ $LOCALARCH = "YES" ]]; then
+   echo "Both HPSS and local archiving selected.  Please choose one or the other."
+   exit 2
+fi
 export ARCH_CYC=00           # Archive data at this cycle for warm_start capability
 export ARCH_WARMICFREQ=4     # Archive frequency in days for warm_start capability
 export ARCH_FCSTICFREQ=1     # Archive frequency in days for gdas and gfs forecast-only capability

diff --git a/ush/hpssarch_gen.sh b/ush/hpssarch_gen.sh
@@ -456,10 +456,18 @@ if [ $type = "enkfgdas" -o $type = "enkfgfs" ]; then
         fi
      fi 
   done # loop over FHR
-  for fstep in eobs eomg ecen esfc eupd efcs epos ; do
+  for fstep in eobs ecen esfc eupd efcs epos ; do
    echo  "logs/${CDATE}/${CDUMP}${fstep}*.log        " >>enkf${CDUMP}.txt
   done
 
+# eomg* are optional jobs
+  for log in $ROTDIR/logs/${CDATE}/${CDUMP}eomg*.log; do
+     if [ -s "$log" ]; then
+        echo  "logs/${CDATE}/${CDUMP}eomg*.log        " >>enkf${CDUMP}.txt
+     fi
+     break
+  done
+
 
 # Ensemble spread file only available with netcdf output
   fh=3

diff --git a/ush/rocoto/setup_expt.py b/ush/rocoto/setup_expt.py
@@ -140,6 +140,8 @@ def edit_baseconfig(host, inputs):
         "@CHGRP_RSTPROD@": host.info["chgrp_rstprod"],
         "@CHGRP_CMD@": host.info["chgrp_cmd"],
         "@HPSSARCH@": host.info["hpssarch"],
+        "@LOCALARCH@": host.info["localarch"],
+        "@ATARDIR@": host.info["atardir"],
         "@gfs_cyc@": inputs.gfs_cyc,
         "@APP@": inputs.app,
     }

diff --git a/ush/rocoto/workflow_utils.py b/ush/rocoto/workflow_utils.py
@@ -78,6 +78,8 @@ def wcoss_c(self):
             'chgrp_rstprod': 'YES',
             'chgrp_cmd': 'chgrp rstprod',
             'hpssarch': 'YES',
+            'localarch': 'NO',
+            'atardir': '/NCEPDEV/$HPSS_PROJECT/1year/$USER/$machine/scratch/$PSLOT',
              }
 
         return info
@@ -102,6 +104,8 @@ def wcoss_dell_p3(self):
             'chgrp_rstprod': 'YES',
             'chgrp_cmd': 'chgrp rstprod',
             'hpssarch': 'YES',
+            'localarch': 'NO',
+            'atardir': '/NCEPDEV/$HPSS_PROJECT/1year/$USER/$machine/scratch/$PSLOT',
              }
 
         return info
@@ -125,6 +129,8 @@ def wcoss_dell_p3p5(self):
             'chgrp_rstprod': 'YES',
             'chgrp_cmd': 'chgrp rstprod',
             'hpssarch': 'YES',
+            'localarch': 'NO',
+            'atardir': '/NCEPDEV/$HPSS_PROJECT/1year/$USER/$machine/scratch/$PSLOT',
              }
 
         return info
@@ -149,6 +155,8 @@ def hera(self):
             'chgrp_rstprod': 'YES',
             'chgrp_cmd': 'chgrp rstprod',
             'hpssarch': 'YES',
+            'localarch': 'NO',
+            'atardir': '/NCEPDEV/$HPSS_PROJECT/1year/$USER/$machine/scratch/$PSLOT',
              }
 
         return info
@@ -173,6 +181,8 @@ def orion(self):
             'chgrp_rstprod': 'YES',
             'chgrp_cmd': 'chgrp rstprod',
             'hpssarch': 'NO',
+            'localarch': 'NO',
+            'atardir': '$NOSCRUB/archive_rotdir/$PSLOT',
              }
 
         return info