Reusing test suites (#310)

* Modify cice.setup so that test suites can use incremental compilation

* Update documentation

* Rename incremental option to buildincremental, add buildclean option

* Update documentation

cice.setup

@@ -380,9 +380,14 @@ else
   cp -f ${ICE_SCRIPTS}/tests/timeseries.csh ${tsdir}
   cp -f ${ICE_SCRIPTS}/tests/poll_queue.csh ${tsdir}
 
-cat >! ${tsdir}/suite.run << EOF0
+  foreach file (${tsdir}/suite.run ${tsdir}/suite.submit)
+  cat >! $file << EOF0
 #!/bin/csh -f
+
+set nonomatch && rm -f ciceexe.* && unset nonomatch
+
 EOF0
+  end
 
 cat >! ${tsdir}/results.csh << EOF0
 #!/bin/csh -f
@@ -403,6 +408,7 @@ echo "#------- " >> results.log
 EOF0
 
   chmod +x ${tsdir}/suite.run
+  chmod +x ${tsdir}/suite.submit
   chmod +x ${tsdir}/results.csh
 
 endif
@@ -439,7 +445,7 @@ foreach compiler ( $ncompilers )
       continue
     # Check if line is a sleep line, can only happen with suites
     else if ( $line:q =~ 'sleep'*) then
-cat >> ${tsdir}/suite.run << EOF
+cat >> ${tsdir}/suite.submit << EOF
 echo "-------test--------------"
 echo "$line:q"
 $line:q
@@ -829,24 +835,34 @@ EOF2
 
     if ( ${dosuite} == 1 ) then
       cd ${ICE_SANDBOX}
-      # Write build and run commands to suite.run
+      # Write build and run commands to suite.run and suite.submit
 
 cat >> ${tsdir}/results.csh << EOF
 cat ${testname_base}/test_output >> results.log
 EOF
-
-cat >> ${tsdir}/suite.run << EOF
+  foreach file (${tsdir}/suite.run ${tsdir}/suite.submit)
+    cat >> $file  << EOF
 echo "-------test--------------"
 echo "${testname_base}"
 cd ${testname_base}
 source ./cice.settings
 set ciceexe = "../ciceexe.\${ICE_COMPILER}.\${ICE_COMMDIR}.\${ICE_BLDDEBUG}.\${ICE_THREADED}.\${ICE_IOTYPE}"
 ./cice.build \${ciceexe}
 if !(-e \${ciceexe}) cp -p \${ICE_RUNDIR}/cice \${ciceexe}
+EOF
+  end
+  cat >> ${tsdir}/suite.submit << EOF
 ./cice.submit | tee -a ../suite.jobs
+EOF
+  cat >> ${tsdir}/suite.run << EOF
+./cice.test
+EOF
+  foreach file (${tsdir}/suite.run ${tsdir}/suite.submit)
+    cat >> $file << EOF
 cd ..
 
 EOF
+  end
     
       # Reset case for the next test in suite
       set case = ${spval}
@@ -905,7 +921,7 @@ EOF0
 
   # build and submit tests
   cd ${tsdir}
-  ./suite.run | tee suite.log
+  ./suite.submit | tee suite.log
   if ($report == 1) then
     echo "Reporting results"
     ./poll_queue.csh

configuration/scripts/options/set_env.buildclean

@@ -0,0 +1 @@
+setenv ICE_CLEANBUILD true

configuration/scripts/options/set_env.buildincremental

@@ -0,0 +1 @@
+setenv ICE_CLEANBUILD false

doc/source/user_guide/ug_running.rst

@@ -80,9 +80,9 @@ Some hints:
 - To change the block sizes required at build time, edit the **cice.settings** file.
 - To change namelist, manually edit the **ice_in** file
 - To change batch settings, manually edit the top of the **cice.run** or **cice.test** (if running a test) file
-- To turn on the debug compiler flags, set ``ICE_BLDDEBUG`` in **cice.setttings** to true
+- To turn on the debug compiler flags, set ``ICE_BLDDEBUG`` in **cice.setttings** to true.  It is also possible to use the ``debug`` option  (``-s debug``) when creating the case with **cice.setup** to set this option automatically.
 - To change compiler options, manually edit the Macros file
-- To clean the build before each compile, set ``ICE_CLEANBUILD`` in **cice.settings** to true.  To not clean before the build, set ``ICE_CLEANBUILD`` in **cice.settings** to false
+- To clean the build before each compile, set ``ICE_CLEANBUILD`` in **cice.settings** to true (this is the default value), or use the ``buildclean`` option (``-s buildclean``)  when creating the case with **cice.setup**.  To not clean before the build, set ``ICE_CLEANBUILD`` in **cice.settings** to false, or use the ``buildincremental`` option  (``-s buildincremental``) when creating the case with **cice.setup**.
 
 To build and run::
 
@@ -164,6 +164,10 @@ Some of the options are
 
 ``debug`` which turns on the compiler debug flags
 
+``buildclean`` which turns on the option to clean the build before each compile
+
+``buildincremental`` which turns off the option to clean the build before each compile
+
 ``short``, ``medium``, ``long`` which change the batch time limit
 
 ``gx3``, ``gx1``, ``tx1`` are associate with grid specific settings

doc/source/user_guide/ug_testing.rst

@@ -393,7 +393,7 @@ Test Suite Examples
 
      ./cice.setup --suite base_suite --mach conrad --env cray --testid v01a
      cd testsuite.v01a
-     #wait for runs to complete
+     # wait for runs to complete
      ./results.csh
 
  2) **Basic test suite with user defined test directory**
@@ -403,7 +403,7 @@ Test Suite Examples
 
      ./cice.setup --suite base_suite --mach conrad --env cray --testid v01a --tdir /scratch/$user/ts.v01a
      cd /scratch/$user/ts.v01a
-     #wait for runs to complete
+     # wait for runs to complete
      ./results.csh
 
  3) **Basic test suite on multiple environments**
@@ -413,7 +413,7 @@ Test Suite Examples
 
       ./cice.setup --suite base_suite --mach conrad --env cray,pgi,intel,gnu --testid v01a
       cd testsuite.v01a
-      #wait for runs to complete
+      # wait for runs to complete
       ./results.csh
 
     Each env can be run as a separate invokation of `cice.setup` but if that
@@ -426,7 +426,7 @@ Test Suite Examples
 
        ./cice.setup --suite base_suite --mach conrad --env gnu --testid v01b --set diag1
        cd testsuite.v01b
-       #wait for runs to complete
+       # wait for runs to complete
       ./results.csh
 
     If there are conflicts between the ``--set`` options in the suite and on the command line,
@@ -439,11 +439,11 @@ Test Suite Examples
 
       ./cice.setup --suite base_suite,decomp_suite --mach conrad --env gnu --testid v01c
       cd testsuite.v01c
-      #wait for runs to complete
+      # wait for runs to complete
       ./results.csh
 
-     If there are redundant tests in multiple suites, the scripts will understand that and only
-     create one test.
+    If there are redundant tests in multiple suites, the scripts will understand that and only
+    create one test.
 
  6) **Basic test suite, store baselines in user defined name**
 
@@ -452,10 +452,10 @@ Test Suite Examples
 
       ./cice.setup --suite base_suite --mach conrad --env cray --testid v01a --bgen cice.v01a
       cd testsuite.v01a
-      #wait for runs to complete
+      # wait for runs to complete
       ./results.csh
 
-     This will store the results in the default [bdir] directory under the subdirectory cice.v01a.
+    This will store the results in the default [bdir] directory under the subdirectory cice.v01a.
 
  7) **Basic test suite, store baselines in user defined top level directory**
 
@@ -464,7 +464,7 @@ Test Suite Examples
 
       ./cice.setup --suite base_suite --mach conrad --env cray --testid v01a --bgen cice.v01a --bdir /tmp/user/CICE_BASELINES
       cd testsuite.v01a
-      #wait for runs to complete
+      # wait for runs to complete
       ./results.csh
 
     This will store the results in /tmp/user/CICE_BASELINES/cice.v01a.
@@ -476,11 +476,10 @@ Test Suite Examples
 
       ./cice.setup --suite base_suite --mach conrad --env cray --testid v01a --bgen default
       cd testsuite.v01a
-      #wait for runs to complete
+      # wait for runs to complete
       ./results.csh
 
-     This will store the results in the default [bdir] directory under a directory name generated by the script
-     that includes the hash and date.
+    This will store the results in the default [bdir] directory under a directory name generated by the script that includes the hash and date.
 
  9) **Basic test suite, compare to prior baselines**
 
@@ -489,7 +488,7 @@ Test Suite Examples
 
       ./cice.setup --suite base_suite --mach conrad --env cray --testid v02a --bcmp cice.v01a
       cd testsuite.v02a
-      #wait for runs to complete
+      # wait for runs to complete
       ./results.csh
 
     This will compare to results saved in the baseline [bdir] directory under
@@ -526,7 +525,23 @@ Test Suite Examples
     When this is invoked, a new set of baselines will be generated and compared to the prior
     results each time without having to change the arguments.
 
- 11) **Create and test a custom suite**
+ 11) **Reusing a test suite**
+
+    Add the buildincremental option (``-s buildincremental``). This permits the suite to be rerun without recompiling the whole code.
+    ::
+
+      ./cice.setup --suite base_suite --mach conrad --env intel --testid v01b --set buildincremental
+      cd testsuite.v01b
+      # wait for runs to complete
+      ./results.csh
+      # modify code
+      ./suite.submit # or ./suite.run to run the suite interactively
+      # wait for runs to complete
+      ./results.csh
+
+    Only modified files will be recompiled, and the suite will be rerun.
+
+ 12) **Create and test a custom suite**
 
     Create your own input text file consisting of 5 columns of data,
      - Test
@@ -549,7 +564,7 @@ Test Suite Examples
 
       ./cice.setup --suite mysuite --mach conrad --env cray --testid v01a --bgen default
       cd testsuite.v01a
-      #wait for runs to complete
+      # wait for runs to complete
       ./results.csh
 
     You can use all the standard regression testing options (``--bgen``, ``--bcmp``,