Update testreporter and change hobart queue to medium.

config/cesm/machines/config_batch.xml

@@ -232,7 +232,8 @@
       <directive default="/bin/bash" > -S {{ shell }}  </directive>
     </directives>
     <queues>
-      <queue walltimemax="02:00:00" jobmin="1" jobmax="192" default="true">short</queue>
+      <queue walltimemax="02:00:00" jobmin="1" jobmax="192" >short</queue>
+      <queue walltimemax="06:00:00" jobmin="1" jobmax="192" default="true">medium</queue>
       <queue walltimemax="40:00:00" jobmin="1" jobmax="144" >long</queue>
     </queues>
   </batch_system>

scripts/Tools/testreporter.py

@@ -181,13 +181,31 @@ def get_testreporter_xml(testroot, testid, tagname, testtype):
                 if line[0:4] == "FAIL":
                     test_status['COMMENT']+="Hybrid fail! "
                     break
+            if "COMPARE_base_multiinst" in line:
+                test_status['STATUS']=line[0:4]
+                if line[0:4] == "FAIL":
+                    test_status['COMMENT']+="Multi instance fail! "
+                    break
+            if "COMPARE_base_test" in line:
+                test_status['STATUS']=line[0:4]
+                if line[0:4] == "FAIL":
+                    test_status['COMMENT']+="Base test fail! "
+                    break
+            if "COMPARE_base_single_thread" in line:
+                test_status['STATUS']=line[0:4]
+                if line[0:4] == "FAIL":
+                    test_status['COMMENT']+="Thread test fail! "
+                    break
 
             #
             #  Do not include time comments.  Just a preference to have cleaner comments in the test database
             #
             try:
-                if 'time=' not in line:
-                    test_status['COMMENT']+=line.split(' ',3)[3]+' '
+                if 'time=' not in line and 'GENERATE' not in line:
+                    if 'BASELINE' not in line:
+                        test_status['COMMENT']+=line.split(' ',3)[3]+' '
+                    else:
+                        test_status['COMMENT']+=line.split(' ',4)[4]+' '
             except:
                 pass
 

scripts/lib/CIME/SystemTests/pre.py

@@ -60,7 +60,7 @@ def _case_two_setup(self):
         self._case.set_value("PAUSE_N", pausen)
         comps = [ x.lower() for x in self._case.get_values("COMP_CLASSES") ]
         pcl = self._case.get_value("PAUSE_COMPONENT_LIST")
-        expect(pcl == "all" or set(pcl.split(':')).issubset(comps), 
+        expect(pcl == "all" or set(pcl.split(':')).issubset(comps),
                "PRE ERROR: Invalid PAUSE_COMPONENT_LIST, '%s'"%pcl)
 
         self._case.flush()
@@ -101,7 +101,7 @@ def run_phase(self):
             expect((len(restart_files_2) == 1),
                    "Missing case2 restart file, %s", glob_str)
             rfile2 = restart_files_2[0]
-            ok, out = cprnc(comp, rfile1, rfile2, self._case, rundir2) # pylint: disable=unused-variable
+            ok = cprnc(comp, rfile1, rfile2, self._case, rundir2)[0]
             logger.warning("CPRNC result for %s: %s"%(os.path.basename(rfile1), "PASS" if (ok == should_match) else "FAIL"))
             compare_ok = compare_ok and (should_match == ok)
 

scripts/lib/CIME/hist_utils.py

@@ -178,6 +178,7 @@ def _compare_hists(case, from_dir1, from_dir2, suffix1="", suffix2="", outfile_s
         expect(suffix1 != suffix2, "Comparing files to themselves?")
 
     testcase = case.get_value("CASE")
+    casedir = case.get_value("CASEROOT")
     all_success = True
     num_compared = 0
     comments = "Comparing hists for case '%s' dir1='%s', suffix1='%s',  dir2='%s' suffix2='%s'\n" % \
@@ -204,20 +205,23 @@ def _compare_hists(case, from_dir1, from_dir2, suffix1="", suffix2="", outfile_s
         num_compared += len(match_ups)
 
         for hist1, hist2 in match_ups:
-            success, cprnc_comments = cprnc(model, hist1, hist2, case, from_dir1,
+            success, cprnc_log_file = cprnc(model, hist1, hist2, case, from_dir1,
                                             multiinst_cpl_compare = multiinst_cpl_compare,
                                             outfile_suffix = outfile_suffix)
             if success:
                 comments += "    %s matched %s\n" % (hist1, hist2)
             else:
                 comments += "    %s did NOT match %s\n" % (hist1, hist2)
-                comments += cprnc_comments + "\n"
+                comments += "    cat " + cprnc_log_file + "\n"
+                shutil.copy(cprnc_log_file, casedir)
                 all_success = False
 
     if num_compared == 0:
         all_success = False
         comments += "Did not compare any hist files! Missing baselines?\n"
 
+    comments += "PASS" if all_success else "FAIL"
+
     return all_success, comments
 
 def compare_test(case, suffix1, suffix2):
@@ -245,7 +249,7 @@ def cprnc(model, file1, file2, case, rundir, multiinst_cpl_compare=False, outfil
     outfile_suffix - if non-blank, then the output file name ends with this
         suffix (with a '.' added before the given suffix)
 
-    returns True if the files matched
+    returns (True if the files matched, log_name)
     """
     cprnc_exe = case.get_value("CCSM_CPRNC")
     basename = os.path.basename(file1)
@@ -276,9 +280,9 @@ def cprnc(model, file1, file2, case, rundir, multiinst_cpl_compare=False, outfil
         # dimensions and so cprnc will indicate that the files seem to be DIFFERENT
         # in this case we only want to check that the fields we are able to compare
         # have no differences.
-        return (cpr_stat == 0 and " 0 had non-zero differences" in out, out)
+        return (cpr_stat == 0 and " 0 had non-zero differences" in out, output_filename)
     else:
-        return (cpr_stat == 0 and "files seem to be IDENTICAL" in out, out)
+        return (cpr_stat == 0 and "files seem to be IDENTICAL" in out, output_filename)
 
 def compare_baseline(case, baseline_dir=None, outfile_suffix=""):
     """

scripts/lib/CIME/wait_for_tests.py

@@ -152,14 +152,17 @@ def create_cdash_upload_xml(results, cdash_build_name, cdash_build_group, utc_ti
             if (test_status not in [TEST_PASS_STATUS, NAMELIST_FAIL_STATUS]):
                 ts = TestStatus(os.path.dirname(test_path))
 
-                build_status = ts.get_status(MODEL_BUILD_PHASE)
-                run_status   = ts.get_status(RUN_PHASE)
-                if ( build_status == TEST_FAIL_STATUS or run_status == TEST_FAIL_STATUS ):
+                build_status    = ts.get_status(MODEL_BUILD_PHASE)
+                run_status      = ts.get_status(RUN_PHASE)
+                baseline_status = ts.get_status(BASELINE_PHASE)
+                if ( build_status == TEST_FAIL_STATUS or run_status == TEST_FAIL_STATUS or baseline_status == TEST_FAIL_STATUS):
                     param = "EXEROOT" if build_status == TEST_FAIL_STATUS else "RUNDIR"
-                    src_dir = CIME.utils.run_cmd_no_fail("./xmlquery %s -value" % param, from_dir=os.path.dirname(test_path))
+                    log_src_dir = CIME.utils.run_cmd_no_fail("./xmlquery %s -value" % param, from_dir=os.path.dirname(test_path))
                     log_dst_dir = os.path.join(log_dir, "%s_%s_logs" % (test_name, param))
                     os.makedirs(log_dst_dir)
-                    for log_file in glob.glob(os.path.join(src_dir, "*log*")):
+                    for log_file in glob.glob(os.path.join(log_src_dir, "*log*")):
+                        shutil.copy(log_file, log_dst_dir)
+                    for log_file in glob.glob(os.path.join(log_src_dir, "*.cprnc.out*")):
                         shutil.copy(log_file, log_dst_dir)
 
                     need_to_upload = True