Merge pull request #437 from ESMCI/jgfouca/fix_longstanding_nightly_fail

Ensure exceptions are added to TestStatus.log
E3SM-Project · Aug 22, 2016 · 4fb5c77 · 4fb5c77
2 parents 38ab9f1 + 8b8cbef
commit 4fb5c77
Show file tree

Hide file tree

Showing 4 changed files with 25 additions and 14 deletions.
diff --git a/utils/python/CIME/SystemTests/system_tests_common.py b/utils/python/CIME/SystemTests/system_tests_common.py
@@ -90,8 +90,9 @@ def build(self, sharedlib_only=False, model_only=False):
                                      model_only=(phase_name==MODEL_BUILD_PHASE))
                 except:
                     success = False
-                    logger.warning("Exception during build:\n%s" % (sys.exc_info()[1]))
-                    logger.warning(traceback.format_exc())
+                    excmsg = "Exception during build:\n%s\n%s" % (sys.exc_info()[1], traceback.format_exc())
+                    logger.warning(excmsg)
+                    append_status(excmsg, sfile="TestStatus.log")
 
                 time_taken = time.time() - start_time
                 with self._test_status:
@@ -147,8 +148,9 @@ def run(self):
 
         except:
             success = False
-            logger.warning("Exception during run:\n%s" % (sys.exc_info()[1]))
-            logger.warning(traceback.format_exc())
+            excmsg = "Exception during run:\n%s\n%s" % (sys.exc_info()[1], traceback.format_exc())
+            logger.warning(excmsg)
+            append_status(excmsg, sfile="TestStatus.log")
 
         # Always try to report, should NOT throw an exception
         self.report()

diff --git a/utils/python/CIME/case_test.py b/utils/python/CIME/case_test.py
@@ -3,9 +3,11 @@
 """
 
 from CIME.XML.standard_module_setup import *
-from CIME.utils import expect, find_system_test
+from CIME.utils import expect, find_system_test, append_status
 from CIME.SystemTests.system_tests_common import *
 
+import sys
+
 def case_test(case, testname=None):
     if testname is None:
         testname = case.get_value('TESTCASE')
@@ -23,6 +25,7 @@ def case_test(case, testname=None):
         caseroot = case.get_value("CASEROOT")
         with TestStatus(test_dir=caseroot) as ts:
             ts.set_status(RUN_PHASE, TEST_FAIL_STATUS, comments="failed to initialize")
+        append_status(sys.exc_info()[1], sfile="TestStatus.log")
         raise
 
     success = test.run()

diff --git a/utils/python/CIME/test_scheduler.py b/utils/python/CIME/test_scheduler.py
@@ -348,9 +348,12 @@ def _shell_cmd_for_phase(self, test, cmd, phase, from_dir=None):
                 else:
                     break
             else:
-                self._log_output(test,
-                                 "%s PASSED for test '%s'.\nCommand: %s\nOutput: %s\n\nErrput: %s" %
-                                 (phase, test, cmd, output, errput))
+                # We don't want "RUN PASSED" in the TestStatus.log if the only thing that
+                # succeeded was the submission.
+                if phase != RUN_PHASE or self._no_batch:
+                    self._log_output(test,
+                                     "%s PASSED for test '%s'.\nCommand: %s\nOutput: %s\n\nErrput: %s" %
+                                     (phase, test, cmd, output, errput))
                 break
 
         return rc == 0

diff --git a/utils/python/tests/scripts_regression_tests.py b/utils/python/tests/scripts_regression_tests.py
@@ -618,18 +618,21 @@ def test_b_full(self):
             log_file = log_files[0]
             if (test_name == build_fail_test):
                 self.assertEqual(ts.get_status(CIME.test_scheduler.MODEL_BUILD_PHASE), TEST_FAIL_STATUS)
-                self.assertTrue("Intentional fail for testing infrastructure" in open(log_file, "r").read(),
-                                "Broken test did not report build error")
+                data = open(log_file, "r").read()
+                self.assertTrue("Intentional fail for testing infrastructure" in data,
+                                "Broken test did not report build error:\n%s" % data)
             elif (test_name == build_fail_exc_test):
+                data = open(log_file, "r").read()
                 self.assertEqual(ts.get_status(CIME.test_scheduler.SHAREDLIB_BUILD_PHASE), TEST_FAIL_STATUS)
-                self.assertTrue("Exception from init" in open(log_file, "r").read(),
-                                "Broken test did not report build error")
+                self.assertTrue("Exception from init" in data,
+                                "Broken test did not report build error:\n%s" % data)
             elif (test_name == run_fail_test):
                 self.assertEqual(ts.get_status(CIME.test_scheduler.RUN_PHASE), TEST_FAIL_STATUS)
             elif (test_name == run_fail_exc_test):
                 self.assertEqual(ts.get_status(CIME.test_scheduler.RUN_PHASE), TEST_FAIL_STATUS)
-                self.assertTrue("Exception from run_phase" in open(log_file, "r").read(),
-                                "Broken test did not report build error")
+                data = open(log_file, "r").read()
+                self.assertTrue("Exception from run_phase" in data,
+                                "Broken test did not report run error:\n%s" % data)
             elif (test_name == mem_fail_test):
                 self.assertEqual(ts.get_status(MEMLEAK_PHASE), TEST_FAIL_STATUS)
                 self.assertEqual(ts.get_status(CIME.test_scheduler.RUN_PHASE), TEST_PASS_STATUS)