From ba69385d4e3a87f28dd0b5695734c11fba218612 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Fri, 5 Aug 2016 17:17:49 -0600 Subject: [PATCH] progress --- scripts/Tools/case.build | 13 +- scripts/Tools/component_compgen_baseline.sh | 2 - utils/python/CIME/SystemTests/eri.py | 29 +- utils/python/CIME/SystemTests/erp.py | 22 +- utils/python/CIME/SystemTests/err.py | 6 +- utils/python/CIME/SystemTests/ers.py | 22 +- utils/python/CIME/SystemTests/ert.py | 23 +- utils/python/CIME/SystemTests/icp.py | 10 +- utils/python/CIME/SystemTests/nck.py | 39 +- utils/python/CIME/SystemTests/ncr.py | 31 +- utils/python/CIME/SystemTests/pea.py | 28 +- utils/python/CIME/SystemTests/pem.py | 27 +- utils/python/CIME/SystemTests/pet.py | 27 +- utils/python/CIME/SystemTests/pfs.py | 7 +- utils/python/CIME/SystemTests/seq.py | 29 +- utils/python/CIME/SystemTests/sms.py | 6 - utils/python/CIME/SystemTests/ssp.py | 15 +- .../CIME/SystemTests/system_tests_common.py | 425 ++++++++++-------- utils/python/CIME/build.py | 2 + utils/python/CIME/case_setup.py | 18 +- utils/python/CIME/case_test.py | 16 +- utils/python/CIME/test_scheduler.py | 30 +- utils/python/CIME/test_status.py | 220 +++++---- 23 files changed, 514 insertions(+), 533 deletions(-) diff --git a/scripts/Tools/case.build b/scripts/Tools/case.build index 5fce8b348815..4bcac3489b6f 100755 --- a/scripts/Tools/case.build +++ b/scripts/Tools/case.build @@ -87,17 +87,18 @@ def _main_func(description): test = find_system_test(testname, case)(case) append_status("case.testbuild starting ", - caseroot=caseroot,sfile="CaseStatus") - test.build(sharedlib_only=sharedlib_only, model_only=model_only) + caseroot=caseroot,sfile="CaseStatus") + with test: + test.build(sharedlib_only=sharedlib_only, model_only=model_only) append_status("case.testbuild complete", - caseroot=caseroot,sfile="CaseStatus") + caseroot=caseroot,sfile="CaseStatus") else: append_status("case.build starting", - caseroot=caseroot,sfile="CaseStatus") + caseroot=caseroot,sfile="CaseStatus") build.case_build(caseroot, case=case, sharedlib_only=sharedlib_only, - model_only=model_only) + model_only=model_only) append_status("case.build complete", - caseroot=caseroot,sfile="CaseStatus") + caseroot=caseroot,sfile="CaseStatus") if __name__ == "__main__": _main_func(__doc__) diff --git a/scripts/Tools/component_compgen_baseline.sh b/scripts/Tools/component_compgen_baseline.sh index 7afffd9b9b94..6b5aa1cd3935 100755 --- a/scripts/Tools/component_compgen_baseline.sh +++ b/scripts/Tools/component_compgen_baseline.sh @@ -348,5 +348,3 @@ if [ -n "$generate_tag" ]; then exit 2 fi fi - - diff --git a/utils/python/CIME/SystemTests/eri.py b/utils/python/CIME/SystemTests/eri.py index 6f92a483e1ff..9ae9ec92a9c3 100644 --- a/utils/python/CIME/SystemTests/eri.py +++ b/utils/python/CIME/SystemTests/eri.py @@ -26,7 +26,7 @@ def __init__(self, case): SystemTestsCommon.__init__(self, case) self._testname = "ERI" - def run(self): + def run_phase(self): caseroot = self._case.get_value("CASEROOT") clone1_path = "%s.ref1" % caseroot clone2_path = "%s.ref2" % caseroot @@ -99,10 +99,8 @@ def run(self): with open("user_nl_cam", "a") as fd: fd.write("inithist = 'ENDOFRUN'\n") - success = self._run(coupler_log_path=os.path.join(dout_sr1, "logs"), - st_archive=True) - if not success: - return False + self.run_indv(coupler_log_path=os.path.join(dout_sr1, "logs"), + st_archive=True) # # (2) Test run: @@ -148,11 +146,9 @@ def run(self): # run ref2 case (all component history files will go to short term archiving) - success = self._run(suffix="hybrid", - coupler_log_path=os.path.join(dout_sr2, "logs"), - st_archive=True) - if not success: - return False + self.run_indv(suffix="hybrid", + coupler_log_path=os.path.join(dout_sr2, "logs"), + st_archive=True) # # (3a) Test run: @@ -199,9 +195,8 @@ def run(self): self._component_compare_move("hybrid") # run branch case (short term archiving is off) - success = self._run() - if not success: - return False + self.run_indv() + # # (3b) Test run: # do a restart continue from (3a) (short term archiving off) @@ -218,9 +213,7 @@ def run(self): self._case.flush() # do the restart run (short term archiving is off) - success = self._run(suffix="rest") - if not success: - return False + self.run_indv(suffix="rest") - return self._component_compare_test("base", "hybrid") and \ - self._component_compare_test("base", "rest") + self._component_compare_test("base", "hybrid") + self._component_compare_test("base", "rest") diff --git a/utils/python/CIME/SystemTests/erp.py b/utils/python/CIME/SystemTests/erp.py index 56a370749705..c31e072d93bb 100644 --- a/utils/python/CIME/SystemTests/erp.py +++ b/utils/python/CIME/SystemTests/erp.py @@ -24,14 +24,14 @@ def __init__(self, case): """ SystemTestsCommon.__init__(self, case) - def build(self, sharedlib_only=False, model_only=False): + def build_phase(self, sharedlib_only=False, model_only=False): """ Build two cases. Case one uses defaults, case2 uses half the number of threads and tasks. This test will fail for components (e.g. pop) that do not reproduce exactly with different numbers of mpi tasks. """ if sharedlib_only: - SystemTestsCommon.build(self, sharedlib_only=sharedlib_only, model_only=model_only) + self.build_indv(self, sharedlib_only=sharedlib_only, model_only=model_only) return exeroot = self._case.get_value("EXEROOT") @@ -81,7 +81,7 @@ def build(self, sharedlib_only=False, model_only=False): # Now rebuild the system, given updated information in env_build.xml - SystemTestsCommon.build(self, sharedlib_only=sharedlib_only, model_only=model_only) + self.build_indv(self, sharedlib_only=sharedlib_only, model_only=model_only) shutil.move("%s/%s.exe"%(exeroot,cime_model), "%s/%s.ERP%s.exe"%(exeroot,cime_model,bld)) @@ -92,7 +92,7 @@ def build(self, sharedlib_only=False, model_only=False): # # - def run(self): + def run_phase(self): # run will have values 1,2 for run in range(1,3): @@ -138,16 +138,6 @@ def run(self): self._case.set_value("CONTINUE_RUN", True) self._case.set_value("REST_OPTION","never") suffix = "rest" - success = SystemTestsCommon._run(self, suffix=suffix) - if not success: - break + self.run_indv(suffix=suffix) - if success: - return self._component_compare_test("base", "rest") - else: - return False - - - - def report(self): - SystemTestsCommon.report(self) + self._component_compare_test("base", "rest") diff --git a/utils/python/CIME/SystemTests/err.py b/utils/python/CIME/SystemTests/err.py index e32ea0b234fb..ffe05f956c16 100644 --- a/utils/python/CIME/SystemTests/err.py +++ b/utils/python/CIME/SystemTests/err.py @@ -17,13 +17,13 @@ def __init__(self, case): """ ERS.__init__(self, case) - def run(self): + def run_phase(self): first_phase = self._case.get_value("RESUBMIT") == 1 if first_phase: self._case.set_value("DOUT_S", True) self._case.flush() - return self._ers_first_phase() + self._ers_first_phase() else: dout_s_root = self._case.get_value("DOUT_S_ROOT") rundir = self._case.get_value("RUNDIR") @@ -31,4 +31,4 @@ def run(self): for item in glob.glob(os.path.join(dout_s_root, "rest", "*", "*")): shutil.copy(item, rundir) - return self._ers_second_phase() + self._ers_second_phase() diff --git a/utils/python/CIME/SystemTests/ers.py b/utils/python/CIME/SystemTests/ers.py index 3adc101e172c..d08dcb5c5611 100644 --- a/utils/python/CIME/SystemTests/ers.py +++ b/utils/python/CIME/SystemTests/ers.py @@ -31,7 +31,7 @@ def _ers_first_phase(self): expect(stop_n > 2, "ERROR: stop_n value %d too short"%stop_n) logger.info("doing an %s %s initial test with restart file at %s %s" %(str(stop_n), stop_option, str(rest_n), stop_option)) - return SystemTestsCommon.run(self) + self.run_indv() def _ers_second_phase(self): stop_n = self._case.get_value("STOP_N") @@ -47,21 +47,11 @@ def _ers_second_phase(self): self._case.flush() logger.info("doing an %s %s restart test" %(str(stop_n), stop_option)) - success = SystemTestsCommon._run(self, "rest") + self.run_indv(suffix="rest") # Compare restart file - if success: - return self._component_compare_test("base", "rest") - else: - return False + self._component_compare_test("base", "rest") - def run(self): - success = self._ers_first_phase() - - if success: - return self._ers_second_phase() - else: - return False - - def report(self): - SystemTestsCommon.report(self) + def run_phase(self): + self._ers_first_phase() + self._ers_second_phase() diff --git a/utils/python/CIME/SystemTests/ert.py b/utils/python/CIME/SystemTests/ert.py index 6549c7524203..18664c58cc77 100644 --- a/utils/python/CIME/SystemTests/ert.py +++ b/utils/python/CIME/SystemTests/ert.py @@ -30,7 +30,7 @@ def _ert_first_phase(self): self._case.flush() logger.info("doing a 2 month initial test with restart files at 1 month") - return SystemTestsCommon.run(self) + self.run_indv() def _ert_second_phase(self): @@ -40,21 +40,10 @@ def _ert_second_phase(self): self._case.flush() logger.info("doing an 1 month restart test with no restart files") - success = SystemTestsCommon._run(self, "rest") - + self.run_indv(suffix="rest") # Compare restart file - if success: - return self._component_compare_test("base", "rest") - else: - return False - - def run(self): - success = self._ert_first_phase() - - if success: - return self._ert_second_phase() - else: - return False + self._component_compare_test("base", "rest") - def report(self): - SystemTestsCommon.report(self) + def run_phase(self): + self._ert_first_phase() + self._ert_second_phase() diff --git a/utils/python/CIME/SystemTests/icp.py b/utils/python/CIME/SystemTests/icp.py index 96105498cb5f..86195c193e48 100644 --- a/utils/python/CIME/SystemTests/icp.py +++ b/utils/python/CIME/SystemTests/icp.py @@ -12,16 +12,14 @@ def __init__(self, case): """ SystemTestsCommon.__init__(self, case) - def build(self, sharedlib_only=False, model_only=False): - self._case.set_value("CICE_AUTO_DECOMP","false") + def build_phase(self, sharedlib_only=False, model_only=False): + self._case.set_value("CICE_AUTO_DECOMP", "false") - def run(self): + def run_phase(self): self._case.set_value("CONTINUE_RUN",False) self._case.set_value("REST_OPTION","none") self._case.set_value("HIST_OPTION","$STOP_OPTION") self._case.set_value("HIST_N","$STOP_N") self._case.flush() - SystemTestsCommon.run(self) - def report(self): - SystemTestsCommon.report(self) + self.run_indv(self) diff --git a/utils/python/CIME/SystemTests/nck.py b/utils/python/CIME/SystemTests/nck.py index b4cd7a92d2e5..00bb1a8f164b 100644 --- a/utils/python/CIME/SystemTests/nck.py +++ b/utils/python/CIME/SystemTests/nck.py @@ -21,7 +21,7 @@ def __init__(self, case): """ SystemTestsCommon.__init__(self, case) - def build(self, sharedlib_only=False, model_only=False): + def build_phase(self, sharedlib_only=False, model_only=False): ''' build can be called once (sharedlib_only and model_only both False) or twice (once with each true) @@ -62,7 +62,7 @@ def build(self, sharedlib_only=False, model_only=False): if not sharedlib_only: self.clean_build() - SystemTestsCommon.build(self, sharedlib_only=sharedlib_only, model_only=model_only) + self.build_indv(self, sharedlib_only=sharedlib_only, model_only=model_only) if not model_only: shutil.copy("env_mach_pes.xml", machpes) if not sharedlib_only: @@ -75,7 +75,7 @@ def build(self, sharedlib_only=False, model_only=False): # shutil.copy("env_mach_pes.xml", # os.path.join("LockedFiles","env_mach_pes.xml")) - def run(self): + def run_phase(self): os.chdir(self._caseroot) exeroot = self._case.get_value("EXEROOT") @@ -106,30 +106,21 @@ def run(self): # do an initial run test with NINST 1 #====================================================================== logger.info("default: doing a %s %s with NINST1" % (stop_n, stop_option)) - success = SystemTestsCommon.run(self) + self.run_indv() #====================================================================== # do an initial run test with NINST 2 # want to run on same pe counts per instance and same cpl pe count #====================================================================== - if success: - os.remove("%s/%s.exe" % (exeroot, cime_model)) - shutil.copy("%s/%s.exe.NCK2" % (exeroot, cime_model), - "%s/%s.exe" % (exeroot, cime_model)) - shutil.copy("LockedFiles/env_build.NCK2.xml", "env_build.xml") - shutil.copy("env_build.xml", "LockedFiles/env_build.xml") - shutil.copy("LockedFiles/env_mach_pes.NCK2.xml", "env_mach_pes.xml") - shutil.copy("env_mach_pes.xml", "LockedFiles/env_mach_pes.xml") - - logger.info("default: doing a %s %s with NINST2" % (stop_n, stop_option)) - success = SystemTestsCommon._run(self, "multiinst") - - # Compare - if success: - return self._component_compare_test("base", "multiinst") - else: - return False - - def report(self): - SystemTestsCommon.report(self) + os.remove("%s/%s.exe" % (exeroot, cime_model)) + shutil.copy("%s/%s.exe.NCK2" % (exeroot, cime_model), + "%s/%s.exe" % (exeroot, cime_model)) + shutil.copy("LockedFiles/env_build.NCK2.xml", "env_build.xml") + shutil.copy("env_build.xml", "LockedFiles/env_build.xml") + shutil.copy("LockedFiles/env_mach_pes.NCK2.xml", "env_mach_pes.xml") + shutil.copy("env_mach_pes.xml", "LockedFiles/env_mach_pes.xml") + + logger.info("default: doing a %s %s with NINST2" % (stop_n, stop_option)) + self.run_indv(suffix="multiinst") + self._component_compare_test("base", "multiinst") diff --git a/utils/python/CIME/SystemTests/ncr.py b/utils/python/CIME/SystemTests/ncr.py index 5afdcd7d6215..c2fd835e8e4d 100644 --- a/utils/python/CIME/SystemTests/ncr.py +++ b/utils/python/CIME/SystemTests/ncr.py @@ -21,7 +21,7 @@ def __init__(self, case): """ SystemTestsCommon.__init__(self, case) - def build(self, sharedlib_only=False, model_only=False): + def build_phase(self, sharedlib_only=False, model_only=False): exeroot = self._case.get_value("EXEROOT") cime_model = CIME.utils.get_model() @@ -51,7 +51,7 @@ def build(self, sharedlib_only=False, model_only=False): case_setup(self._case, test_mode=True, reset=True) self.clean_build() - SystemTestsCommon.build(self, sharedlib_only, model_only) + self.build_indv(self, sharedlib_only, model_only) shutil.move("%s/%s.exe"%(exeroot,cime_model), "%s/%s.exe.NCR%s"%(exeroot,cime_model,bld)) shutil.copy("env_build.xml",os.path.join("LockedFiles","env_build.NCR%s.xml"%bld)) @@ -63,7 +63,7 @@ def build(self, sharedlib_only=False, model_only=False): shutil.copy("env_mach_pes.xml", os.path.join("LockedFiles","env_mach_pes.xml")) - def run(self): + def run_phase(self): os.chdir(self._caseroot) exeroot = self._case.get_value("EXEROOT") @@ -94,28 +94,21 @@ def run(self): # do an initial run test with NINST 1 #====================================================================== logger.info("default: doing a %s %s with NINST1" % (stop_n, stop_option)) - success = SystemTestsCommon.run(self) + self.run_indv() #====================================================================== # do an initial run test with NINST 2 # want to run on same pe counts per instance and same cpl pe count #====================================================================== - if success: - os.remove("%s/%s.exe" % (exeroot, cime_model)) - shutil.copy("%s/%s.exe.NCR2" % (exeroot, cime_model), - "%s/%s.exe" % (exeroot, cime_model)) - shutil.copy("LockedFiles/env_build.NCR2.xml", "env_build.xml") - shutil.copy("env_build.xml", "LockedFiles/env_build.xml") + os.remove("%s/%s.exe" % (exeroot, cime_model)) + shutil.copy("%s/%s.exe.NCR2" % (exeroot, cime_model), + "%s/%s.exe" % (exeroot, cime_model)) + shutil.copy("LockedFiles/env_build.NCR2.xml", "env_build.xml") + shutil.copy("env_build.xml", "LockedFiles/env_build.xml") - logger.info("default: doing a %s %s with NINST2" % (stop_n, stop_option)) - success = SystemTestsCommon._run(self, "multiinst") + logger.info("default: doing a %s %s with NINST2" % (stop_n, stop_option)) + self.run_indv(suffix="multiinst") # Compare - if success: - return self._component_compare_test("base", "multiinst") - else: - return False - - def report(self): - SystemTestsCommon.report(self) + self._component_compare_test("base", "multiinst") diff --git a/utils/python/CIME/SystemTests/pea.py b/utils/python/CIME/SystemTests/pea.py index 780e35411149..6585558a56bf 100644 --- a/utils/python/CIME/SystemTests/pea.py +++ b/utils/python/CIME/SystemTests/pea.py @@ -17,7 +17,7 @@ def __init__(self, case): """ SystemTestsCommon.__init__(self, case) - def build(self, sharedlib_only=False, model_only=False): + def build_phase(self, sharedlib_only=False, model_only=False): exeroot = self._case.get_value("EXEROOT") cime_model = CIME.utils.get_model() @@ -36,7 +36,7 @@ def build(self, sharedlib_only=False, model_only=False): self._case.flush() case_setup(self._case, reset=True) self.clean_build() - SystemTestsCommon.build(self, sharedlib_only=sharedlib_only, model_only=model_only) + self.build_indv(self, sharedlib_only=sharedlib_only, model_only=model_only) if (not sharedlib_only): shutil.move("%s/%s.exe"%(exeroot,cime_model), "%s/%s.exe.PEA_%s"%(exeroot,cime_model,mpilib)) @@ -64,7 +64,7 @@ def _pea_first_phase(self): logger.info("doing an %d %s initial test with 1pe and mpi, no restarts written" % (stop_n, stop_option)) - return SystemTestsCommon._run(self) + self.run_indv() def _pea_second_phase(self): @@ -90,21 +90,9 @@ def _pea_second_phase(self): logger.info("doing an %d %s initial test with 1pe and serial mpi, no restarts written" % (stop_n, stop_option)) - success = SystemTestsCommon._run(self, "mpiserial") + self.run_indv(suffix="mpiserial") + self._component_compare_test("base", "mpiserial") - # Compare restart file - if success: - return self._component_compare_test("base", "mpiserial") - else: - return False - - def run(self): - success = self._pea_first_phase() - - if success: - return self._pea_second_phase() - else: - return False - - def report(self): - SystemTestsCommon.report(self) + def run_phase(self): + self._pea_first_phase() + self._pea_second_phase() diff --git a/utils/python/CIME/SystemTests/pem.py b/utils/python/CIME/SystemTests/pem.py index 1d52f141a9d6..84df095c3ccf 100644 --- a/utils/python/CIME/SystemTests/pem.py +++ b/utils/python/CIME/SystemTests/pem.py @@ -24,7 +24,7 @@ def __init__(self, case): """ SystemTestsCommon.__init__(self, case) - def build(self, sharedlib_only=False, model_only=False): + def build_phase(self, sharedlib_only=False, model_only=False): """ Build two cases. Case one uses defaults, case2 uses half the number of threads and tasks. This test will fail for components (e.g. pop) that do not reproduce exactly @@ -54,7 +54,7 @@ def build(self, sharedlib_only=False, model_only=False): self._case.flush() case_setup(self._case, test_mode=True, reset=True) self.clean_build() - SystemTestsCommon.build(self, sharedlib_only=sharedlib_only, model_only=model_only) + self.build_indv(self, sharedlib_only=sharedlib_only, model_only=model_only) if (not sharedlib_only): shutil.move("%s/%s.exe"%(exeroot,cime_model), "%s/%s.exe.PEM%s"%(exeroot,cime_model,bld)) @@ -94,7 +94,7 @@ def _pem_first_phase(self): self._case.flush() logger.info("doing an %d %s initial test, no restarts written" % (stop_n, stop_option)) - return SystemTestsCommon._run(self) + self.run_indv() def _pem_second_phase(self): @@ -122,20 +122,9 @@ def _pem_second_phase(self): self._case.flush() logger.info("doing an %d %s initial test, no restarts written" % (stop_n, stop_option)) - success = SystemTestsCommon._run(self, "modpes") + self.run_indv(suffix="modpes") + self._component_compare_test("base", "modpes") - if success: - return self._component_compare_test("base", "modpes") - else: - return False - - def run(self): - success = self._pem_first_phase() - - if success: - return self._pem_second_phase() - else: - return False - - def report(self): - SystemTestsCommon.report(self) + def run_phase(self): + self._pem_first_phase() + self._pem_second_phase() diff --git a/utils/python/CIME/SystemTests/pet.py b/utils/python/CIME/SystemTests/pet.py index c35a232377fd..fbb82198b8bc 100644 --- a/utils/python/CIME/SystemTests/pet.py +++ b/utils/python/CIME/SystemTests/pet.py @@ -21,7 +21,7 @@ def __init__(self, case): """ SystemTestsCommon.__init__(self, case) - def build(self, sharedlib_only=False, model_only=False): + def build_phase(self, sharedlib_only=False, model_only=False): # first make sure that all components have threaded settings for comp in ['ATM','CPL','OCN','WAV','GLC','ICE','ROF','LND']: if self._case.get_value("NTHRDS_%s"%comp) <= 1: @@ -31,7 +31,7 @@ def build(self, sharedlib_only=False, model_only=False): case_setup(self._case, reset=True) self.clean_build() - SystemTestsCommon.build(self, sharedlib_only=sharedlib_only, model_only=model_only) + self.build_indv(self, sharedlib_only=sharedlib_only, model_only=model_only) def _pet_first_phase(self): #Do a run with default threading @@ -46,7 +46,7 @@ def _pet_first_phase(self): logger.info("doing a %d %s initial test with default threading, no restarts written" % (stop_n, stop_option)) - return SystemTestsCommon._run(self) + self.run_indv() def _pet_second_phase(self): #Do a run with all threads set to 1 @@ -60,20 +60,9 @@ def _pet_second_phase(self): logger.info("doing a %d %s initial test with threads set to 1, no restarts written" % (stop_n, stop_option)) - success = SystemTestsCommon._run(self, "single_thread") + self.run_indv(suffix="single_thread") + self._component_compare_test("base", "single_thread") - if success: - return self._component_compare_test("base", "single_thread") - else: - return False - - def run(self): - success = self._pet_first_phase() - - if success: - return self._pet_second_phase() - else: - return False - - def report(self): - SystemTestsCommon.report(self) + def run_phase(self): + self._pet_first_phase() + self._pet_second_phase() diff --git a/utils/python/CIME/SystemTests/pfs.py b/utils/python/CIME/SystemTests/pfs.py index 0afd9d04806a..d943e8d45f28 100644 --- a/utils/python/CIME/SystemTests/pfs.py +++ b/utils/python/CIME/SystemTests/pfs.py @@ -17,7 +17,7 @@ def __init__(self, case): """ SystemTestsCommon.__init__(self, case) - def run(self): + def run_phase(self): self._case.set_value("STOP_OPTION", "ndays") self._case.set_value("STOP_N", 20) self._case.set_value("REST_OPTION","none") @@ -25,7 +25,4 @@ def run(self): self._case.flush() logger.info("doing an 20 day initial test, no restarts written") - return SystemTestsCommon._run(self) - - def report(self): - SystemTestsCommon.report(self) + self.run_indv() diff --git a/utils/python/CIME/SystemTests/seq.py b/utils/python/CIME/SystemTests/seq.py index ec86d3bd5f10..f94e24bfa12c 100644 --- a/utils/python/CIME/SystemTests/seq.py +++ b/utils/python/CIME/SystemTests/seq.py @@ -16,12 +16,12 @@ def __init__(self, case): """ SystemTestsCommon.__init__(self, case, expected=["TEST"]) - def build(self, sharedlib_only=False, model_only=False): + def build_phase(self, sharedlib_only=False, model_only=False): """ Build two cases. """ # Build the default configuration - SystemTestsCommon.build(self, sharedlib_only=sharedlib_only, model_only=model_only) + self.build_indv(self, sharedlib_only=sharedlib_only, model_only=model_only) if sharedlib_only: return @@ -62,14 +62,14 @@ def build(self, sharedlib_only=False, model_only=False): self._case.flush() case_setup(self._case, test_mode=True, reset=True) self.clean_build() - SystemTestsCommon.build(self, sharedlib_only=sharedlib_only, model_only=model_only) + self.build_indv(self, sharedlib_only=sharedlib_only, model_only=model_only) shutil.move("%s/%s.exe"%(exeroot,cime_model), "%s/%s.exe.SEQ2"%(exeroot,cime_model)) machpes2 = os.path.join("LockedFiles","env_mach_pes.SEQ2.xml") logging.info("Copying env_mach_pes.xml to %s"%(machpes2)) shutil.copy("env_mach_pes.xml", machpes2) - def run(self): + def run_phase(self): # Move to config_tests.xml once that's ready. self._case.set_value("CONTINUE_RUN", False) self._case.set_value("REST_OPTION", "never") @@ -79,8 +79,8 @@ def run(self): stop_n = self._case.get_value("STOP_N") stop_option = self._case.get_value("STOP_OPTION") - exeroot = self._case.get_value("EXEROOT") - cime_model = self._case.get_value("MODEL") + exeroot = self._case.get_value("EXEROOT") + cime_model = self._case.get_value("MODEL") # # do an initial run test with default layout @@ -91,22 +91,15 @@ def run(self): "%s/%s.exe"%(exeroot,cime_model)) shutil.copy(os.path.join("LockedFiles", "env_mach_pes.SEQ1.xml"), "env_mach_pes.xml") shutil.copy("env_mach_pes.xml", os.path.join("LockedFiles", "env_mach_pes.xml")) - success = SystemTestsCommon._run(self) - if not success: - return False + self.run_indv() + shutil.copy(os.path.join("LockedFiles", "env_mach_pes.SEQ2.xml"), "env_mach_pes.xml") shutil.copy("env_mach_pes.xml", os.path.join("LockedFiles", "env_mach_pes.xml")) os.remove("%s/%s.exe"%(exeroot,cime_model)) shutil.copy("%s/%s.exe.SEQ1"%(exeroot,cime_model), "%s/%s.exe"%(exeroot,cime_model)) - logger.info("doing a second %d %s test with rootpes set to zero" % (stop_n, stop_option)) - success = SystemTestsCommon._run(self, "seq") - if success: - return self._component_compare_test("base", "seq") - else: - return False - - def report(self): - SystemTestsCommon.report(self) + logger.info("doing a second %d %s test with rootpes set to zero" % (stop_n, stop_option)) + self.run_indv(suffix="seq") + self._component_compare_test("base", "seq") diff --git a/utils/python/CIME/SystemTests/sms.py b/utils/python/CIME/SystemTests/sms.py index 4c80ed331016..5c5aaafd44f2 100644 --- a/utils/python/CIME/SystemTests/sms.py +++ b/utils/python/CIME/SystemTests/sms.py @@ -15,9 +15,3 @@ def __init__(self, case): initialize an object interface to the SMS system test """ SystemTestsCommon.__init__(self, case) - - def run(self): - return SystemTestsCommon.run(self) - - def report(self): - SystemTestsCommon.report(self) diff --git a/utils/python/CIME/SystemTests/ssp.py b/utils/python/CIME/SystemTests/ssp.py index 07fa0cd32208..e4bebfd2aeb1 100644 --- a/utils/python/CIME/SystemTests/ssp.py +++ b/utils/python/CIME/SystemTests/ssp.py @@ -24,7 +24,7 @@ def __init__(self, case): """ SystemTestsCommon.__init__(self, case) - def run(self): + def run_phase(self): caseroot = self._case.get_value("CASEROOT") orig_case = self._case orig_casevar = self._case.get_value("CASE") @@ -56,11 +56,9 @@ def run(self): clone.flush() dout_sr = clone.get_value("DOUT_S_ROOT") - success = self._run(suffix="spinup", - coupler_log_path=os.path.join(dout_sr, "logs"), - st_archive=True) - if not success: - return False + self.run_indv(suffix="spinup", + coupler_log_path=os.path.join(dout_sr, "logs"), + st_archive=True) #------------------------------------------------------------------- # (2) do a hybrid, non-spinup run in orig_case @@ -93,7 +91,4 @@ def run(self): self._case.flush() # do the restart run (short term archiving is off) - success = self._run(suffix="base") - - def report(self): - SystemTestsCommon.report(self) + self.run_indv() diff --git a/utils/python/CIME/SystemTests/system_tests_common.py b/utils/python/CIME/SystemTests/system_tests_common.py index 4e86e1fffead..e0f93b1fb59b 100644 --- a/utils/python/CIME/SystemTests/system_tests_common.py +++ b/utils/python/CIME/SystemTests/system_tests_common.py @@ -8,6 +8,7 @@ from CIME.case_setup import case_setup from CIME.case_run import case_run from CIME.case_st_archive import case_st_archive +from CIME.test_status import * import CIME.build as build @@ -28,6 +29,9 @@ def __init__(self, case, expected=None): self._orig_caseroot = caseroot self._runstatus = None self._casebaseid = self._case.get_value("CASEBASEID") + self._test_status = TestStatus(test_dir=caseroot, test_name=self._casebaseid) + self._ok_to_use = False + # Needed for sh scripts os.environ["CASEROOT"] = caseroot @@ -49,6 +53,14 @@ def __init__(self, case, expected=None): self._case.set_value("TEST",True) self._case.flush() + def __enter__(self): + self._ok_to_use = True + return self + + def __exit__(self): + # Due to wait_for_tests, the RUN phase must be the very last thing updated + with self._test_status: + def fail_test(self): self._runstatus = "FAIL" @@ -63,22 +75,100 @@ def has_passed(self): return self._runstatus == "PASS" def build(self, sharedlib_only=False, model_only=False): - status = "PASS" - try: - build.case_build(self._caseroot, case=self._case, - sharedlib_only=sharedlib_only, model_only=model_only) - except: - status = "FAIL" - if not model_only: - self.update_test_status(status, "SHAREDLIB_BUILD") - if not sharedlib_only: - self.update_test_status(status, "MODEL_BUILD") + """ + Do NOT override this method, this method is the framework that + controls the build phase. build_phase is the extension point + that subclasses should use. + """ + success = True + for phase_name, phase_bool in [(SHAREDLIB_BUILD_PHASE, not model_only), + (MODEL_BUILD_PHASE, not sharedlib_only)]: + if phase_bool: + with self._test_status: + self._test_status.set_status(phase_name, TEST_PENDING_STATUS) + + try: + success = self.build_phase(sharedlib_only=(phase_name==SHAREDLIB_BUILD_PHASE), + model_only=(phase_name==MODEL_BUILD_PHASE)) + if not isinstance(success, bool): + logger.warning("build_phase did not return a bool, assuming PASS!") + success = True + except: + success = False + + with self._test_status: + self._test_status.set_status(phase_name, TEST_PASSED_STATUS if success else TEST_FAILED_STATUS) + + if not success: + break + + return success + + def build_phase(self, sharedlib_only=False, model_only=False): + """ + This is the default build phase implementation, it just does an individual build. + This is the subclass' extension point if they need to define a custom build + phase. + + PLEASE THROW EXCEPTION OR RETURN FALSE ON FAIL + """ + return self.build_indv(sharedlib_only=sharedlib_only, model_only=model_only) + + def build_indv(self, sharedlib_only=False, model_only=False): + """ + Perform an individual build + """ + return build.case_build(self._caseroot, case=self._case, + sharedlib_only=sharedlib_only, model_only=model_only) def clean_build(self, comps=None): build.clean(self._case, cleanlist=comps) def run(self): - return self._run() + """ + Do NOT override this method, this method is the framework that controls + the run phase. run_phase is the extension point that subclasses should use. + """ + success = True + try: + expect(self._test_status.get_status(MODEL_BUILD_PHASE) == TEST_PASSED_STATUS, + "Model was not built!") + with self._test_status: + self._test_status.set_status(RUN_PHASE, TEST_PENDING_STATUS) + + success = self.run_phase() + if not isinstance(success, bool): + logger.warning("build_phase did not return a bool, assuming PASS!") + success = True + + if success: + self.pass_test() + + if self._case.get_value("GENERATE_BASELINE"): + self.generate_baseline() + + if self._case.get_value("COMPARE_BASELINE"): + self.compare_baseline() + else: + self.fail_test() + except: + success = False + self.fail_test() + logger.warning("Exception during run: %s" % (sys.exc_info()[1])) + + # Always try to report + test.report() + + return success + + def run_phase(self): + """ + This is the default run phase implementation, it just does an individual run. + This is the subclass' extension point if they need to define a custom run phase. + + PLEASE THROW AN EXCEPTION OR RETURN FALSE ON FAIL + """ + return self.run_indv() def _set_active_case(self, case): """ @@ -87,68 +177,32 @@ def _set_active_case(self, case): self._case = case self._caseroot = case.get_value("CASEROOT") - def _run(self, suffix="base", coupler_log_path=None, st_archive=False): - self._runstatus = "PEND" - self.update_test_status(self._runstatus,"RUN") - - stop_n = self._case.get_value("STOP_N") + def run_indv(self, suffix="base", coupler_log_path=None, st_archive=False): + """ + Perform an individual run. Raises an EXCEPTION on fail. + """ + stop_n = self._case.get_value("STOP_N") stop_option = self._case.get_value("STOP_OPTION") - run_type = self._case.get_value("RUN_TYPE") + run_type = self._case.get_value("RUN_TYPE") rest_option = self._case.get_value("REST_OPTION") - rest_n = self._case.get_value("REST_N") - infostr = "doing an %d %s %s test" % (stop_n, stop_option,run_type) + rest_n = self._case.get_value("REST_N") + infostr = "doing an %d %s %s test" % (stop_n, stop_option,run_type) + if rest_option == "none": infostr += ", no restarts written" else: infostr += ", with restarts every %d %s"%(rest_n, rest_option) logger.info(infostr) - try: - success = case_run(self._case) - if success and st_archive: - success = case_st_archive(self._case) + case_run(self._case) + if st_archive: + case_st_archive(self._case) - if success and self._coupler_log_indicates_run_complete(coupler_log_path): - self._runstatus = "PASS" - else: - success = False - self._runstatus = "FAIL" - if success and suffix is not None: - self._component_compare_move(suffix) - except: - # An exception must not prevent the TestStatus file from - # being marked FAIL - success = False - self._runstatus = "FAIL" - logger.warning("Exception during run: %s" % (sys.exc_info()[1])) - - return success - - def __del__(self): - if self._runstatus is not None: - self.update_test_status(self._runstatus, "RUN") - - def update_test_status(self, status, phase): - test_status = os.path.join(self._orig_caseroot, "TestStatus") - with open(test_status, 'r') as f: - teststatusfile = f.read() - string = "%s %s"%(self._casebaseid, phase) - total_time = int(time.time() - self._start_time) - self._start_time = time.time() - found = False - with open(test_status, 'w') as f: - for line in teststatusfile.splitlines(): - if string in line: - f.write("%s %s %d\n"%(status, string, total_time)) - found = True - else: - f.write(line+"\n") - if not found: - f.write("%s %s %d\n"%(status, string, total_time)) - # This prevents the __del__ method from overwritting an up to date status - if phase.startswith("RUN"): - self._runstatus = None + if not self._coupler_log_indicates_run_complete(coupler_log_path): + except(False, "Coupler did not indicate run passed") + if suffix is not None: + self._component_compare_move(suffix) def _coupler_log_indicates_run_complete(self, coupler_log_path): newestcpllogfile = self._get_latest_cpl_log(coupler_log_path) @@ -162,6 +216,9 @@ def _coupler_log_indicates_run_complete(self, coupler_log_path): return False def report(self): + """ + Please explain what kind of things happen in report + """ newestcpllogfile = self._get_latest_cpl_log() self._check_for_memleak(newestcpllogfile) @@ -173,8 +230,8 @@ def _component_compare_move(self, suffix): if rc == 0: append_status(out, sfile="TestStatus.log") else: - append_status("Component_compare_test.sh failed out: %s\n\nerr: %s\n"%(out,err) - ,sfile="TestStatus.log") + append_status("Component_compare_test.sh failed out: %s\n\nerr: %s\n" % (out, err), + sfile="TestStatus.log") def _component_compare_test(self, suffix1, suffix2): cmd = os.path.join(self._case.get_value("SCRIPTSROOT"),"Tools", @@ -183,10 +240,11 @@ def _component_compare_test(self, suffix1, suffix2): %(cmd, self._case.get_value('RUNDIR'), self._case.get_value('CASE'), self._case.get_value('CASEBASEID'), suffix1, suffix2, suffix1, suffix2)) logger.debug("run %s results %d %s %s"%(cmd,rc,out,err)) - if rc == 0: - append_status(out.replace("compare","compare functionality", 1) + "\n", - sfile="TestStatus") - else: + status = TEST_PASSED_STATUS if rc == 0 else TEST_FAILED_STATUS + with self._test_status: + self._test_status.set_status("%s_%s_%s" % (COMPARE_PHASE, suffix1, suffix2), status) + + if rc != 0: append_status("Component_compare_test.sh failed out: %s\n\nerr: %s\n"%(out,err), sfile="TestStatus.log") return False @@ -232,24 +290,26 @@ def _check_for_memleak(self, cpllog): memlist = self._get_mem_usage(cpllog) - if len(memlist)<3: - append_status("COMMENT: insuffiencient data for memleak test",sfile="TestStatus") - else: - finaldate = int(memlist[-1][0]) - originaldate = int(memlist[0][0]) - finalmem = float(memlist[-1][1]) - originalmem = float(memlist[0][1]) - memdiff = -1 - if originalmem > 0: - memdiff = (finalmem - originalmem)/originalmem - if memdiff < 0: - append_status("COMMENT: insuffiencient data for memleak test",sfile="TestStatus") - elif memdiff < 0.1: - self.update_test_status("PASS","memleak") + with self._test_status: + if len(memlist)<3: + self._test_status.set_status(MEMLEAK_PHASE, TEST_PASSED_STATUS, comment="insuffiencient data for memleak test") else: - append_status("memleak detected, memory went from %f to %f in %d days" - %(originalmem, finalmem, finaldate-originaldate),sfile="TestStatus.log") - self.update_test_status("FAIL","memleak") + finaldate = int(memlist[-1][0]) + originaldate = int(memlist[0][0]) + finalmem = float(memlist[-1][1]) + originalmem = float(memlist[0][1]) + memdiff = -1 + if originalmem > 0: + memdiff = (finalmem - originalmem)/originalmem + + if memdiff < 0: + self._test_status.set_status(MEMLEAK_PHASE, TEST_PASSED_STATUS, comment="insuffiencient data for memleak test") + elif memdiff < 0.1: + self._test_status.set_status(MEMLEAK_PHASE, TEST_PASSED_STATUS) + else: + comment = "memleak detected, memory went from %f to %f in %d days" % (originalmem, finalmem, finaldate-originaldate) + append_status(comment, sfile="TestStatus.log") + self._test_status.set_status(MEMLEAK_PHASE, TEST_FAILED_STATUS, comment=comment) def compare_env_run(self, expected=None): f1obj = EnvRun(self._caseroot, "env_run.xml") @@ -283,97 +343,96 @@ def compare_baseline(self): """ compare the current test output to a baseline result """ - if not self.has_passed(): - append_status("Cannot compare baselines, test did not pass.\n", sfile="TestStatus.log") - return - - baselineroot = self._case.get_value("BASELINE_ROOT") - basecmp_dir = os.path.join(baselineroot, self._case.get_value("BASECMP_CASE")) - for bdir in (baselineroot, basecmp_dir): - if not os.path.isdir(bdir): - append_status("BFAIL %s compare\n"%self._case.get_value("CASEBASEID"), - sfile="TestStatus") - append_status("ERROR %s does not exist"%bdir, sfile="TestStatus.log") - return -1 - compgen = os.path.join(self._case.get_value("SCRIPTSROOT"),"Tools", - "component_compgen_baseline.sh") - compgen += " -baseline_dir "+basecmp_dir - compgen += " -test_dir "+self._case.get_value("RUNDIR") - compgen += " -compare_tag "+self._case.get_value("BASELINE_NAME_CMP") - compgen += " -testcase "+self._case.get_value("CASE") - compgen += " -testcase_base "+self._case.get_value("CASEBASEID") - rc, out, err = run_cmd(compgen) - - append_status(out.replace("compare","compare baseline", 1),sfile="TestStatus") - if rc != 0: - append_status("Error in Baseline compare: %s\n%s"%(out,err), sfile="TestStatus.log") - - # compare memory usage to baseline - newestcpllogfile = self._get_latest_cpl_log() - memlist = self._get_mem_usage(newestcpllogfile) - baselog = os.path.join(basecmp_dir, "cpl.log.gz") - if not os.path.isfile(baselog): - # for backward compatibility - baselog = os.path.join(basecmp_dir, "cpl.log") - if os.path.isfile(baselog) and len(memlist) > 3: - blmem = self._get_mem_usage(baselog)[-1][1] - curmem = memlist[-1][1] - diff = (curmem-blmem)/blmem - if(diff < 0.1): - append_status("PASS %s memcomp\n"%self._case.get_value("CASEBASEID"), - sfile="TestStatus") - else: - append_status("FAIL %s memcomp\n"%self._case.get_value("CASEBASEID"), - sfile="TestStatus") - append_status("Error in memory compare: Memory usage increase > 10% from baseline", - sfile="TestStatus.log") - # compare throughput to baseline - current = self._get_throughput(newestcpllogfile) - baseline = self._get_throughput(baselog) - #comparing ypd so bigger is better - if baseline is not None and current is not None: - diff = (baseline - current)/baseline - if(diff < 0.25): - append_status("PASS %s tputcomp\n"%self._case.get_value("CASEBASEID"), - sfile="TestStatus") - else: - append_status("FAIL %s tputcomp\n"%self._case.get_value("CASEBASEID"), - sfile="TestStatus") - append_status("Error in throughput compare: Computation time increase > 25% from baseline", - sfile="TestStatus.log") + expect(self.has_passed(), "Should not be trying to compare baselines if test didn't pass") + + with self._test_status: + baselineroot = self._case.get_value("BASELINE_ROOT") + basecmp_dir = os.path.join(baselineroot, self._case.get_value("BASECMP_CASE")) + for bdir in (baselineroot, basecmp_dir): + if not os.path.isdir(bdir): + comment = "ERROR %s does not exist" % bdir + self._test_status("%s_baseline" % COMPARE_PHASE, TEST_FAILED_STATUS, comment=comment) + append_status(comment, sfile="TestStatus.log") + return -1 + + compgen = os.path.join(self._case.get_value("SCRIPTSROOT"),"Tools", + "component_compgen_baseline.sh") + compgen += " -baseline_dir "+basecmp_dir + compgen += " -test_dir "+self._case.get_value("RUNDIR") + compgen += " -compare_tag "+self._case.get_value("BASELINE_NAME_CMP") + compgen += " -testcase "+self._case.get_value("CASE") + compgen += " -testcase_base "+self._case.get_value("CASEBASEID") + rc, out, err = run_cmd(compgen) + + status = TEST_PASSED_STATUS if rc == 0 else TEST_FAILED_STATUS + self._test_status("%s_baseline" % COMPARE_PHASE, status) + append_status("Baseline compare results: %s\n%s"%(out,err), sfile="TestStatus.log") + + # compare memory usage to baseline + newestcpllogfile = self._get_latest_cpl_log() + memlist = self._get_mem_usage(newestcpllogfile) + baselog = os.path.join(basecmp_dir, "cpl.log.gz") + if not os.path.isfile(baselog): + # for backward compatibility + baselog = os.path.join(basecmp_dir, "cpl.log") + if os.path.isfile(baselog) and len(memlist) > 3: + blmem = self._get_mem_usage(baselog)[-1][1] + curmem = memlist[-1][1] + diff = (curmem-blmem)/blmem + if(diff < 0.1): + self._test_status.set_status(MEMCOMP_PHASE, TEST_PASSED_STATUS) + else: + comment = "Error: Memory usage increase > 10% from baseline" + self._test_status.set_status(MEMCOMP_PHASE, TEST_FAILED_STATUS, comment=comment) + append_status(comment, sfile="TestStatus.log") + + # compare throughput to baseline + current = self._get_throughput(newestcpllogfile) + baseline = self._get_throughput(baselog) + #comparing ypd so bigger is better + if baseline is not None and current is not None: + diff = (baseline - current)/baseline + if(diff < 0.25): + self._test_status.set_status(THROUGHPUT_PHASE, TEST_PASSED_STATUS) + else: + comment = "Error: Computation time increase > 25% from baseline" + self._test_status.set_status(THROUGHPUT_PHASE, TEST_FAILED_STATUS, comment=comment) + append_status(comment, sfile="TestStatus.log") def generate_baseline(self): """ generate a new baseline case based on the current test """ - if not self.has_passed(): - append_status("Cannot generate baselines, test did not pass.\n", sfile="TestStatus.log") - return - - newestcpllogfile = self._get_latest_cpl_log() - baselineroot = self._case.get_value("BASELINE_ROOT") - basegen_dir = os.path.join(baselineroot, self._case.get_value("BASEGEN_CASE")) - for bdir in (baselineroot, basegen_dir): - if not os.path.isdir(bdir): - append_status("FAIL %s generate\n" % self._case.get_value("CASEBASEID"), - sfile="TestStatus") - append_status("ERROR %s does not exist" % bdir, sfile="TestStatus.log") - return -1 - compgen = os.path.join(self._case.get_value("SCRIPTSROOT"),"Tools", - "component_compgen_baseline.sh") - compgen += " -baseline_dir "+basegen_dir - compgen += " -test_dir "+self._case.get_value("RUNDIR") - compgen += " -generate_tag "+self._case.get_value("BASELINE_NAME_GEN") - compgen += " -testcase "+self._case.get_value("CASE") - compgen += " -testcase_base "+self._case.get_value("CASEBASEID") - rc, out, err = run_cmd(compgen) - # copy latest cpl log to baseline - # drop the date so that the name is generic - shutil.copyfile(newestcpllogfile, - os.path.join(basegen_dir,"cpl.log.gz")) - append_status(out,sfile="TestStatus") - if rc != 0: - append_status("Error in Baseline Generate: %s"%err,sfile="TestStatus.log") + expect(self.has_passed(), "Should not be trying to generate baselines if test didn't pass") + + with self._test_status: + newestcpllogfile = self._get_latest_cpl_log() + baselineroot = self._case.get_value("BASELINE_ROOT") + basegen_dir = os.path.join(baselineroot, self._case.get_value("BASEGEN_CASE")) + for bdir in (baselineroot, basegen_dir): + if not os.path.isdir(bdir): + comment = "ERROR %s does not exist" % bdir + self._test_status("%s" % GENERATE_PHASE, TEST_FAILED_STATUS, comment=comment) + append_status(comment, sfile="TestStatus.log") + return -1 + + compgen = os.path.join(self._case.get_value("SCRIPTSROOT"),"Tools", + "component_compgen_baseline.sh") + compgen += " -baseline_dir "+basegen_dir + compgen += " -test_dir "+self._case.get_value("RUNDIR") + compgen += " -generate_tag "+self._case.get_value("BASELINE_NAME_GEN") + compgen += " -testcase "+self._case.get_value("CASE") + compgen += " -testcase_base "+self._case.get_value("CASEBASEID") + rc, out, err = run_cmd(compgen) + + status = TEST_PASSED_STATUS if rc == 0 else TEST_FAILED_STATUS + self._test_status("%s" % GENERATE_PHASE, status) + append_status("Baseline generate results: %s\n%s"%(out,err), sfile="TestStatus.log") + + # copy latest cpl log to baseline + # drop the date so that the name is generic + shutil.copyfile(newestcpllogfile, + os.path.join(basegen_dir,"cpl.log.gz")) class FakeTest(SystemTestsCommon): """ @@ -386,7 +445,7 @@ class FakeTest(SystemTestsCommon): def _set_script(self, script): self._script = script # pylint: disable=attribute-defined-outside-init - def build(self, sharedlib_only=False, model_only=False): + def build_phase(self, sharedlib_only=False, model_only=False): if (not sharedlib_only): exeroot = self._case.get_value("EXEROOT") cime_model = self._case.get_value("MODEL") @@ -400,14 +459,12 @@ def build(self, sharedlib_only=False, model_only=False): self._case.set_value("BUILD_COMPLETE", True) self._case.flush() - def run(self): - success = SystemTestsCommon._run(self, suffix=None) - if success and self._runstatus is None: - self._runstatus = "PASS" + def run_phase(self): + self.run_indv(self, suffix=None) class TESTRUNPASS(FakeTest): - def build(self, sharedlib_only=False, model_only=False): + def build_phase(self, sharedlib_only=False, model_only=False): rundir = self._case.get_value("RUNDIR") script = \ """ @@ -426,7 +483,7 @@ class TESTRUNDIFF(FakeTest): 3) Re-run the same test from step 1 but do a baseline comparison instead of generation 3.a) This should give you a DIFF """ - def build(self, sharedlib_only=False, model_only=False): + def build_phase(self, sharedlib_only=False, model_only=False): rundir = self._case.get_value("RUNDIR") cimeroot = self._case.get_value("CIMEROOT") case = self._case.get_value("CASE") @@ -446,7 +503,7 @@ def build(self, sharedlib_only=False, model_only=False): class TESTRUNFAIL(FakeTest): - def build(self, sharedlib_only=False, model_only=False): + def build_phase(self, sharedlib_only=False, model_only=False): rundir = self._case.get_value("RUNDIR") script = \ """ @@ -460,13 +517,13 @@ def build(self, sharedlib_only=False, model_only=False): class TESTBUILDFAIL(FakeTest): - def build(self, sharedlib_only=False, model_only=False): + def build_phase(self, sharedlib_only=False, model_only=False): if (not sharedlib_only): expect(False, "ERROR: Intentional fail for testing infrastructure") class TESTRUNSLOWPASS(FakeTest): - def build(self, sharedlib_only=False, model_only=False): + def build_phase(self, sharedlib_only=False, model_only=False): rundir = self._case.get_value("RUNDIR") script = \ """ @@ -479,7 +536,7 @@ def build(self, sharedlib_only=False, model_only=False): sharedlib_only=sharedlib_only, model_only=model_only) class TESTMEMLEAKFAIL(FakeTest): - def build(self, sharedlib_only=False, model_only=False): + def build_phase(self, sharedlib_only=False, model_only=False): rundir = self._case.get_value("RUNDIR") cimeroot = self._case.get_value("CIMEROOT") testfile = os.path.join(cimeroot,"utils","python","tests","cpl.log.failmemleak.gz") @@ -493,7 +550,7 @@ def build(self, sharedlib_only=False, model_only=False): sharedlib_only=sharedlib_only, model_only=model_only) class TESTMEMLEAKPASS(FakeTest): - def build(self, sharedlib_only=False, model_only=False): + def build_phase(self, sharedlib_only=False, model_only=False): rundir = self._case.get_value("RUNDIR") cimeroot = self._case.get_value("CIMEROOT") testfile = os.path.join(cimeroot,"utils","python","tests","cpl.log.passmemleak.gz") diff --git a/utils/python/CIME/build.py b/utils/python/CIME/build.py index e5f4714b5e44..8e2aa10d43a0 100644 --- a/utils/python/CIME/build.py +++ b/utils/python/CIME/build.py @@ -307,6 +307,8 @@ def case_build(caseroot, case, sharedlib_only=False, model_only=False): logger.info("Time spent not building: %f sec" % (t2 - t1)) logger.info("Time spent building: %f sec" % (t3 - t2)) + return True + ############################################################################### def check_all_input_data(case): ############################################################################### diff --git a/utils/python/CIME/case_setup.py b/utils/python/CIME/case_setup.py index 16ac5cfd4d42..45a315f59a91 100644 --- a/utils/python/CIME/case_setup.py +++ b/utils/python/CIME/case_setup.py @@ -11,6 +11,7 @@ from CIME.XML.compilers import Compilers from CIME.utils import append_status, parse_test_name from CIME.user_mod_support import apply_user_mods +from CIME.test_status import SETUP_PHASE, TEST_FAIL_STATUS, TEST_PASS_STATUS import shutil, time, glob @@ -77,9 +78,8 @@ def _build_usernl_files(case, model, comp): shutil.copy(model_nl, nlfile) ############################################################################### -def case_setup(case, clean=False, test_mode=False, reset=False): +def _case_setup_impl(case, caseroot, casebaseid, clean=False, test_mode=False, reset=False): ############################################################################### - caseroot = case.get_value("CASEROOT") os.chdir(caseroot) msg = "case.setup starting" append_status(msg, caseroot=caseroot, sfile="CaseStatus") @@ -256,7 +256,7 @@ def case_setup(case, clean=False, test_mode=False, reset=False): _build_usernl_files(case, "drv", "cpl") if case.get_value("TEST"): - test_mods = parse_test_name(case.get_value("CASEBASEID"))[6] + test_mods = parse_test_name(casebaseid)[6] if test_mods is not None: user_mods_path = os.path.join(case.get_value("TESTS_MODS_DIR"), test_mods) apply_user_mods(caseroot, user_mods_path=user_mods_path, ninst=ninst) @@ -279,3 +279,15 @@ def case_setup(case, clean=False, test_mode=False, reset=False): msg = "case.setup complete" append_status(msg, caseroot=caseroot, sfile="CaseStatus") +############################################################################### +def case_setup(case, clean=False, test_mode=False, reset=False): +############################################################################### + caseroot, casebaseid = case.get_value("CASEROOT"), case.get_value("CASEBASEID") + with TestStatus(test_dir=caseroot, test_name=casebaseid) as ts: + try: + case_setup(case, caseroot, casebaseid, clean=clean, test_mode=test_mode, reset=reset) + except: + ts.set_status(SETUP_PHASE, TEST_FAIL_STATUS) + raise + else: + ts.set_status(SETUP_PHASE, TEST_PASS_STATUS) diff --git a/utils/python/CIME/case_test.py b/utils/python/CIME/case_test.py index af17b9a1ce16..d841d7cc8563 100644 --- a/utils/python/CIME/case_test.py +++ b/utils/python/CIME/case_test.py @@ -12,20 +12,8 @@ def case_test(case, testname=None): expect(testname is not None, "testname argument not resolved") logging.warn("Running test for %s" % testname) - try: - test = find_system_test(testname, case)(case) + test = find_system_test(testname, case)(case) + with test: success = test.run() - test.report() - - if case.get_value("GENERATE_BASELINE"): - test.generate_baseline() - - if case.get_value("COMPARE_BASELINE"): - test.compare_baseline() - except: - # An uncaught except MUST cause the test to report FAIL - test.fail_test() - raise - return success diff --git a/utils/python/CIME/test_scheduler.py b/utils/python/CIME/test_scheduler.py index d15766fe16f0..89814e2f347b 100644 --- a/utils/python/CIME/test_scheduler.py +++ b/utils/python/CIME/test_scheduler.py @@ -217,12 +217,10 @@ def __init__(self, test_names, if use_existing: for test in self._tests: - test_status_file = os.path.join(self._get_test_dir(test), TEST_STATUS_FILENAME) - statuses = wait_for_tests.parse_test_status_file(test_status_file)[0] - for phase, status in statuses.iteritems(): - if phase != INITIAL_PHASE: - self._update_test_status(test, phase, TEST_PENDING_STATUS) - self._update_test_status(test, phase, status) + ts = TestStatus(self._get_test_dir(test)) + for phase, status in ts: + self._update_test_status(test, phase, TEST_PENDING_STATUS) + self._update_test_status(test, phase, status) else: # None of the test directories should already exist. for test in self._tests: @@ -679,10 +677,13 @@ def _consumer(self, test, test_phase, phase_method): status_str += " Case dir: %s" % self._get_test_dir(test) logger.info(status_str) - if test_phase in [CREATE_NEWCASE_PHASE, XML_PHASE]: + if test_phase in [CREATE_NEWCASE_PHASE, XML_PHASE, NAMELIST_PHASE]: # These are the phases for which TestScheduler is reponsible for # updating the TestStatus file - append_status("%s %s %s" % (status, test, test_phase)) + test_dir = self._get_test_dir(test) + + with TestStatus(test_dir=test_dir, test_name=test) as ts: + ts.set_status(test_phase, status) # On batch systems, we want to immediately submit to the queue, because # it's very cheap to submit and will get us a better spot in line @@ -801,20 +802,11 @@ def run_tests(self): # Setup cs files self._setup_cs_files() - # Return True if all tests passed + # Return True if all tests passed from our point of view logger.info( "At test-scheduler close, state is:") rv = True for test in self._tests: phase, status, nl_fail = self._get_test_data(test) - logger.debug("phase %s status %s" % (phase, status)) - if status == TEST_PASS_STATUS and phase == RUN_PHASE: - # Be cautious about telling the user that the test passed. This - # status should match what they would see on the dashboard. Our - # self._test_states does not include comparison fail information, - # so we need to parse test status. - test_status_file = os.path.join(self._get_test_dir(test), TEST_STATUS_FILENAME) - status = wait_for_tests.interpret_status_file(test_status_file)[1] - if status not in [TEST_PASS_STATUS, TEST_PENDING_STATUS]: logger.info( "%s %s (phase %s)" % (status, test, phase)) rv = False @@ -824,7 +816,7 @@ def run_tests(self): rv = False else: - logger.info("status=%s test=%s phase=%s"%( status, test, phase)) + logger.info("%s %s %s" % (status, test, phase)) logger.info( " Case dir: %s" % self._get_test_dir(test)) diff --git a/utils/python/CIME/test_status.py b/utils/python/CIME/test_status.py index 51ca70531e41..e2c66b90400c 100644 --- a/utils/python/CIME/test_status.py +++ b/utils/python/CIME/test_status.py @@ -29,8 +29,10 @@ MODEL_BUILD_PHASE = "MODEL_BUILD" RUN_PHASE = "RUN" THROUGHPUT_PHASE = "TPUTCOMP" -MEMORY_PHASE = "MEMCOMP" -COMPARE_PHASE = "COMPARE" +MEMCOMP_PHASE = "MEMCOMP" +MEMLEAK_PHASE = "MEMLEAK" +COMPARE_PHASE = "COMPARE" # This is one special, real phase will be COMPARE_$WHAT +GENERATE_PHASE = "GENERATE" ALL_PHASES = [INITIAL_PHASE, CREATE_NEWCASE_PHASE, @@ -42,31 +44,84 @@ RUN_PHASE, THROUGHPUT_PHASE, MEMORY_PHASE, - COMPARE_PHASE] + GENERATE_PHASE] -MULTI_PHASES = [COMPARE_PHASE] +def _test_helper1(file_contents): + ts = TestStatus(test_dir="/", test_name="ERS.foo.A") + ts._parse_test_status(file_contents) # pylint: disable=protected-access + return ts._phase_statuses # pylint: disable=protected-access + +def _test_helper2(file_contents, wait_for_run=False, check_throughput=False, check_memory=False, ignore_namelists=False): + ts = TestStatus(test_dir="/", test_name="ERS.foo.A") + ts._parse_test_status(file_contents) # pylint: disable=protected-access + return ts.get_overall_test_status(wait_for_run=wait_for_run, + check_throughput=check_throughput, + check_memory=check_memory, + ignore_namelists=ignore_namelists) class TestStatus(object): - def __init__(self, test_dir=os.getcwd()): + def __init__(self, test_dir=os.getcwd(), test_name=None): self._filename = os.path.join(test_dir, TEST_STATUS_FILENAME) self._phase_statuses = OrderedDict() # {name -> (status, comments)} - self._test_name = None + self._test_name = test_name + self._ok_to_modify = False if os.path.exists(self._filename): self._parse_test_status_file() + else: + expect(test_name is not None, "Must provide test_name if TestStatus file doesn't exist") + + def __enter__(self): + self._ok_to_modify = True + return self + + def __exit__(self, *_): + self._ok_to_modify = False + self.flush() + + def __iter__(self): + for phase, data in self._phase_statuses.iteritems(): + yield phase, data[0] + + def set_status(self, phase, status, comments=""): + """ + >>> with TestStatus(test_dir="/", test_name="ERS.foo.A") as ts: + ... ts.set_status(CREATE_NEWCASE_PHASE, "PASS") + ... ts.set_status(XML_PHASE, "PASS") + ... ts.set_status(SETUP_PHASE, "PASS") + ... ts.set_status(SETUP_PHASE, "FAIL") + ... result = OrderedDict(ts._phase_statuses) + ... ts._phase_statuses = None + >>> result + OrderedDict([('CREATE_NEWCASE', ('PASS', '')), ('XML', ('PASS', '')), ('SETUP', ('FAIL', ''))]) + """ + expect(self._ok_to_modify, "TestStatus not in a modifiable state, use 'with' syntax") + expect(phase in ALL_PHASES or phase.startswith(COMPARE_PHASE), + "Invalid phase '%s'" % phase) + expect(status in ALL_PHASE_STATUSES, "Invalid status '%s'" % status) + + self._phase_statuses[phase] = (status, comments) # Can overwrite old phase info + + def get_status(self, phase): + return self._phase_statuses[phase][0] if phase in self._phase_statuses else None + + def flush(self): + if self._phase_statuses: + with open(self._filename, "w") as fd: + for phase, data in self._phase_statuses.iteritems(): + status, comments = data + fd.write("%s %s %s %s\n" % (status, self._test_name, phase, comments)) def _parse_test_status(self, file_contents): """ - >>> ts = TestStatus() >>> contents = ''' - ... CREATE_NEWCASE ERS.foo.A PASS - ... XML_PHASE ERS.foo.A PASS - ... SETUP_PHASE ERS.foo.A FAIL + ... PASS ERS.foo.A CREATE_NEWCASE + ... PASS ERS.foo.A XML + ... FAIL ERS.foo.A SETUP ... ''' - >>> ts._parse_test_status(contents) - >>> ts._phase_statuses - '{stuff}' + >>> _test_helper1(contents) + OrderedDict([('CREATE_NEWCASE', ('PASS', '')), ('XML', ('PASS', '')), ('SETUP', ('FAIL', ''))]) """ for line in file_contents.splitlines(): line = line.strip() @@ -78,18 +133,17 @@ def _parse_test_status(self, file_contents): if (self._test_name is None): self._test_name = curr_test_name else: - expect(self._test_name == curr_test_name, "inconsistent test name in parse_test_status: '%s' != '%s'"%(self._test_name, curr_test_name)) + expect(self._test_name == curr_test_name, + "inconsistent test name in parse_test_status: '%s' != '%s'" % (self._test_name, curr_test_name)) expect(status in ALL_PHASE_STATUSES, "Unexpected status '%s' in parse_test_status for test '%s'" % (status, self._test_name)) - expect(phase in ALL_PHASES, + expect(phase in ALL_PHASES or phase.startswith(COMPARE_PHASE), "phase '%s' not expected in parse_test_status for test '%s'" % (phase, self._test_name)) + expect(phase not in self._phase_statuses, + "Should not have seen multiple instances of phase '%s' for test '%s'" % (phase, self._test_name)) - if (phase in rv): - # Phase names don't matter here, just need something unique - rv[phase] = reduce_stati({"%s_" % phase : status, phase : rv[phase]}) - else: - rv[phase] = status + self._phase_statuses[phase] = (status, " ".join(tokens[3:])) else: logging.warning("In TestStatus file for test '%s', line '%s' not in expected format" % (self._test_name, line)) @@ -97,78 +151,66 @@ def _parse_test_status_file(self): with open(self._filename, "r") as fd: self._parse_test_status(fd.read()) + def get_overall_test_status(self, wait_for_run=False, check_throughput=False, check_memory=False, ignore_namelists=False): + r""" + Given the current phases and statuses, produce a single results for this test. Preference + is given to PEND since we don't want to stop waiting for a test + that hasn't finished. Namelist diffs are given the lowest precedence. + + >>> _test_helper2('PASS ERS.foo.A RUN') + 'PASS' + >>> _test_helper2('PASS ERS.foo.A SHAREDLIB_BUILD\nPEND ERS.foo.A RUN') + 'PEND' + >>> _test_helper2('FAIL ERS.foo.A MODEL_BUILD\nPEND ERS.foo.A RUN') + 'PEND' + >>> _test_helper2('PASS ERS.foo.A MODEL_BUILD\nPASS ERS.foo.A RUN') + 'PASS' + >>> _test_helper2('PASS ERS.foo.A RUN\nFAIL ERS.foo.A TPUTCOMP') + 'PASS' + >>> _test_helper2('PASS ERS.foo.A RUN\nFAIL ERS.foo.A TPUTCOMP', check_throughput=True) + 'FAIL' + >>> _test_helper2('PASS ERS.foo.A RUN\nFAIL ERS.foo.A NLCOMP') + 'NLFAIL' + >>> _test_helper2('PASS ERS.foo.A RUN\nFAIL ERS.foo.A MEMLEAK') + 'PASS' + >>> _test_helper2('PASS ERS.foo.A RUN\nFAIL ERS.foo.A NLCOMP', ignore_namelists=True) + 'PASS' + >>> _test_helper2('PASS ERS.foo.A COMPARE_1\nFAIL ERS.foo.A NLCOMP\nFAIL ERS.foo.A COMPARE_2\nPASS ERS.foo.A RUN') + 'DIFF' + >>> _test_helper2('PASS ERS.foo.A MODEL_BUILD') + 'PASS' + >>> _test_helper2('PASS ERS.foo.A MODEL_BUILD', wait_for_run=True) + 'PEND' + """ + rv = TEST_PASS_STATUS + run_phase_found = False + for phase, data in self._phase_statuses.iteritems(): + status = data[0] + if phase == RUN_PHASE: + run_phase_found = True -def reduce_stati(stati, wait_for_run=False, check_throughput=False, check_memory=False, ignore_namelists=False): - """ - Given a collection of stati for a test, produce a single result. Preference - is given to unfinished stati since we don't want to stop waiting for a test - that hasn't finished. Namelist diffs are given the lowest precedence. - """ - rv = TEST_PASS_STATUS - run_phase_found = False - for phase, status in stati.iteritems(): - if phase == RUN_PHASE: - run_phase_found = True + if (status == TEST_PENDING_STATUS): + return status - if (status == TEST_PENDING_STATUS): - return status + elif (status == TEST_FAIL_STATUS): + if ( (not check_throughput and phase == THROUGHPUT_PHASE) or + (not check_memory and phase == MEMORY_PHASE) or + (ignore_namelists and phase == NAMELIST_PHASE) ): + continue - elif (status != TEST_PASS_STATUS): - if ( (not check_throughput and THROUGHPUT_TEST_STR in phase) or - (not check_memory and MEMORY_TEST_STR in phase) or - (ignore_namelists and phase == NAMELIST_PHASE) ): - continue + if (phase == NAMELIST_PHASE): + if (rv == TEST_PASS_STATUS): + rv = NAMELIST_FAIL_STATUS - if (status == NAMELIST_FAIL_STATUS): - if (rv == TEST_PASS_STATUS): - rv = NAMELIST_FAIL_STATUS + elif (rv in [NAMELIST_FAIL_STATUS, TEST_PASS_STATUS] and phase.startswith(COMPARE_PHASE)): + rv = TEST_DIFF_STATUS - elif (rv in [NAMELIST_FAIL_STATUS, TEST_PASS_STATUS] and phase == HIST_COMPARE_PHASE): - rv = TEST_DIFF_STATUS + else: + rv = TEST_FAIL_STATUS - else: - rv = status - - # The test did not fail but the RUN phase was not found, so if the user requested - # that we wait for the RUN phase, then the test must still be considered pending. - if rv != TEST_FAIL_STATUS and not run_phase_found and wait_for_run: - rv = TEST_PENDING_STATUS - - return rv - - -def interpret_status(file_contents, wait_for_run=False, check_throughput=False, check_memory=False, ignore_namelists=False): - r""" - >>> interpret_status('PASS testname RUN') - ('testname', 'PASS') - >>> interpret_status('PASS testname SHAREDLIB_BUILD\nPEND testname RUN') - ('testname', 'PEND') - >>> interpret_status('FAIL testname MODEL_BUILD\nPEND testname RUN') - ('testname', 'PEND') - >>> interpret_status('PASS testname MODEL_BUILD\nPASS testname RUN') - ('testname', 'PASS') - >>> interpret_status('PASS testname RUN\nFAIL testname tputcomp') - ('testname', 'PASS') - >>> interpret_status('PASS testname RUN\nFAIL testname tputcomp', check_throughput=True) - ('testname', 'FAIL') - >>> interpret_status('PASS testname RUN\nNLFAIL testname nlcomp') - ('testname', 'NLFAIL') - >>> interpret_status('PASS testname RUN\nFAIL testname memleak') - ('testname', 'FAIL') - >>> interpret_status('PASS testname RUN\nNLFAIL testname nlcomp', ignore_namelists=True) - ('testname', 'PASS') - >>> interpret_status('PASS testname compare\nNLFAIL testname nlcomp\nFAIL testname compare\nPASS testname RUN') - ('testname', 'DIFF') - >>> interpret_status('PASS testname MODEL_BUILD') - ('testname', 'PASS') - >>> interpret_status('PASS testname MODEL_BUILD', wait_for_run=True) - ('testname', 'PEND') - """ - statuses, test_name = parse_test_status(file_contents) - reduced_status = reduce_stati(statuses, wait_for_run, check_throughput, check_memory, ignore_namelists) - - return test_name, reduced_status - -def interpret_status_file(file_name, wait_for_run=False, check_throughput=False, check_memory=False, ignore_namelists=False): - with open(file_name, "r") as fd: - return interpret_status(fd.read(), wait_for_run, check_throughput, check_memory, ignore_namelists) + # The test did not fail but the RUN phase was not found, so if the user requested + # that we wait for the RUN phase, then the test must still be considered pending. + if rv != TEST_FAIL_STATUS and not run_phase_found and wait_for_run: + rv = TEST_PENDING_STATUS + + return rv