diff --git a/ChangeLog b/ChangeLog index 16519468fde9..4b5f21015c61 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,117 @@ ====================================================================== +Originator: Bill Sacks +Date: Aug 02, 2016 +Tag: cime5.0.5 +Answer Changes: None +Tests: scripts_regression_tests +Dependencies: None (removes dependency of cime5.0.4 on clm branch) + +Description: +- move LII test from CLM back to CIME (until new API is solidified) + +Modified files: +utils/python/CIME/SystemTests/lii.py + +====================================================================== + +Originator: Jim Edwards +Date: Aug 01, 2016 +Tag: cime5.0.4 +Answer Changes: None +Tests: scripts_regression_tests +Dependencies: Needs clm branch containing LII test + +Description: + +- Updates of hobart machine +- Put provenance back in README - +- Update TestStatus to run tests multiple times in a case - Fix for +- Add README files back to test directories +- Dynamically search for system tests, move LII test to CLM cime_config +- Add python profiling tool +- Minor bug fixes in prep_ocn_mod and prep_wav_mod pointed out by CRAY compiler + +Modified files: + +cime_config/acme/config_files.xml +cime_config/cesm/config_files.xml +cime_config/cesm/machines/config_batch.xml +cime_config/cesm/machines/config_compilers.xml +cime_config/cesm/machines/config_machines.xml +driver_cpl/driver/prep_ocn_mod.F90 +driver_cpl/driver/prep_wav_mod.F90 +scripts/Testing/Testcases/README +scripts/Tools/advanced-py-prof +scripts/Tools/bless_test_results +scripts/Tools/case.build +scripts/Tools/simple-py-prof +scripts/create_test +utils/python/CIME/SystemTests/README +utils/python/CIME/SystemTests/lii.py +utils/python/CIME/SystemTests/system_tests_common.py +utils/python/CIME/XML/env_test.py +utils/python/CIME/case.py +utils/python/CIME/case_test.py +utils/python/CIME/check_input_data.py +utils/python/CIME/system_test.py +utils/python/CIME/test_scheduler.py +utils/python/CIME/test_utils.py +utils/python/CIME/user_mod_support.py +utils/python/CIME/utils.py +utils/python/cs.submit.template +utils/python/tests/scripts_regression_tests.py + + + +==================================================================== + +Originator: Jim Edwards +Date: July 26, 2016 +Tag: cime5.0.3 +Answer Changes: None +Tests: scripts_regression_tests +Dependencies: + +Brief Summary: +- fixes as a result of cesm2_alpha01b testing +- print output from buildnml +- improve mismatch argument to NAG compiler +- fix memleak error if baseline not found +- add support to create_test for ascii testfile +- move pecount support from create_test to create_newcase +- fix LII test, fix ERP test +- reintroduce --verbose option + +User interface changes: none + +Modified files: git diff --name-status +M ChangeLog +M cime_config/acme/config_files.xml +M cime_config/cesm/machines/Makefile +M cime_config/cesm/machines/config_compilers.xml +A cime_config/cesm/machines/nag_mpi_argument.txt +M driver_cpl/cime_config/buildexe +M scripts/Tools/case.build +M scripts/Tools/code_checker +M scripts/Tools/component_compare_test.sh +M scripts/Tools/preview_namelists +M scripts/create_newcase +M scripts/create_test +M utils/python/CIME/SystemTests/erp.py +M utils/python/CIME/SystemTests/lii.py +M utils/python/CIME/SystemTests/system_tests_common.py +M utils/python/CIME/XML/env_batch.py +M utils/python/CIME/build.py +M utils/python/CIME/case.py +M utils/python/CIME/preview_namelists.py +M utils/python/CIME/system_test.py +M utils/python/CIME/utils.py +M utils/python/tests/scripts_regression_tests.py +M utils/python/update_acme_tests.py + +====================================================================== + Originator: Jim Edwards Date: July 22, 2016 Tag: cime5.0.2 diff --git a/cime_config/acme/config_files.xml b/cime_config/acme/config_files.xml index 1b3ef416bb9e..39972886c493 100644 --- a/cime_config/acme/config_files.xml +++ b/cime_config/acme/config_files.xml @@ -117,6 +117,23 @@ file containing specification of all pe-layouts for primary component (for documentation only - DO NOT EDIT) + + char + + $CIMEROOT/utils/python/CIME/SystemTests + $SRCROOT/components/clm/cime_config/SystemTests + $SRCROOT/components/cam/cime_config/SystemTests + $SRCROOT/components/pop/cime_config/SystemTests + $SRCROOT/components/cice/cime_config/SystemTests + $SRCROOT/components/cism/cime_config/SystemTests + + test + env_test.xml + directories containing cime compatible system test modules + + + + char unset diff --git a/cime_config/cesm/config_files.xml b/cime_config/cesm/config_files.xml index 13643c765dcd..bcbb55130d00 100644 --- a/cime_config/cesm/config_files.xml +++ b/cime_config/cesm/config_files.xml @@ -113,6 +113,23 @@ file containing specification of all pe-layouts for primary component (for documentation only - DO NOT EDIT) + + char + + $CIMEROOT/utils/python/CIME/SystemTests + $SRCROOT/components/clm/cime_config/SystemTests + $SRCROOT/components/cam/cime_config/SystemTests + $SRCROOT/components/pop/cime_config/SystemTests + $SRCROOT/components/cice/cime_config/SystemTests + $SRCROOT/components/cism/cime_config/SystemTests + $SRCROOT/components/rtm/cime_config/SystemTests + $SRCROOT/components/mosart/cime_config/SystemTests + + test + env_test.xml + directories containing cime compatible system test modules + + char unset diff --git a/cime_config/cesm/machines/config_batch.xml b/cime_config/cesm/machines/config_batch.xml index fb5407c34886..12c31b06e3b8 100644 --- a/cime_config/cesm/machines/config_batch.xml +++ b/cime_config/cesm/machines/config_batch.xml @@ -131,7 +131,6 @@ - --job-name={{ job_id }} diff --git a/cime_config/cesm/machines/config_compilers.xml b/cime_config/cesm/machines/config_compilers.xml index 6e72e5a47af0..b54a9f92f38a 100644 --- a/cime_config/cesm/machines/config_compilers.xml +++ b/cime_config/cesm/machines/config_compilers.xml @@ -843,7 +843,6 @@ for mct, etc. - mpi mpich /usr/lib64 $(shell $(NETCDF_PATH)/bin/nf-config --flibs) @@ -857,6 +856,9 @@ for mct, etc. -Wl,-rpath,$(COMPILER_PATH)/mkl/lib/intel64 -Wl,-rpath,$(MPI_PATH)/lib -mkl=cluster + -lifcore + -lifcore + -lifcore /home/santos/pFUnit/pFUnit_Intel_3_0 -mcmodel medium diff --git a/cime_config/cesm/machines/config_machines.xml b/cime_config/cesm/machines/config_machines.xml index 27c6335b1328..37f13cbfc31b 100644 --- a/cime_config/cesm/machines/config_machines.xml +++ b/cime_config/cesm/machines/config_machines.xml @@ -369,7 +369,7 @@ PrgEnv-cray - cce cce/8.5.0 + cce cce/8.5.1 PrgEnv-gnu @@ -468,7 +468,7 @@ intel,gnu,cray edison mpt,mpi-serial - $ENV{SCRATCH} + $ENV{CSCRATCH} $CESMSCRATCHROOT/$CASE/run $CESMSCRATCHROOT/$CASE/bld /project/projectdirs/ccsm1/inputdata @@ -534,7 +534,7 @@ PrgEnv-cray - cce cce/8.4.3 + cce cce/8.5.1 PrgEnv-gnu @@ -577,10 +577,7 @@ "NCAR CGD Linux Cluster 48 pes/node, batch system is PBS" hobart intel,pgi,nag,gnu - openmpi - mvapich2 - mvapich2 - mvapich2 + mvapich2 /scratch/cluster/$USER/$CASE/run /scratch/cluster/$USER/$CASE/bld /scratch/cluster/$USER @@ -621,22 +618,22 @@ compiler/intel/15.0.2.164 - tool/parallel-netcdf/1.6.1/intel/mvapich2 + tool/parallel-netcdf/1.7.0/intel/mvapich2 compiler/pgi/15.1 - tool/parallel-netcdf/1.6.1/pgi/mvapich2 + tool/parallel-netcdf/1.7.0/pgi/mvapich2 - compiler/nag/6.0 + compiler/nag/6.1 - - tool/parallel-netcdf/1.6.1/nag/openmpi + + tool/parallel-netcdf/1.7.0/nag/mvapich2 - compiler/gnu/4.8.3 + compiler/gnu/4.8.5 diff --git a/driver_cpl/driver/prep_ocn_mod.F90 b/driver_cpl/driver/prep_ocn_mod.F90 index 424bd91449f6..44e4ec22f2e7 100644 --- a/driver_cpl/driver/prep_ocn_mod.F90 +++ b/driver_cpl/driver/prep_ocn_mod.F90 @@ -1191,8 +1191,8 @@ subroutine prep_ocn_calc_r2x_ox(timer) call seq_map_map(mapper_Rr2o, r2x_rx, r2x_ox(eri), & fldlist='Forr_rofl:Forr_rofi: & - Forr_rofl_16O:Forr_rofi_16O:Forr_rofl_18O: & - Forr_rofi_18O:Forr_rofl_HDO:Forr_rofi_HDO', norm=.false.) + & Forr_rofl_16O:Forr_rofi_16O:Forr_rofl_18O: & + & Forr_rofi_18O:Forr_rofl_HDO:Forr_rofi_HDO', norm=.false.) if (flood_present) then call seq_map_map(mapper_Fr2o, r2x_rx, r2x_ox(eri), & fldlist='Flrr_flood', norm=.true.) diff --git a/driver_cpl/driver/prep_wav_mod.F90 b/driver_cpl/driver/prep_wav_mod.F90 index e14332b67317..3384f3560f35 100644 --- a/driver_cpl/driver/prep_wav_mod.F90 +++ b/driver_cpl/driver/prep_wav_mod.F90 @@ -97,6 +97,7 @@ subroutine prep_wav_init(infodata, atm_c2_wav, ocn_c2_wav, ice_c2_wav) wav_present=wav_present , & ocn_gnam=ocn_gnam , & wav_gnam=wav_gnam , & + atm_gnam=atm_gnam , & esmf_map_flag=esmf_map_flag ) allocate(mapper_sa2w) diff --git a/scripts/Testing/Testcases/README b/scripts/Testing/Testcases/README new file mode 100644 index 000000000000..33abd639d05e --- /dev/null +++ b/scripts/Testing/Testcases/README @@ -0,0 +1,39 @@ + +NOTES: +Most of the tests have been moved to python in cime/utils/python/CIME/SystemTests +See the README there for more info. + +The remaining tests are: + +====================================================================== + Multi-Instance Tests (smoke) +====================================================================== + +NOC multi-instance validation for single instance ocean (default length) + do an initial run test with NINST 2 (other than ocn), with mod to instance 1 (suffix: inst1_base, inst2_mod) + do an initial run test with NINST 2 (other than ocn), with mod to instance 2 (suffix: inst1_base, inst2_mod) + compare inst1_base with inst2_base + compare inst1_mod with inst2_mod + + +====================================================================== + Performance Tests +====================================================================== + +ICP cice performance test +OCP pop performance test + +====================================================================== + Archiving Tests +====================================================================== + +LAR long term archive test + +====================================================================== + Other +====================================================================== + +HOMME + + + diff --git a/scripts/Tools/advanced-py-prof b/scripts/Tools/advanced-py-prof new file mode 100755 index 000000000000..5aecf8397107 --- /dev/null +++ b/scripts/Tools/advanced-py-prof @@ -0,0 +1,14 @@ +#! /bin/bash -e + +if [ "$#" -eq 0 ]; then + echo "Usage: advanced-py-prof " + exit 0 +fi + +# Requires graphviz and gprof2dot + +DATE_STAMP=$(date "+%Y-%m-%d_%H%M%S") +FILE_BASENAME=$(basename $1).${DATE_STAMP} +python -m cProfile -o ${FILE_BASENAME}.prof "$@" +gprof2dot -f pstats ${FILE_BASENAME}.prof -o ${FILE_BASENAME}.dot || { echo "This tool requires gprof2dot"; exit 1; } +dot -Teps ${FILE_BASENAME}.dot -o ${FILE_BASENAME}.eps || { echo "This tool requires graphviz (dot)"; exit 1; } diff --git a/scripts/Tools/bless_test_results b/scripts/Tools/bless_test_results index 9ffcd4dd6e60..1f58a729eaf5 100755 --- a/scripts/Tools/bless_test_results +++ b/scripts/Tools/bless_test_results @@ -13,7 +13,7 @@ with versions of the files that you should not bless. from standard_script_setup import * import wait_for_tests, compare_namelists, simple_compare -from CIME.system_test import NAMELIST_PHASE +from CIME.test_scheduler import NAMELIST_PHASE from CIME.utils import run_cmd, run_cmd_no_fail, expect from CIME.XML.machines import Machines diff --git a/scripts/Tools/case.build b/scripts/Tools/case.build index 6e2baec37641..5fce8b348815 100755 --- a/scripts/Tools/case.build +++ b/scripts/Tools/case.build @@ -5,27 +5,10 @@ Script to build a case. """ from standard_script_setup import * -from CIME.SystemTests.eri import ERI -from CIME.SystemTests.erp import ERP -from CIME.SystemTests.err import ERR -from CIME.SystemTests.ers import ERS -from CIME.SystemTests.lii import LII -from CIME.SystemTests.nck import NCK -from CIME.SystemTests.ncr import NCR -from CIME.SystemTests.noc import NOC -from CIME.SystemTests.pea import PEA -from CIME.SystemTests.pem import PEM -from CIME.SystemTests.pet import PET -from CIME.SystemTests.pfs import PFS -from CIME.SystemTests.rep import REP -from CIME.SystemTests.seq import SEQ -from CIME.SystemTests.sms import SMS -from CIME.SystemTests.ssp import SSP -from CIME.SystemTests.system_tests_common import * import CIME.build as build from CIME.case import Case -from CIME.utils import expect, append_status +from CIME.utils import expect, append_status, find_system_test from CIME.XML.files import Files from CIME.XML.component import Component @@ -101,10 +84,8 @@ def _main_func(description): elif(testname is not None): logging.warn("Building test for %s in directory %s" % (testname, caseroot)) - try: - test = globals()[testname](case) - except KeyError: - expect(False, "Could not find a test called '%s'" % testname) + test = find_system_test(testname, case)(case) + append_status("case.testbuild starting ", caseroot=caseroot,sfile="CaseStatus") test.build(sharedlib_only=sharedlib_only, model_only=model_only) diff --git a/scripts/Tools/code_checker b/scripts/Tools/code_checker index 55fe92a50252..0d185e0ec7e3 100755 --- a/scripts/Tools/code_checker +++ b/scripts/Tools/code_checker @@ -88,9 +88,11 @@ def check_code(dir_to_check, num_procs, files): os.environ["PYTHONPATH"] = dir_to_check # Get list of files to check - files_to_check = run_cmd_no_fail('find %s -name "*.py"' % dir_to_check).splitlines() + files_to_check = run_cmd_no_fail('git ls-files %s' % dir_to_check).splitlines() if files: files_to_check = [item for item in files_to_check if matches(item, files)] + else: + files_to_check = [item for item in files_to_check if item.endswith(".py")] pool = ThreadPool(num_procs) results = pool.map(run_pylint, files_to_check) diff --git a/scripts/Tools/simple-py-prof b/scripts/Tools/simple-py-prof new file mode 100755 index 000000000000..b51e6d2ec53c --- /dev/null +++ b/scripts/Tools/simple-py-prof @@ -0,0 +1,8 @@ +#! /bin/bash + +if [ "$#" -eq 0 ]; then + echo "Usage: simple-py-prof " + exit 0 +fi + +python -m cProfile -s time "$@" diff --git a/scripts/create_test b/scripts/create_test index 19d0a4bfb87a..20bd712ed782 100755 --- a/scripts/create_test +++ b/scripts/create_test @@ -11,7 +11,7 @@ If this tool is missing any feature that you need, please notify jgfouca@sandia. from Tools.standard_script_setup import * import update_acme_tests -from CIME.system_test import SystemTest, RUN_PHASE +from CIME.test_scheduler import TestScheduler, RUN_PHASE from CIME.utils import expect, convert_to_seconds, compute_total_time, convert_to_babylonian_time, run_cmd_no_fail from CIME.XML.machines import Machines from CIME.case import Case @@ -73,6 +73,9 @@ OR parser.add_argument("--no-build", action="store_true", help="Do not build generated tests, implies --no-run") + parser.add_argument("--no-setup", action="store_true", + help="Do not setup generated tests, implies --no-build and --no-run") + parser.add_argument("-u", "--use-existing", action="store_true", help="Use pre-existing case directories. Requires test-id") @@ -190,11 +193,15 @@ OR if args.use_existing: expect(args.test_id is not None, "Must provide test-id of pre-existing cases") + if args.no_setup: + args.no_build = True + if args.no_build: args.no_run = True # Namelist-only forces some other options: if args.namelists_only: + expect(not args.no_setup, "Cannot compare namelists without setup") args.no_build = True args.no_run = True args.no_batch = True @@ -208,14 +215,13 @@ OR if args.test_id is None: args.test_id = CIME.utils.get_utc_timestamp() - if args.testfile is not None: with open(args.testfile[0], "r") as fd: - args.testargs.extend( fd.read().splitlines() ) + args.testargs.extend( fd.read().splitlines() ) # Remove empty items if any args.testargs = filter(None, args.testargs) - return args.testargs, args.compiler, args.machine, args.no_run, args.no_build, args.no_batch,\ + return args.testargs, args.compiler, args.machine, args.no_run, args.no_build, args.no_setup, args.no_batch,\ args.test_root, args.baseline_root, args.clean, args.compare, args.generate, \ args.baseline_name, args.namelists_only, args.project, args.test_id, args.parallel_jobs, \ args.xml_machine, args.xml_compiler, args.xml_category, args.xml_testlist, args.walltime, \ @@ -321,7 +327,7 @@ def single_submit_impl(machine_name, test_id, proc_pool, project, args, job_cost run_cmd_no_fail(submit_cmd, input_str=script, arg_stdout=None, arg_stderr=None, verbose=True) ############################################################################### -def create_test(testargs, compiler, machine_name, no_run, no_build, no_batch, test_root, +def create_test(testargs, compiler, machine_name, no_run, no_build, no_setup, no_batch, test_root, baseline_root, clean, compare, generate, baseline_name, namelists_only, project, test_id, parallel_jobs, xml_machine, xml_compiler, xml_category, xml_testlist, walltime, @@ -337,8 +343,8 @@ def create_test(testargs, compiler, machine_name, no_run, no_build, no_batch, te expect(machine_name == testsplit[4], "ambiguity in machine, please use the --machine option") - impl = SystemTest(testargs, - no_run=no_run, no_build=no_build, no_batch=no_batch, + impl = TestScheduler(testargs, + no_run=no_run, no_build=no_build, no_setup=no_setup, no_batch=no_batch, test_root=test_root, test_id=test_id, baseline_root=baseline_root, baseline_name=baseline_name, clean=clean, machine_name=machine_name, compiler=compiler, @@ -348,7 +354,7 @@ def create_test(testargs, compiler, machine_name, no_run, no_build, no_batch, te xml_category=xml_category, xml_testlist=xml_testlist, walltime=walltime, proc_pool=proc_pool, use_existing=use_existing, save_timing=save_timing) - success = impl.system_test() + success = impl.run_tests() if single_submit: # Get real test root @@ -378,17 +384,17 @@ def create_test(testargs, compiler, machine_name, no_run, no_build, no_batch, te def _main_func(description): ############################################################################### if "--test" in sys.argv: - CIME.utils.run_cmd_no_fail("python -m doctest %s/CIME/system_test.py -v" % + CIME.utils.run_cmd_no_fail("python -m doctest %s/CIME/test_scheduler.py -v" % CIME.utils.get_python_libs_root(), arg_stdout=None, arg_stderr=None) return - testargs, compiler, machine_name, no_run, no_build, no_batch, test_root, baseline_root, clean, \ + testargs, compiler, machine_name, no_run, no_build, no_setup, no_batch, test_root, baseline_root, clean, \ compare, generate, baseline_name, namelists_only, project, test_id, parallel_jobs, \ xml_machine, xml_compiler, xml_category, xml_testlist, walltime, single_submit, proc_pool, \ use_existing, save_timing \ = parse_command_line(sys.argv, description) - sys.exit(create_test(testargs, compiler, machine_name, no_run, no_build, no_batch, test_root, + sys.exit(create_test(testargs, compiler, machine_name, no_run, no_build, no_setup, no_batch, test_root, baseline_root, clean, compare, generate, baseline_name, namelists_only, project, test_id, parallel_jobs, xml_machine, xml_compiler, xml_category, xml_testlist, walltime, single_submit, proc_pool, use_existing, save_timing)) diff --git a/utils/python/CIME/SystemTests/README b/utils/python/CIME/SystemTests/README new file mode 100644 index 000000000000..c2aed89d6948 --- /dev/null +++ b/utils/python/CIME/SystemTests/README @@ -0,0 +1,140 @@ +The following are the test functionality categories: + 1) smoke tests + 2) restart tests + 3) threading/pe-count modification tests + 4) sequencing (layout) modification tests + 5) multi-instance tests + 6) performance tests + 7) spinup tests (TODO) + +Some tests not yet implemented in python. They can be found in +cime/scripts/Testing/Testcases + + +NOTES: +- IOP is currently not functional + +====================================================================== + Smoke Tests +====================================================================== + +SMS smoke startup test (default length) + do a 5 day initial test (suffix: base) + if $IOP_ON is set then suffix is base_iop + success for non-iop is just a successful coupler + +====================================================================== + Restart Tests +====================================================================== + +ERS exact restart from startup (default 6 days + 5 days) + do an 11 day initial test - write a restart at day 6 (suffix: base) + if $IOP_ON is set then suffix is base_iop + do a 5 day restart test starting from restart at day 6 (suffix: rest) + if $IOP_ON is set then suffix is rest_iop + compare component history files ".base" and ".rest" at day 11 + +ERP pes counts hybrid (open-MP/MPI) restart bfb test from startup, default 6 days + 5 days (previousy PER) + initial pes set up out of the box + do an 11 day initial test - write a restart at day 6 (suffix base) + half the number of tasks and threads for each component + do a 5 day restart test starting from restart at day 6 (suffix rest) + this is just like an ERS test but the pe-counts/threading count are modified on retart + +ERI hybrid/branch/exact restart test, default (by default STOP_N is 22 days) + (1) ref1case + do an initial for ${STOP_N}/6 writing restarts at ${STOP_N}/6 + ref1 case is a clone of the main case (by default this will be 4 days) + short term archiving is on + (2) ref2case + do a hybrid for ${STOP_N}-${STOP_N}/6 running with ref1 restarts from ${STOP_N}/6 + and writing restarts at ( ${STOP_N} - ${STOP_N}/6 )/2 +1 + (by default will run for 18 days and write a restart after 10 days) + ref2 case is a clone of the main case + short term archiving is on + (3) case + do a branch run starting from restart written in ref2 case + and run for ??? days + (4) case do a restart run from the branch case + +ERT Similar to ERS but longer. 2 months + 1 month + + +====================================================================== + Restart and Archive Tests +====================================================================== +ERR does an ERS test except that after the initial run the short term archive tool is run + which moves model output out of the run directory into the short-term archive directory + then the restart run is staged from the short term archive directory. In batch mode there are + four submitted jobs for this test (mira excepted) these are run1, sta1, run2 and sta2 + run1 and sta1 are submitted together with RESUBMIT=1. sta1 has a batch system dependancy + on successful completion of run1, when sta1 is completed it uses the cime resubmit capabilty + to submit run2. + + +====================================================================== + Threading/PE-Counts/Pe-Sequencing Tests +====================================================================== + +PET modified threading openmp bfb test (seq tests) + do an initial run where all components are threaded by default (suffix: base) + do another initial run with nthrds=1 for all components (suffix: single_thread) + compare base and single_thread + +PEM modified pe counts mpi bfb test (seq tests) + do an initial run with default pe layout (suffix: base) + do another initial run with modified pes (NTASKS_XXX => NTASKS_XXX/2) (suffix: modpes) + compare base and single_thread + +PEA single pe bfb test + do an initial run on 1 pe with mpi (suffix: base) + do the same run on 1 pe with mpiserial (suffix: mpiserial) + +====================================================================== + Sequencing (layout) Tests (smoke) +====================================================================== + +SEQ different sequencing bfb test + do an initial run test with out-of-box PE-layout (suffix: base) + do a second run where all root pes are at pe-0 (suffix: seq) + compare base and seq + +====================================================================== + Multi-Instance Tests (smoke) +====================================================================== + +NCK multi-instance validation vs single instance - sequential PE for instances (default length) + do an initial run test with NINST 1 (suffix: base) + do an initial run test with NINST 2 (suffix: multiinst for both _0001 and _0002) + compare base and _0001 and _0002 + +NCR multi-instance validation vs single instance - concurrent PE for instances (default length) + do an initial run test with NINST 1 (suffix: base) + do an initial run test with NINST 2 (suffix: multiinst for both _0001 and _0002) + compare base and _0001 and _0002 + (***note that NCR_script and NCK_script are the same - but NCR_build.csh and NCK_build.csh are different***) + +NOC multi-instance validation for single instance ocean (default length) + do an initial run test with NINST 2 (other than ocn), with mod to instance 1 (suffix: inst1_base, inst2_mod) + do an initial run test with NINST 2 (other than ocn), with mod to instance 2 (suffix: inst1_base, inst2_mod) + compare inst1_base with inst2_base + compare inst1_mod with inst2_mod + + +====================================================================== + Performance Tests +====================================================================== + +PFS system performance test. Do 20 day run, no restarts +ICP cice performance test + +====================================================================== + SPINUP tests +====================================================================== + +SSP smoke CLM spinup test (only valid for CLM compsets with CLM45 and CN or BGC) (TODO - change to SPL) + do an initial spin test (setting CLM_BLDNML_OTPS to -bgc_spinup_on) + write restarts at the end of the run + short term archiving is on + do a hybrid non-spinup run run from the restart files generated in the first phase + diff --git a/utils/python/CIME/SystemTests/system_tests_common.py b/utils/python/CIME/SystemTests/system_tests_common.py index aeab6fd2bf80..849cd9ec4835 100644 --- a/utils/python/CIME/SystemTests/system_tests_common.py +++ b/utils/python/CIME/SystemTests/system_tests_common.py @@ -366,7 +366,13 @@ def generate_baseline(self): append_status("Error in Baseline Generate: %s"%err,sfile="TestStatus.log") class FakeTest(SystemTestsCommon): - + ''' + Inheriters of the FakeTest Class are intended to test the code. + + All members of the FakeTest Class must + have names beginnig with "TEST" this is so that the find_system_test + in utils.py will work with these classes. + ''' def _set_script(self, script): self._script = script # pylint: disable=attribute-defined-outside-init diff --git a/utils/python/CIME/XML/env_test.py b/utils/python/CIME/XML/env_test.py index 11e324ae3b13..4ccde3408e52 100644 --- a/utils/python/CIME/XML/env_test.py +++ b/utils/python/CIME/XML/env_test.py @@ -91,3 +91,15 @@ def run_phase_get_clone_name(self, phase): return node.attrib["clone"] return None + def cleanupnode(self, node): + ''' + keep the values component set + ''' + fnode = node.find(".//file") + node.remove(fnode) + gnode = node.find(".//group") + node.remove(gnode) + dnode = node.find(".//default_value") + if dnode is not None: + node.remove(dnode) + return node diff --git a/utils/python/CIME/case.py b/utils/python/CIME/case.py index 30209d39c4f0..63dd6a3a5de3 100644 --- a/utils/python/CIME/case.py +++ b/utils/python/CIME/case.py @@ -538,20 +538,18 @@ def configure(self, compset_name, grid_name, machine_name=None, #-------------------------------------------- # pe payout #-------------------------------------------- - match1 = None - match2 = None + match1 = re.match('([0-9]+)x([0-9]+)', "" if pecount is None else pecount) + match2 = re.match('([0-9]+)', "" if pecount is None else pecount) pes_ntasks = {} pes_nthrds = {} pes_rootpe = {} - if pecount is not None: - match1 = re.match('([0-9]+)x([0-9]+)', pecount) - match2 = re.match('([0-9]+)', pecount) - if match1: - opti_tasks = match1.group(1) - opti_thrds = match1.group(2) - elif match2: - opti_tasks = match2.group(1) - opti_thrds = 1 + if match1: + opti_tasks = match1.group(1) + opti_thrds = match1.group(2) + elif match2: + opti_tasks = match2.group(1) + opti_thrds = 1 + if match1 or match2: for component_class in self._component_classes: if component_class == "DRV": diff --git a/utils/python/CIME/case_test.py b/utils/python/CIME/case_test.py index 1ff8a5f90c03..af17b9a1ce16 100644 --- a/utils/python/CIME/case_test.py +++ b/utils/python/CIME/case_test.py @@ -2,24 +2,7 @@ Run a testcase. """ -from CIME.utils import expect - -from CIME.SystemTests.eri import ERI # pylint: disable=unused-import -from CIME.SystemTests.err import ERR # pylint: disable=unused-import -from CIME.SystemTests.erp import ERP # pylint: disable=unused-import -from CIME.SystemTests.ers import ERS # pylint: disable=unused-import -from CIME.SystemTests.ert import ERT # pylint: disable=unused-import -from CIME.SystemTests.lii import LII # pylint: disable=unused-import -from CIME.SystemTests.nck import NCK # pylint: disable=unused-import -from CIME.SystemTests.pea import PEA # pylint: disable=unused-import -from CIME.SystemTests.pem import PEM # pylint: disable=unused-import -from CIME.SystemTests.pet import PET # pylint: disable=unused-import -from CIME.SystemTests.pfs import PFS # pylint: disable=unused-import -from CIME.SystemTests.rep import REP # pylint: disable=unused-import -from CIME.SystemTests.sms import SMS # pylint: disable=unused-import -from CIME.SystemTests.seq import SEQ # pylint: disable=unused-import -from CIME.SystemTests.ssp import SSP # pylint: disable=unused-import - +from CIME.utils import expect, find_system_test from CIME.SystemTests.system_tests_common import * def case_test(case, testname=None): @@ -30,8 +13,7 @@ def case_test(case, testname=None): logging.warn("Running test for %s" % testname) try: - test = globals()[testname](case) - + test = find_system_test(testname, case)(case) success = test.run() test.report() diff --git a/utils/python/CIME/check_input_data.py b/utils/python/CIME/check_input_data.py index 7790313613d4..069776916d91 100644 --- a/utils/python/CIME/check_input_data.py +++ b/utils/python/CIME/check_input_data.py @@ -107,8 +107,10 @@ def check_input_data(case, svn_loc=None, input_data_root=None, data_list_dir="Bu no_files_missing = False else: no_files_missing = False - else: - logging.info("Already had input file: '%s'" % full_path) + + else: + logging.info("Already had input file: '%s'" % full_path) + else: model = os.path.basename(data_list_file).split('.')[0] logging.warning("Model %s no file specified for %s"%(model,description)) diff --git a/utils/python/CIME/system_test.py b/utils/python/CIME/test_scheduler.py similarity index 97% rename from utils/python/CIME/system_test.py rename to utils/python/CIME/test_scheduler.py index 20f2f3e4bca8..5928146af18a 100644 --- a/utils/python/CIME/system_test.py +++ b/utils/python/CIME/test_scheduler.py @@ -1,6 +1,9 @@ """ -Implementation of System Test functionality from CIME +A library for scheduling/running through the phases of a set +of system tests. Supports phase-level parallelism (can make progres +on multiple system tests at once). """ + import shutil, traceback, stat, glob, threading, time, thread from CIME.XML.standard_module_setup import * import compare_namelists @@ -30,12 +33,12 @@ logger = logging.getLogger(__name__) ############################################################################### -class SystemTest(object): +class TestScheduler(object): ############################################################################### ########################################################################### def __init__(self, test_names, - no_run=False, no_build=False, no_batch=None, + no_run=False, no_build=False, no_setup=False, no_batch=None, test_root=None, test_id=None, machine_name=None, compiler=None, baseline_root=None, baseline_name=None, @@ -61,8 +64,9 @@ def __init__(self, test_names, self._machobj = Machines(machine=machine_name) machine_name = self._machobj.get_machine_name() - self._no_build = no_build if not namelists_only else True - self._no_run = no_run if not self._no_build else True + self._no_setup = no_setup + self._no_build = no_build or no_setup or namelists_only + self._no_run = no_run or self._no_build # Figure out what project to use if project is None: @@ -203,10 +207,12 @@ def __init__(self, test_names, # Setup phases self._phases = list(PHASES) - if no_build: + if self._no_setup: + self._phases.remove(SETUP_PHASE) + if self._no_build: self._phases.remove(SHAREDLIB_BUILD_PHASE) self._phases.remove(MODEL_BUILD_PHASE) - if no_run: + if self._no_run: self._phases.remove(RUN_PHASE) if not self._compare and not self._generate: self._phases.remove(NAMELIST_PHASE) @@ -410,6 +416,7 @@ def _xml_phase(self, test): files = Files() drv_config_file = files.get_value("CONFIG_DRV_FILE") drv_comp = Component(drv_config_file) + envtest.add_elements_by_group(files, {}, "env_test.xml") envtest.add_elements_by_group(drv_comp, {}, "env_test.xml") envtest.set_value("TESTCASE", test_case) envtest.set_value("TEST_TESTID", self._test_id) @@ -793,28 +800,32 @@ def _setup_cs_files(self): template_file = os.path.join(python_libs_root, "cs.submit.template") template = open(template_file, "r").read() - build_cmd = "./*.build" if self._no_build else ":" - cmd = "./*.test" if self._no_batch else "./*.submit" - template = template.replace("", build_cmd).\ - replace("", cmd).\ + setup_cmd = "./case.setup" if self._no_setup else ":" + build_cmd = "./case.build" if self._no_build else ":" + test_cmd = "./case.submit" + template = template.replace("", setup_cmd).\ + replace("", build_cmd).\ + replace("", test_cmd).\ replace("", self._test_id) - if self._no_build or self._no_run: + if self._no_run: cs_submit_file = os.path.join(self._test_root, "cs.submit.%s" % self._test_id) with open(cs_submit_file, "w") as fd: fd.write(template) os.chmod(cs_submit_file, os.stat(cs_submit_file).st_mode | stat.S_IXUSR | stat.S_IXGRP) + if CIME.utils.get_model == "cesm": testreporter = os.path.join(self._test_root,"testreporter.pl") shutil.copy(os.path.join(self._cime_root,"scripts","Testing","testreporter.pl"), testreporter) os.chmod(testreporter, os.stat(testreporter).st_mode | stat.S_IXUSR | stat.S_IXGRP) + except Exception as e: logger.warning("FAILED to set up cs files: %s" % str(e)) ########################################################################### - def system_test(self): + def run_tests(self): ########################################################################### """ Main API for this class. @@ -838,7 +849,7 @@ def system_test(self): self._setup_cs_files() # Return True if all tests passed - logger.info( "At system_test close, state is:") + logger.info( "At test-scheduler close, state is:") rv = True for test in self._tests: phase, status, nl_fail = self._get_test_data(test) @@ -864,6 +875,6 @@ def system_test(self): logger.info( " Case dir: %s" % self._get_test_dir(test)) - logger.info( "system_test took %s seconds"% (time.time() - start_time)) + logger.info( "test-scheduler took %s seconds"% (time.time() - start_time)) return rv diff --git a/utils/python/CIME/test_utils.py b/utils/python/CIME/test_utils.py index 5ef436b2ffd8..74e59941cc06 100644 --- a/utils/python/CIME/test_utils.py +++ b/utils/python/CIME/test_utils.py @@ -1,5 +1,5 @@ """ -Utility functions used in system_test.py +Utility functions used in test_scheduler.py """ from CIME.XML.standard_module_setup import * diff --git a/utils/python/CIME/user_mod_support.py b/utils/python/CIME/user_mod_support.py index a0ad28f7d3b6..0efcb557c4de 100644 --- a/utils/python/CIME/user_mod_support.py +++ b/utils/python/CIME/user_mod_support.py @@ -11,36 +11,45 @@ def apply_user_mods(caseroot, user_mods_path, ninst=None): ''' Recursivlely apply user_mods to caseroot - this includes updating user_nl_xxx, - updating SourceMods and creating case_shel_commands and xmlchange_cmds files + updating SourceMods and creating case shell_commands and xmlchange_cmds files + + First remove case shell_commands files if any already exist ''' + case_shell_command_files = [os.path.join(caseroot,"shell_commands"), + os.path.join(caseroot,"xmlchange_cmnds")] + for shell_command_file in case_shell_command_files: + if os.path.isfile(shell_command_file): + os.remove(shell_command_file) + include_dirs = build_include_dirs_list(user_mods_path) for include_dir in include_dirs: # write user_nl_xxx file in caseroot for user_nl in glob.iglob(os.path.join(include_dir,"user_nl_*")): with open(os.path.join(include_dir, user_nl), "r") as fd: - contents = fd.read() + newcontents = fd.read() case_user_nl = user_nl.replace(include_dir, caseroot) comp = case_user_nl.split('_')[-1] if ninst is not None and comp in ninst.keys(): for comp_inst in xrange(1, ninst[comp]+1): + contents = newcontents case_user_nl_inst = case_user_nl + "_%4.4d"%comp_inst logger.info("Pre-pending file %s"%case_user_nl_inst) if os.path.isfile(case_user_nl_inst): with open(case_user_nl_inst, "r") as fd: old_contents = fd.read() - contents = contents + old_contents - with open(case_user_nl_inst, "w") as fd: - fd.write(contents) - - with open(case_user_nl_inst, "a") as fd: + if old_contents.find(contents) == -1: + contents = contents + old_contents + with open(case_user_nl_inst, "w") as fd: fd.write(contents) else: + contents = newcontents logger.info("Pre-pending file %s"%case_user_nl) if os.path.isfile(case_user_nl): with open(case_user_nl, "r") as fd: old_contents = fd.read() - contents = contents + old_contents + if old_contents.find(contents) == -1: + contents = contents + old_contents with open(case_user_nl, "w") as fd: fd.write(contents) @@ -70,9 +79,7 @@ def apply_user_mods(caseroot, user_mods_path, ninst=None): with open(case_shell_commands, "a") as fd: fd.write(new_shell_commands) - shell_command_files = [os.path.join(caseroot,"shell_commands"), - os.path.join(caseroot,"xmlchange_cmnds")] - for shell_command_file in shell_command_files: + for shell_command_file in case_shell_command_files: if os.path.isfile(shell_command_file): os.chmod(shell_command_file, 0777) run_cmd_no_fail(shell_command_file) diff --git a/utils/python/CIME/utils.py b/utils/python/CIME/utils.py index ee5876bb9aab..a29224b8093a 100644 --- a/utils/python/CIME/utils.py +++ b/utils/python/CIME/utils.py @@ -3,6 +3,7 @@ Warning: you cannot use CIME Classes in this module as it causes circular dependencies """ import logging, gzip, sys, os, time, re, shutil +from importlib import import_module # Return this error code if the scripts worked but tests failed TESTS_FAILED_ERR_CODE = 100 @@ -872,3 +873,48 @@ def touch(fname): os.utime(fname, None) else: open(fname, 'a').close() + +def find_system_test(testname, case): + ''' + Find and import the test matching testname + Look through the paths set in config_files.xml variable SYSTEM_TESTS_DIR + for components used in this case to find a test matching testname. Add the + path to that directory to sys.path if its not there and return the test object + Fail if the test is not found in any of the paths. + ''' + system_test_path = None + if testname.startswith("TEST"): + system_test_path = "CIME.SystemTests.system_tests_common.%s"%(testname) + else: + components = ["any"] + components.extend( case.get_compset_components()) + env_test = case.get_env("test") + for component in components: + tdir = env_test.get_value("SYSTEM_TESTS_DIR", + attribute={"component":component}) + + if tdir is not None: + tdir = os.path.abspath(tdir) + system_test_file = os.path.join(tdir ,"%s.py"%testname.lower()) + if os.path.isfile(system_test_file): + logger.debug( "found "+system_test_file) + if component == "any": + system_test_path = "CIME.SystemTests.%s.%s"%(testname.lower(),testname) + else: + system_test_dir = os.path.dirname(system_test_file) + if system_test_dir not in sys.path: + sys.path.append(system_test_dir) + system_test_path = "%s.%s"%(testname.lower(),testname) + break + + expect(system_test_path is not None, "No test %s found"%testname) + + path, m = system_test_path.rsplit('.',1) + mod = import_module(path) + return getattr(mod, m) + + + + + + diff --git a/utils/python/cs.submit.template b/utils/python/cs.submit.template index a04f875266ae..34c4eb297142 100644 --- a/utils/python/cs.submit.template +++ b/utils/python/cs.submit.template @@ -2,6 +2,6 @@ for item in $(\ls -1 *./TestStatus); do cd $(dirname $item) - && + && && cd - done diff --git a/utils/python/tests/scripts_regression_tests.py b/utils/python/tests/scripts_regression_tests.py index 58ea89c3000f..5a832502c58f 100755 --- a/utils/python/tests/scripts_regression_tests.py +++ b/utils/python/tests/scripts_regression_tests.py @@ -13,8 +13,8 @@ from CIME.utils import run_cmd, run_cmd_no_fail import CIME.utils, update_acme_tests, wait_for_tests -import CIME.system_test -from CIME.system_test import SystemTest +import CIME.test_scheduler +from CIME.test_scheduler import TestScheduler from CIME.XML.machines import Machines from CIME.XML.files import Files from CIME.case import Case @@ -476,7 +476,7 @@ def test_create_test_rebless_namelist(self): self.simple_test(True, "-c -n -b %s -t %s-%s" % (self._baseline_name, self._baseline_name, CIME.utils.get_utc_timestamp())) ############################################################################### -class E_TestSystemTest(TestCreateTestCommon): +class E_TestTestScheduler(TestCreateTestCommon): ############################################################################### ########################################################################### @@ -487,7 +487,7 @@ def test_a_phases(self): "^TESTMEMLEAKFAIL_Mmpi-serial.f19_g16.X", "^TESTMEMLEAKPASS_Mmpi-serial.f19_g16.X"], self._machine, self._compiler) self.assertEqual(len(tests), 3) - ct = SystemTest(tests) + ct = TestScheduler(tests) build_fail_test = [item for item in tests if "TESTBUILDFAIL" in item][0] run_fail_test = [item for item in tests if "TESTRUNFAIL" in item][0] @@ -499,9 +499,9 @@ def test_a_phases(self): for idx, phase in enumerate(ct._phases): for test in ct._tests: - if (phase == CIME.system_test.INITIAL_PHASE): + if (phase == CIME.test_scheduler.INITIAL_PHASE): continue - elif (phase == CIME.system_test.MODEL_BUILD_PHASE): + elif (phase == CIME.test_scheduler.MODEL_BUILD_PHASE): ct._update_test_status(test, phase, wait_for_tests.TEST_PENDING_STATUS) if (test == build_fail_test): @@ -513,7 +513,7 @@ def test_a_phases(self): self.assertFalse(ct._is_broken(test)) self.assertTrue(ct._work_remains(test)) - elif (phase == CIME.system_test.RUN_PHASE): + elif (phase == CIME.test_scheduler.RUN_PHASE): if (test == build_fail_test): with self.assertRaises(SystemExit): ct._update_test_status(test, phase, wait_for_tests.TEST_PENDING_STATUS) @@ -557,7 +557,7 @@ def test_b_full(self): ########################################################################### tests = update_acme_tests.get_full_test_names(["acme_test_only"], self._machine, self._compiler) test_id="%s-%s" % (self._baseline_name, CIME.utils.get_utc_timestamp()) - ct = SystemTest(tests, test_id=test_id, no_batch=NO_BATCH) + ct = TestScheduler(tests, test_id=test_id, no_batch=NO_BATCH) build_fail_test = [item for item in tests if "TESTBUILDFAIL" in item][0] run_fail_test = [item for item in tests if "TESTRUNFAIL" in item][0] @@ -568,7 +568,7 @@ def test_b_full(self): log_lvl = logging.getLogger().getEffectiveLevel() logging.disable(logging.CRITICAL) try: - ct.system_test() + ct.run_tests() finally: logging.getLogger().setLevel(log_lvl) @@ -581,16 +581,16 @@ def test_b_full(self): for test_status in test_statuses: status, test_name = wait_for_tests.parse_test_status_file(test_status) if (test_name == build_fail_test): - self.assertEqual(status[CIME.system_test.MODEL_BUILD_PHASE], wait_for_tests.TEST_FAIL_STATUS) + self.assertEqual(status[CIME.test_scheduler.MODEL_BUILD_PHASE], wait_for_tests.TEST_FAIL_STATUS) elif (test_name == run_fail_test): - self.assertEqual(status[CIME.system_test.RUN_PHASE], wait_for_tests.TEST_FAIL_STATUS) + self.assertEqual(status[CIME.test_scheduler.RUN_PHASE], wait_for_tests.TEST_FAIL_STATUS) elif (test_name == mem_fail_test): self.assertTrue("memleak" in status, "memleak missing in %s for test %s" % (status, test_name)) self.assertEqual(status["memleak"], wait_for_tests.TEST_FAIL_STATUS) - self.assertEqual(status[CIME.system_test.RUN_PHASE], wait_for_tests.TEST_PASS_STATUS) + self.assertEqual(status[CIME.test_scheduler.RUN_PHASE], wait_for_tests.TEST_PASS_STATUS) else: self.assertTrue(test_name in [pass_test, mem_pass_test]) - self.assertEqual(status[CIME.system_test.RUN_PHASE], wait_for_tests.TEST_PASS_STATUS) + self.assertEqual(status[CIME.test_scheduler.RUN_PHASE], wait_for_tests.TEST_PASS_STATUS) if (test_name == mem_pass_test): self.assertEqual(status["memleak"], wait_for_tests.TEST_PASS_STATUS) @@ -731,7 +731,8 @@ def simple_test(self, expect_works, extra_args): if (self._hasbatch): self.assertEqual(stat, 0, msg="COMMAND '%s' SHOULD HAVE WORKED\ncreate_test output:\n%s\n\nerrput:\n%s\n\ncode: %d" % (cmd, output, errput, stat)) test_id = extra_args.split()[extra_args.split().index("-t") + 1] - stat, output, errput = run_cmd("%s/wait_for_tests *%s*/TestStatus" % (TOOLS_DIR, test_id), from_dir=self._testroot) + cmd = "%s/wait_for_tests *%s*/TestStatus" % (TOOLS_DIR, test_id) + stat, output, errput = run_cmd(cmd, from_dir=self._testroot) if (expect_works): self.assertEqual(stat, 0, msg="COMMAND '%s' SHOULD HAVE WORKED\nOutput:\n%s\n\nerrput:\n%s\n\ncode: %d" % (cmd, output, errput, stat))