From 73016a846dc0a69f08cf5d28308b252900ce665c Mon Sep 17 00:00:00 2001 From: James Foucar Date: Thu, 17 Aug 2017 16:40:05 -0600 Subject: [PATCH 1/2] Upgrade build system. 1) Make incompatible flags to case.build mutually exclusive 2) Add new flag for building individual components 3) Never run buildnml when building individual libs/components --- scripts/Tools/case.build | 31 ++++++++++------- scripts/lib/CIME/build.py | 70 +++++++++++++++++++++------------------ 2 files changed, 57 insertions(+), 44 deletions(-) diff --git a/scripts/Tools/case.build b/scripts/Tools/case.build index 84bb90048f2a..4f57b10028c9 100755 --- a/scripts/Tools/case.build +++ b/scripts/Tools/case.build @@ -36,11 +36,13 @@ OR parser.add_argument("caseroot", nargs="?", default=os.getcwd(), help="Case directory to build") - parser.add_argument("--sharedlib-only", action="store_true", - help="Only build sharedlibs") + mutex_group = parser.add_mutually_exclusive_group() - parser.add_argument("-m", "--model-only", action="store_true", - help="Assume shared libs already built") + mutex_group.add_argument("--sharedlib-only", action="store_true", + help="Only build sharedlibs") + + mutex_group.add_argument("-m", "--model-only", action="store_true", + help="Assume shared libs already built") files = Files() config_file = files.get_value("CONFIG_CPL_FILE") @@ -49,18 +51,22 @@ OR libs = ["csmshare", "mct", "pio", "gptl"] allobjs = comps + libs - parser.add_argument("--clean", nargs="*", choices=allobjs, - help="objects to clean" - "if no arguments then clean all objects other than sharedlib objects") + mutex_group.add_argument("--clean", nargs="*", choices=allobjs, + help="objects to clean" + "if no arguments then clean all objects other than sharedlib objects") + + mutex_group.add_argument("-b", "--build", nargs="+", choices=allobjs, + help="libs to build") - parser.add_argument("--clean-all", action="store_true", - help="clean all objects including sharedlibobjects that may be used by other builds") + mutex_group.add_argument("--clean-all", action="store_true", + help="clean all objects including sharedlibobjects that may be used by other builds") args = CIME.utils.parse_args_and_handle_standard_logging_options(args, parser) cleanlist = args.clean if args.clean is None or len(args.clean) else comps + buildlist = None if args.build is None or len(args.build) == 0 else args.build - return args.caseroot, args.sharedlib_only, args.model_only, cleanlist, args.clean_all + return args.caseroot, args.sharedlib_only, args.model_only, cleanlist, args.clean_all, buildlist ############################################################################### def _main_func(description): @@ -69,7 +75,7 @@ def _main_func(description): test_results = doctest.testmod(verbose=True) sys.exit(1 if test_results.failed > 0 else 0) - caseroot, sharedlib_only, model_only, cleanlist, clean_all = parse_command_line(sys.argv, description) + caseroot, sharedlib_only, model_only, cleanlist, clean_all, buildlist = parse_command_line(sys.argv, description) success = True with Case(caseroot, read_only=False) as case: @@ -92,10 +98,11 @@ def _main_func(description): ts.set_status(phase_to_fail, TEST_FAIL_STATUS, comments="failed to initialize") raise + expect(buildlist is None, "Build lists don't work with tests") success = test.build(sharedlib_only=sharedlib_only, model_only=model_only) else: success = build.case_build(caseroot, case=case, sharedlib_only=sharedlib_only, - model_only=model_only) + model_only=model_only, buildlist=buildlist) sys.exit(0 if success else 1) diff --git a/scripts/lib/CIME/build.py b/scripts/lib/CIME/build.py index cf146921319a..f3b414bd1659 100644 --- a/scripts/lib/CIME/build.py +++ b/scripts/lib/CIME/build.py @@ -12,12 +12,14 @@ ############################################################################### def _build_model(build_threaded, exeroot, clm_config_opts, incroot, complist, - lid, caseroot, cimeroot, compiler): + lid, caseroot, cimeroot, compiler, buildlist): ############################################################################### logs = [] thread_bad_results = [] for model, comp, nthrds, _, config_dir in complist: + if buildlist is not None and model.lower() not in buildlist: + continue # aquap has a dependency on atm so we will build it after the threaded loop if comp == "aquap": @@ -73,34 +75,35 @@ def _build_model(build_threaded, exeroot, clm_config_opts, incroot, complist, # Now build the executable # - cime_model = get_model() - file_build = os.path.join(exeroot, "{}.bldlog.{}".format(cime_model, lid)) - - config_dir = os.path.join(cimeroot, "src", "drivers", "mct", "cime_config") - f = open(file_build, "w") - bldroot = os.path.join(exeroot, "cpl", "obj") - if not os.path.isdir(bldroot): - os.makedirs(bldroot) - logger.info("Building {} with output to {} ".format(cime_model, file_build)) - stat = run_cmd("{}/buildexe {} {} {}" - .format(config_dir, caseroot, libroot, bldroot), - from_dir=bldroot, verbose=False, arg_stdout=f, - arg_stderr=subprocess.STDOUT)[0] - f.close() - analyze_build_log("{} exe".format(cime_model), file_build, compiler) - expect(stat == 0, "BUILD FAIL: buildexe failed, cat {}".format(file_build)) + if not buildlist: + cime_model = get_model() + file_build = os.path.join(exeroot, "{}.bldlog.{}".format(cime_model, lid)) + + config_dir = os.path.join(cimeroot, "src", "drivers", "mct", "cime_config") + f = open(file_build, "w") + bldroot = os.path.join(exeroot, "cpl", "obj") + if not os.path.isdir(bldroot): + os.makedirs(bldroot) + logger.info("Building {} with output to {} ".format(cime_model, file_build)) + stat = run_cmd("{}/buildexe {} {} {}" + .format(config_dir, caseroot, libroot, bldroot), + from_dir=bldroot, verbose=False, arg_stdout=f, + arg_stderr=subprocess.STDOUT)[0] + f.close() + analyze_build_log("{} exe".format(cime_model), file_build, compiler) + expect(stat == 0, "BUILD FAIL: buildexe failed, cat {}".format(file_build)) - # Copy the just-built ${MODEL}.exe to ${MODEL}.exe.$LID - shutil.copy("{}/{}.exe".format(exeroot, cime_model), "{}/{}.exe.{}".format(exeroot, cime_model, lid)) + # Copy the just-built ${MODEL}.exe to ${MODEL}.exe.$LID + shutil.copy("{}/{}.exe".format(exeroot, cime_model), "{}/{}.exe.{}".format(exeroot, cime_model, lid)) - logs.append(file_build) + logs.append(file_build) return logs ############################################################################### def _build_checks(case, build_threaded, comp_interface, use_esmf_lib, debug, compiler, mpilib, complist, ninst_build, smp_value, - model_only): + model_only, buildlist): ############################################################################### """ check if a build needs to be done and warn if a clean is warrented first @@ -196,14 +199,14 @@ def _build_checks(case, build_threaded, comp_interface, use_esmf_lib, create_dirs(case) case.flush() - if not model_only: + if not model_only and not buildlist: logger.info("Generating component namelists as part of build") create_namelists(case) return sharedpath ############################################################################### -def _build_libraries(case, exeroot, sharedpath, caseroot, cimeroot, libroot, lid, compiler): +def _build_libraries(case, exeroot, sharedpath, caseroot, cimeroot, libroot, lid, compiler, buildlist): ############################################################################### shared_lib = os.path.join(exeroot, sharedpath, "lib") @@ -218,6 +221,9 @@ def _build_libraries(case, exeroot, sharedpath, caseroot, cimeroot, libroot, lid logs = [] sharedlibroot = os.path.abspath(case.get_value("SHAREDLIBROOT")) for lib in libs: + if buildlist is not None and lib not in buildlist: + continue + if lib == "csm_share": # csm_share adds its own dir name full_lib_path = os.path.join(sharedlibroot, sharedpath) @@ -248,9 +254,8 @@ def _build_libraries(case, exeroot, sharedpath, caseroot, cimeroot, libroot, lid if re.search("Current setting for", line): logger.warn(line) - # clm not a shared lib for ACME - if get_model() != "acme": + if get_model() != "acme" and not (buildlist is not None and "lnd" not in buildlist): comp_lnd = case.get_value("COMP_LND") clm_config_opts = case.get_value("CLM_CONFIG_OPTS") if comp_lnd == "clm" and not "clm4_0" in clm_config_opts: @@ -356,7 +361,7 @@ def _clean_impl(case, cleanlist, clean_all): case.flush() ############################################################################### -def _case_build_impl(caseroot, case, sharedlib_only, model_only): +def _case_build_impl(caseroot, case, sharedlib_only, model_only, buildlist): ############################################################################### t1 = time.time() @@ -493,23 +498,24 @@ def _case_build_impl(caseroot, case, sharedlib_only, model_only): sharedpath = _build_checks(case, build_threaded, comp_interface, use_esmf_lib, debug, compiler, mpilib, - complist, ninst_build, smp_value, model_only) + complist, ninst_build, smp_value, model_only, buildlist) t2 = time.time() logs = [] if not model_only: logs = _build_libraries(case, exeroot, sharedpath, caseroot, - cimeroot, libroot, lid, compiler) + cimeroot, libroot, lid, compiler, buildlist) if not sharedlib_only: os.environ["INSTALL_SHAREDPATH"] = os.path.join(exeroot, sharedpath) # for MPAS makefile generators logs.extend(_build_model(build_threaded, exeroot, clm_config_opts, incroot, complist, - lid, caseroot, cimeroot, compiler)) + lid, caseroot, cimeroot, compiler, buildlist)) - if not sharedlib_only: + if not sharedlib_only and not buildlist: # in case component build scripts updated the xml files, update the case object case.read_xml() + # Note, doing buildlists will never result in the system thinking the build is complete post_build(case, logs) t3 = time.time() @@ -553,9 +559,9 @@ def post_build(case, logs): save_build_provenance(case, lid=lid) ############################################################################### -def case_build(caseroot, case, sharedlib_only=False, model_only=False): +def case_build(caseroot, case, sharedlib_only=False, model_only=False, buildlist=None): ############################################################################### - functor = lambda: _case_build_impl(caseroot, case, sharedlib_only, model_only) + functor = lambda: _case_build_impl(caseroot, case, sharedlib_only, model_only, buildlist) return run_and_log_case_status(functor, "case.build", caseroot=caseroot) ############################################################################### From f62ff714274ba6cef92ae01d2a4c99a18691d326 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Fri, 18 Aug 2017 11:42:12 -0600 Subject: [PATCH 2/2] Updates based on github feedback --- scripts/Tools/case.build | 2 +- scripts/lib/CIME/build.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/Tools/case.build b/scripts/Tools/case.build index 4f57b10028c9..c8ec78096c07 100755 --- a/scripts/Tools/case.build +++ b/scripts/Tools/case.build @@ -56,7 +56,7 @@ OR "if no arguments then clean all objects other than sharedlib objects") mutex_group.add_argument("-b", "--build", nargs="+", choices=allobjs, - help="libs to build") + help="libs to build. Will cause namelist generation to be skipped.") mutex_group.add_argument("--clean-all", action="store_true", help="clean all objects including sharedlibobjects that may be used by other builds") diff --git a/scripts/lib/CIME/build.py b/scripts/lib/CIME/build.py index f3b414bd1659..ee7ac269f970 100644 --- a/scripts/lib/CIME/build.py +++ b/scripts/lib/CIME/build.py @@ -255,10 +255,10 @@ def _build_libraries(case, exeroot, sharedpath, caseroot, cimeroot, libroot, lid logger.warn(line) # clm not a shared lib for ACME - if get_model() != "acme" and not (buildlist is not None and "lnd" not in buildlist): + if get_model() != "acme" and (buildlist is None or "lnd" in buildlist): comp_lnd = case.get_value("COMP_LND") clm_config_opts = case.get_value("CLM_CONFIG_OPTS") - if comp_lnd == "clm" and not "clm4_0" in clm_config_opts: + if comp_lnd == "clm" and "clm4_0" not in clm_config_opts: logging.info(" - Building clm4_5/clm5_0 Library ") esmfdir = "esmf" if case.get_value("USE_ESMF_LIB") else "noesmf" bldroot = os.path.join(sharedlibroot, sharedpath, case.get_value("COMP_INTERFACE"), esmfdir, "clm","obj" )