Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Upgrade build system. #1823

Merged
merged 2 commits into from
Aug 18, 2017
Merged

Upgrade build system. #1823

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
73016a8
Upgrade build system.
jgfouca Aug 17, 2017
f62ff71
Updates based on github feedback
jgfouca Aug 18, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 19 additions & 12 deletions scripts/Tools/case.build
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,11 +36,13 @@ OR
parser.add_argument("caseroot", nargs="?", default=os.getcwd(),
help="Case directory to build")

parser.add_argument("--sharedlib-only", action="store_true",
help="Only build sharedlibs")
mutex_group = parser.add_mutually_exclusive_group()

parser.add_argument("-m", "--model-only", action="store_true",
help="Assume shared libs already built")
mutex_group.add_argument("--sharedlib-only", action="store_true",
help="Only build sharedlibs")

mutex_group.add_argument("-m", "--model-only", action="store_true",
help="Assume shared libs already built")

files = Files()
config_file = files.get_value("CONFIG_CPL_FILE")
Expand All @@ -49,18 +51,22 @@ OR
libs = ["csmshare", "mct", "pio", "gptl"]
allobjs = comps + libs

parser.add_argument("--clean", nargs="*", choices=allobjs,
help="objects to clean"
"if no arguments then clean all objects other than sharedlib objects")
mutex_group.add_argument("--clean", nargs="*", choices=allobjs,
help="objects to clean"
"if no arguments then clean all objects other than sharedlib objects")

mutex_group.add_argument("-b", "--build", nargs="+", choices=allobjs,
help="libs to build. Will cause namelist generation to be skipped.")

parser.add_argument("--clean-all", action="store_true",
help="clean all objects including sharedlibobjects that may be used by other builds")
mutex_group.add_argument("--clean-all", action="store_true",
help="clean all objects including sharedlibobjects that may be used by other builds")

args = CIME.utils.parse_args_and_handle_standard_logging_options(args, parser)

cleanlist = args.clean if args.clean is None or len(args.clean) else comps
buildlist = None if args.build is None or len(args.build) == 0 else args.build

return args.caseroot, args.sharedlib_only, args.model_only, cleanlist, args.clean_all
return args.caseroot, args.sharedlib_only, args.model_only, cleanlist, args.clean_all, buildlist

###############################################################################
def _main_func(description):
Expand All @@ -69,7 +75,7 @@ def _main_func(description):
test_results = doctest.testmod(verbose=True)
sys.exit(1 if test_results.failed > 0 else 0)

caseroot, sharedlib_only, model_only, cleanlist, clean_all = parse_command_line(sys.argv, description)
caseroot, sharedlib_only, model_only, cleanlist, clean_all, buildlist = parse_command_line(sys.argv, description)

success = True
with Case(caseroot, read_only=False) as case:
Expand All @@ -92,10 +98,11 @@ def _main_func(description):
ts.set_status(phase_to_fail, TEST_FAIL_STATUS, comments="failed to initialize")
raise

expect(buildlist is None, "Build lists don't work with tests")
success = test.build(sharedlib_only=sharedlib_only, model_only=model_only)
else:
success = build.case_build(caseroot, case=case, sharedlib_only=sharedlib_only,
model_only=model_only)
model_only=model_only, buildlist=buildlist)

sys.exit(0 if success else 1)

Expand Down
72 changes: 39 additions & 33 deletions scripts/lib/CIME/build.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,12 +12,14 @@

###############################################################################
def _build_model(build_threaded, exeroot, clm_config_opts, incroot, complist,
lid, caseroot, cimeroot, compiler):
lid, caseroot, cimeroot, compiler, buildlist):
###############################################################################
logs = []

thread_bad_results = []
for model, comp, nthrds, _, config_dir in complist:
if buildlist is not None and model.lower() not in buildlist:
continue

# aquap has a dependency on atm so we will build it after the threaded loop
if comp == "aquap":
Expand Down Expand Up @@ -73,34 +75,35 @@ def _build_model(build_threaded, exeroot, clm_config_opts, incroot, complist,
# Now build the executable
#

cime_model = get_model()
file_build = os.path.join(exeroot, "{}.bldlog.{}".format(cime_model, lid))

config_dir = os.path.join(cimeroot, "src", "drivers", "mct", "cime_config")
f = open(file_build, "w")
bldroot = os.path.join(exeroot, "cpl", "obj")
if not os.path.isdir(bldroot):
os.makedirs(bldroot)
logger.info("Building {} with output to {} ".format(cime_model, file_build))
stat = run_cmd("{}/buildexe {} {} {}"
.format(config_dir, caseroot, libroot, bldroot),
from_dir=bldroot, verbose=False, arg_stdout=f,
arg_stderr=subprocess.STDOUT)[0]
f.close()
analyze_build_log("{} exe".format(cime_model), file_build, compiler)
expect(stat == 0, "BUILD FAIL: buildexe failed, cat {}".format(file_build))
if not buildlist:
cime_model = get_model()
file_build = os.path.join(exeroot, "{}.bldlog.{}".format(cime_model, lid))

config_dir = os.path.join(cimeroot, "src", "drivers", "mct", "cime_config")
f = open(file_build, "w")
bldroot = os.path.join(exeroot, "cpl", "obj")
if not os.path.isdir(bldroot):
os.makedirs(bldroot)
logger.info("Building {} with output to {} ".format(cime_model, file_build))
stat = run_cmd("{}/buildexe {} {} {}"
.format(config_dir, caseroot, libroot, bldroot),
from_dir=bldroot, verbose=False, arg_stdout=f,
arg_stderr=subprocess.STDOUT)[0]
f.close()
analyze_build_log("{} exe".format(cime_model), file_build, compiler)
expect(stat == 0, "BUILD FAIL: buildexe failed, cat {}".format(file_build))

# Copy the just-built ${MODEL}.exe to ${MODEL}.exe.$LID
shutil.copy("{}/{}.exe".format(exeroot, cime_model), "{}/{}.exe.{}".format(exeroot, cime_model, lid))
# Copy the just-built ${MODEL}.exe to ${MODEL}.exe.$LID
shutil.copy("{}/{}.exe".format(exeroot, cime_model), "{}/{}.exe.{}".format(exeroot, cime_model, lid))

logs.append(file_build)
logs.append(file_build)

return logs

###############################################################################
def _build_checks(case, build_threaded, comp_interface, use_esmf_lib,
debug, compiler, mpilib, complist, ninst_build, smp_value,
model_only):
model_only, buildlist):
###############################################################################
"""
check if a build needs to be done and warn if a clean is warrented first
Expand Down Expand Up @@ -196,14 +199,14 @@ def _build_checks(case, build_threaded, comp_interface, use_esmf_lib,
create_dirs(case)

case.flush()
if not model_only:
if not model_only and not buildlist:
logger.info("Generating component namelists as part of build")
create_namelists(case)

return sharedpath

###############################################################################
def _build_libraries(case, exeroot, sharedpath, caseroot, cimeroot, libroot, lid, compiler):
def _build_libraries(case, exeroot, sharedpath, caseroot, cimeroot, libroot, lid, compiler, buildlist):
###############################################################################

shared_lib = os.path.join(exeroot, sharedpath, "lib")
Expand All @@ -218,6 +221,9 @@ def _build_libraries(case, exeroot, sharedpath, caseroot, cimeroot, libroot, lid
logs = []
sharedlibroot = os.path.abspath(case.get_value("SHAREDLIBROOT"))
for lib in libs:
if buildlist is not None and lib not in buildlist:
continue

if lib == "csm_share":
# csm_share adds its own dir name
full_lib_path = os.path.join(sharedlibroot, sharedpath)
Expand Down Expand Up @@ -248,12 +254,11 @@ def _build_libraries(case, exeroot, sharedpath, caseroot, cimeroot, libroot, lid
if re.search("Current setting for", line):
logger.warn(line)


# clm not a shared lib for ACME
if get_model() != "acme":
if get_model() != "acme" and (buildlist is None or "lnd" in buildlist):
comp_lnd = case.get_value("COMP_LND")
clm_config_opts = case.get_value("CLM_CONFIG_OPTS")
if comp_lnd == "clm" and not "clm4_0" in clm_config_opts:
if comp_lnd == "clm" and "clm4_0" not in clm_config_opts:
logging.info(" - Building clm4_5/clm5_0 Library ")
esmfdir = "esmf" if case.get_value("USE_ESMF_LIB") else "noesmf"
bldroot = os.path.join(sharedlibroot, sharedpath, case.get_value("COMP_INTERFACE"), esmfdir, "clm","obj" )
Expand Down Expand Up @@ -356,7 +361,7 @@ def _clean_impl(case, cleanlist, clean_all):
case.flush()

###############################################################################
def _case_build_impl(caseroot, case, sharedlib_only, model_only):
def _case_build_impl(caseroot, case, sharedlib_only, model_only, buildlist):
###############################################################################

t1 = time.time()
Expand Down Expand Up @@ -493,23 +498,24 @@ def _case_build_impl(caseroot, case, sharedlib_only, model_only):

sharedpath = _build_checks(case, build_threaded, comp_interface,
use_esmf_lib, debug, compiler, mpilib,
complist, ninst_build, smp_value, model_only)
complist, ninst_build, smp_value, model_only, buildlist)

t2 = time.time()
logs = []

if not model_only:
logs = _build_libraries(case, exeroot, sharedpath, caseroot,
cimeroot, libroot, lid, compiler)
cimeroot, libroot, lid, compiler, buildlist)

if not sharedlib_only:
os.environ["INSTALL_SHAREDPATH"] = os.path.join(exeroot, sharedpath) # for MPAS makefile generators
logs.extend(_build_model(build_threaded, exeroot, clm_config_opts, incroot, complist,
lid, caseroot, cimeroot, compiler))
lid, caseroot, cimeroot, compiler, buildlist))

if not sharedlib_only:
if not sharedlib_only and not buildlist:
# in case component build scripts updated the xml files, update the case object
case.read_xml()
# Note, doing buildlists will never result in the system thinking the build is complete
post_build(case, logs)

t3 = time.time()
Expand Down Expand Up @@ -553,9 +559,9 @@ def post_build(case, logs):
save_build_provenance(case, lid=lid)

###############################################################################
def case_build(caseroot, case, sharedlib_only=False, model_only=False):
def case_build(caseroot, case, sharedlib_only=False, model_only=False, buildlist=None):
###############################################################################
functor = lambda: _case_build_impl(caseroot, case, sharedlib_only, model_only)
functor = lambda: _case_build_impl(caseroot, case, sharedlib_only, model_only, buildlist)
return run_and_log_case_status(functor, "case.build", caseroot=caseroot)

###############################################################################
Expand Down