Skip to content

Commit

Permalink
Merge pull request #1823 from ESMCI/jgfouca/build_system_upgrades
Browse files Browse the repository at this point in the history 
Upgrade build system.
Make incompatible flags to case.build mutually exclusive
Add new flag for building individual components
Never run buildnml when building individual libs/components
Test suite: by-hand, scripts_regression_tests
Test baseline:
Test namelist changes:
Test status: bit for bit

Fixes #1741

User interface changes?: Yes, new -b flag to case.build.

Update gh-pages html (Y/N)?: N

Code review: jedwards, sacks
  • Loading branch information
@jedwards4b
jedwards4b authored Aug 18, 2017
2 parents 8db85c7 + f62ff71 commit 2f9eacb
Show file tree
Hide file tree
Showing 2 changed files with 58 additions and 45 deletions.
31 changes: 19 additions & 12 deletions scripts/Tools/case.build
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,11 +36,13 @@ OR
parser.add_argument("caseroot", nargs="?", default=os.getcwd(),
help="Case directory to build")

parser.add_argument("--sharedlib-only", action="store_true",
help="Only build sharedlibs")
mutex_group = parser.add_mutually_exclusive_group()

parser.add_argument("-m", "--model-only", action="store_true",
help="Assume shared libs already built")
mutex_group.add_argument("--sharedlib-only", action="store_true",
help="Only build sharedlibs")

mutex_group.add_argument("-m", "--model-only", action="store_true",
help="Assume shared libs already built")

files = Files()
config_file = files.get_value("CONFIG_CPL_FILE")
Expand All @@ -49,18 +51,22 @@ OR
libs = ["csmshare", "mct", "pio", "gptl"]
allobjs = comps + libs

parser.add_argument("--clean", nargs="*", choices=allobjs,
help="objects to clean"
"if no arguments then clean all objects other than sharedlib objects")
mutex_group.add_argument("--clean", nargs="*", choices=allobjs,
help="objects to clean"
"if no arguments then clean all objects other than sharedlib objects")

mutex_group.add_argument("-b", "--build", nargs="+", choices=allobjs,
help="libs to build. Will cause namelist generation to be skipped.")

parser.add_argument("--clean-all", action="store_true",
help="clean all objects including sharedlibobjects that may be used by other builds")
mutex_group.add_argument("--clean-all", action="store_true",
help="clean all objects including sharedlibobjects that may be used by other builds")

args = CIME.utils.parse_args_and_handle_standard_logging_options(args, parser)

cleanlist = args.clean if args.clean is None or len(args.clean) else comps
buildlist = None if args.build is None or len(args.build) == 0 else args.build

return args.caseroot, args.sharedlib_only, args.model_only, cleanlist, args.clean_all
return args.caseroot, args.sharedlib_only, args.model_only, cleanlist, args.clean_all, buildlist

###############################################################################
def _main_func(description):
Expand All @@ -69,7 +75,7 @@ def _main_func(description):
test_results = doctest.testmod(verbose=True)
sys.exit(1 if test_results.failed > 0 else 0)

caseroot, sharedlib_only, model_only, cleanlist, clean_all = parse_command_line(sys.argv, description)
caseroot, sharedlib_only, model_only, cleanlist, clean_all, buildlist = parse_command_line(sys.argv, description)

success = True
with Case(caseroot, read_only=False) as case:
Expand All @@ -92,10 +98,11 @@ def _main_func(description):
ts.set_status(phase_to_fail, TEST_FAIL_STATUS, comments="failed to initialize")
raise

expect(buildlist is None, "Build lists don't work with tests")
success = test.build(sharedlib_only=sharedlib_only, model_only=model_only)
else:
success = build.case_build(caseroot, case=case, sharedlib_only=sharedlib_only,
model_only=model_only)
model_only=model_only, buildlist=buildlist)

sys.exit(0 if success else 1)

Expand Down
72 changes: 39 additions & 33 deletions scripts/lib/CIME/build.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,12 +12,14 @@

###############################################################################
def _build_model(build_threaded, exeroot, clm_config_opts, incroot, complist,
lid, caseroot, cimeroot, compiler):
lid, caseroot, cimeroot, compiler, buildlist):
###############################################################################
logs = []

thread_bad_results = []
for model, comp, nthrds, _, config_dir in complist:
if buildlist is not None and model.lower() not in buildlist:
continue

# aquap has a dependency on atm so we will build it after the threaded loop
if comp == "aquap":
Expand Down Expand Up @@ -73,34 +75,35 @@ def _build_model(build_threaded, exeroot, clm_config_opts, incroot, complist,
# Now build the executable
#

cime_model = get_model()
file_build = os.path.join(exeroot, "{}.bldlog.{}".format(cime_model, lid))

config_dir = os.path.join(cimeroot, "src", "drivers", "mct", "cime_config")
f = open(file_build, "w")
bldroot = os.path.join(exeroot, "cpl", "obj")
if not os.path.isdir(bldroot):
os.makedirs(bldroot)
logger.info("Building {} with output to {} ".format(cime_model, file_build))
stat = run_cmd("{}/buildexe {} {} {}"
.format(config_dir, caseroot, libroot, bldroot),
from_dir=bldroot, verbose=False, arg_stdout=f,
arg_stderr=subprocess.STDOUT)[0]
f.close()
analyze_build_log("{} exe".format(cime_model), file_build, compiler)
expect(stat == 0, "BUILD FAIL: buildexe failed, cat {}".format(file_build))
if not buildlist:
cime_model = get_model()
file_build = os.path.join(exeroot, "{}.bldlog.{}".format(cime_model, lid))

config_dir = os.path.join(cimeroot, "src", "drivers", "mct", "cime_config")
f = open(file_build, "w")
bldroot = os.path.join(exeroot, "cpl", "obj")
if not os.path.isdir(bldroot):
os.makedirs(bldroot)
logger.info("Building {} with output to {} ".format(cime_model, file_build))
stat = run_cmd("{}/buildexe {} {} {}"
.format(config_dir, caseroot, libroot, bldroot),
from_dir=bldroot, verbose=False, arg_stdout=f,
arg_stderr=subprocess.STDOUT)[0]
f.close()
analyze_build_log("{} exe".format(cime_model), file_build, compiler)
expect(stat == 0, "BUILD FAIL: buildexe failed, cat {}".format(file_build))

# Copy the just-built ${MODEL}.exe to ${MODEL}.exe.$LID
shutil.copy("{}/{}.exe".format(exeroot, cime_model), "{}/{}.exe.{}".format(exeroot, cime_model, lid))
# Copy the just-built ${MODEL}.exe to ${MODEL}.exe.$LID
shutil.copy("{}/{}.exe".format(exeroot, cime_model), "{}/{}.exe.{}".format(exeroot, cime_model, lid))

logs.append(file_build)
logs.append(file_build)

return logs

###############################################################################
def _build_checks(case, build_threaded, comp_interface, use_esmf_lib,
debug, compiler, mpilib, complist, ninst_build, smp_value,
model_only):
model_only, buildlist):
###############################################################################
"""
check if a build needs to be done and warn if a clean is warrented first
Expand Down Expand Up @@ -196,14 +199,14 @@ def _build_checks(case, build_threaded, comp_interface, use_esmf_lib,
create_dirs(case)

case.flush()
if not model_only:
if not model_only and not buildlist:
logger.info("Generating component namelists as part of build")
create_namelists(case)

return sharedpath

###############################################################################
def _build_libraries(case, exeroot, sharedpath, caseroot, cimeroot, libroot, lid, compiler):
def _build_libraries(case, exeroot, sharedpath, caseroot, cimeroot, libroot, lid, compiler, buildlist):
###############################################################################

shared_lib = os.path.join(exeroot, sharedpath, "lib")
Expand All @@ -218,6 +221,9 @@ def _build_libraries(case, exeroot, sharedpath, caseroot, cimeroot, libroot, lid
logs = []
sharedlibroot = os.path.abspath(case.get_value("SHAREDLIBROOT"))
for lib in libs:
if buildlist is not None and lib not in buildlist:
continue

if lib == "csm_share":
# csm_share adds its own dir name
full_lib_path = os.path.join(sharedlibroot, sharedpath)
Expand Down Expand Up @@ -248,12 +254,11 @@ def _build_libraries(case, exeroot, sharedpath, caseroot, cimeroot, libroot, lid
if re.search("Current setting for", line):
logger.warn(line)


# clm not a shared lib for ACME
if get_model() != "acme":
if get_model() != "acme" and (buildlist is None or "lnd" in buildlist):
comp_lnd = case.get_value("COMP_LND")
clm_config_opts = case.get_value("CLM_CONFIG_OPTS")
if comp_lnd == "clm" and not "clm4_0" in clm_config_opts:
if comp_lnd == "clm" and "clm4_0" not in clm_config_opts:
logging.info(" - Building clm4_5/clm5_0 Library ")
esmfdir = "esmf" if case.get_value("USE_ESMF_LIB") else "noesmf"
bldroot = os.path.join(sharedlibroot, sharedpath, case.get_value("COMP_INTERFACE"), esmfdir, "clm","obj" )
Expand Down Expand Up @@ -356,7 +361,7 @@ def _clean_impl(case, cleanlist, clean_all):
case.flush()

###############################################################################
def _case_build_impl(caseroot, case, sharedlib_only, model_only):
def _case_build_impl(caseroot, case, sharedlib_only, model_only, buildlist):
###############################################################################

t1 = time.time()
Expand Down Expand Up @@ -493,23 +498,24 @@ def _case_build_impl(caseroot, case, sharedlib_only, model_only):

sharedpath = _build_checks(case, build_threaded, comp_interface,
use_esmf_lib, debug, compiler, mpilib,
complist, ninst_build, smp_value, model_only)
complist, ninst_build, smp_value, model_only, buildlist)

t2 = time.time()
logs = []

if not model_only:
logs = _build_libraries(case, exeroot, sharedpath, caseroot,
cimeroot, libroot, lid, compiler)
cimeroot, libroot, lid, compiler, buildlist)

if not sharedlib_only:
os.environ["INSTALL_SHAREDPATH"] = os.path.join(exeroot, sharedpath) # for MPAS makefile generators
logs.extend(_build_model(build_threaded, exeroot, clm_config_opts, incroot, complist,
lid, caseroot, cimeroot, compiler))
lid, caseroot, cimeroot, compiler, buildlist))

if not sharedlib_only:
if not sharedlib_only and not buildlist:
# in case component build scripts updated the xml files, update the case object
case.read_xml()
# Note, doing buildlists will never result in the system thinking the build is complete
post_build(case, logs)

t3 = time.time()
Expand Down Expand Up @@ -553,9 +559,9 @@ def post_build(case, logs):
save_build_provenance(case, lid=lid)

###############################################################################
def case_build(caseroot, case, sharedlib_only=False, model_only=False):
def case_build(caseroot, case, sharedlib_only=False, model_only=False, buildlist=None):
###############################################################################
functor = lambda: _case_build_impl(caseroot, case, sharedlib_only, model_only)
functor = lambda: _case_build_impl(caseroot, case, sharedlib_only, model_only, buildlist)
return run_and_log_case_status(functor, "case.build", caseroot=caseroot)

###############################################################################
Expand Down

0 comments on commit 2f9eacb

Please sign in to comment.