From 2b1d8a275762e2962a974c44e0cc095812542f87 Mon Sep 17 00:00:00 2001 From: psakievich Date: Tue, 30 Apr 2024 20:58:46 -0600 Subject: [PATCH] Updates based on use for my dotfiles repo (#597) * Refactor spack-manager to be a plugin (#577) * Refactor spack-manager to be a plugin * Update create-dev-env (#580) * Add pytest compatible patch replacement * Rework unit tests for create_dev_env * Revamp develop tests (#581) * Revamp develop tests * Give env a more specific name * Add fixture to register extension in mutable config * Update tests and calling args * Restore the external command (#582) * Restore the external command * Restore deleted line * Remove some SPACK_MANAGER env variable deps * Missed delete * Add documentation for how to setup a new project (#583) * Add introduction to projects concept * Update the project creation docs - Covers creating and populating configs * Machine detection process explained. * More updates, moving into repos * clean any projects that are loaded for unit tests * Fix doc reference and unit-test * Really fix unit-tests this time * Remove breakpoints * Make code certain a project exists * Remove unused problematic code * Add installation procedure (#586) * Add installation procedure * Fix path * Add include command (#587) * Add support for quick-commands (#588) * Add support for quick-commands * Remove lingering env var * Default view to false * Fix unit-tests * Style * Retool projects core (#589) * Retool projects core Split projects object and concept out of the manager module so they are created/updated each use * Clean up * Add interface for adding and removing projects * Style * Make include relative again * Add a `--project` filter for finding machines and creating environments (#590) * Add --project filter * Add to env creation * Style * Add a tool for configuring a new platform * Update platform_configure_tool.py * Fix scope * Add config location to find-machine * Style and test * Fix missing import * Fix unit test * F/make (#601) * Add 'make' command * Updates for spack-manager parsing * Add a -j arg for most common use case * Style and docs * Add first unit-test * Style --- check.py | 6 +- docs/conf.py | 59 ++++++------ .../developers/developer_workflow.md | 10 +- .../developers/snapshot_workflow.md | 4 + install.py | 16 +++- manager/cmd/manager.py | 2 + manager/manager_cmds/find_machine.py | 23 ++++- manager/manager_cmds/make.py | 94 +++++++++++++++++++ scripts/platform_configure_tool.py | 49 ++++++++++ tests/test_find_machine.py | 8 ++ tests/test_make.py | 26 +++++ 11 files changed, 254 insertions(+), 43 deletions(-) create mode 100644 manager/manager_cmds/make.py create mode 100755 scripts/platform_configure_tool.py create mode 100644 tests/test_make.py diff --git a/check.py b/check.py index b0631076..fc41e2a7 100644 --- a/check.py +++ b/check.py @@ -6,16 +6,18 @@ # for more details. """ -a script to make sure minimum requirements for spack-manager +a script to make sure minimum requirements for spack-manager are met. - python@3.8 or higher """ import sys + def check_spack_manager_requirements(): - if sys.version_info < (3,8): + if sys.version_info < (3, 8): raise ValueError("Spack-Manager requires Python 3.8 or higher.") + if __name__ == "__main__": check_spack_manager_requirements() diff --git a/docs/conf.py b/docs/conf.py index 66d648e4..c90dcdf2 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,42 +1,37 @@ -extensions = ['myst_parser'] +extensions = ["myst_parser"] myst_heading_anchors = 3 -templates_path = ['_templates'] -master_doc = 'index' -project = u'Spack-Manager' -copyright = u'Phil Sakievich' -author = u'Phil Sakievich' -version = u'0.1' -release = u'0.1' -exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] -pygments_style = 'sphinx' +templates_path = ["_templates"] +master_doc = "index" +project = "Spack-Manager" +copyright = "Phil Sakievich" +author = "Phil Sakievich" +version = "0.1" +release = "0.1" +exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] +pygments_style = "sphinx" todo_include_todos = False numfig = True -numfig_format = {'figure': '%s', 'table': '%s', 'code-block': '%s'} -html_theme = 'sphinx_rtd_theme' +numfig_format = {"figure": "%s", "table": "%s", "code-block": "%s"} +html_theme = "sphinx_rtd_theme" html_static_path = [] -html_theme_options = { - 'navigation_depth': 3 -} +html_theme_options = {"navigation_depth": 3} html_show_sourcelink = True html_show_copyright = False -htmlhelp_basename = 'spack-manager-doc' -latex_elements = { -} +htmlhelp_basename = "spack-manager-doc" +latex_elements = {} latex_documents = [ - (master_doc, 'spack-manager.tex', u'Spack-Manager Documentation', - author, 'manual'), -] -man_pages = [ - (master_doc, 'spack-manager', u'Spack-Manager Documentation', - [author], 1) + (master_doc, "spack-manager.tex", "Spack-Manager Documentation", author, "manual") ] +man_pages = [(master_doc, "spack-manager", "Spack-Manager Documentation", [author], 1)] texinfo_documents = [ - (master_doc, 'spack-manager', u'Spack-Manager Documentation', - author, 'Spack-Manager', 'One line description of project.', - 'Miscellaneous'), + ( + master_doc, + "spack-manager", + "Spack-Manager Documentation", + author, + "Spack-Manager", + "One line description of project.", + "Miscellaneous", + ) ] -source_suffix = { - '.rst': 'restructuredtext', - '.txt': 'markdown', - '.md': 'markdown', -} +source_suffix = {".rst": "restructuredtext", ".txt": "markdown", ".md": "markdown"} diff --git a/docs/user_profiles/developers/developer_workflow.md b/docs/user_profiles/developers/developer_workflow.md index f6f4e017..8bf1c989 100644 --- a/docs/user_profiles/developers/developer_workflow.md +++ b/docs/user_profiles/developers/developer_workflow.md @@ -61,7 +61,6 @@ To see the options for the command we can run it with the `--help` command. ```console quick-create-dev -h -+ spack-start ************************************************************* HELP MESSAGE: quick-create-dev sets up a developer environment @@ -143,7 +142,6 @@ git clone --recursive --branch main git@github.com:Exawind/exawind-driver.git ex git clone --recursive --branch master git@github.com:Exawind/nalu-wind.git git clone --recursive --branch main git@github.com:Exawind/amr-wind.git quick-create-dev -s exawind@main amr-wind@main nalu-wind@master -+ spack-start + spack manager create-dev-env -s exawind@master amr-wind@main nalu-wind@master ==> Configuring spec exawind@master for development at path exawind ==> Warning: included configuration files should be updated manually [files=include.yaml] @@ -181,6 +179,10 @@ If they are newer than the install time then it will trigger an incremental buil Any changes you make in a dependency will also trigger a rebuild of the upstream software too. In this environment if you make a change in `amr-wind` it will also trigger a rebuild of the `exawind` package as well. +If you wish to just do a quick incremental build you can use the `spack manager make` command: +```console +spack manager make amr-wind -j=16 +``` ## Running Tests and Coming Back @@ -188,6 +190,10 @@ To run tests in a one off manner you can use the `spack build-env` command to ru This is further documented [here](https://sandialabs.github.io/spack-manager/user_profiles/developers/snapshot_workflow.html#running). We also have a function `build-env-dive` which is a beta feature that launches this same subshell in your terminal and dives into it. It is further documented [here](https://sandialabs.github.io/spack-manager/user_profiles/developers/useful_commands.html#build-env-dive). +Finally, if a `test` target is implementd for your software you can use `spack manager make` +```console +spack manager make --args="test -j16" amr-wind +``` If you wish to come back to an environment later, or in a new shell you can just run ```console diff --git a/docs/user_profiles/developers/snapshot_workflow.md b/docs/user_profiles/developers/snapshot_workflow.md index d5b7eb7b..ae12828d 100644 --- a/docs/user_profiles/developers/snapshot_workflow.md +++ b/docs/user_profiles/developers/snapshot_workflow.md @@ -1,5 +1,9 @@ # Snapshot Developer Workflow Example +**WARNING:** This documentation is fairly specific to ExaWind and has not been generalized for an arbitrary `Spack-Manager` project. +THe information is still useful. However, you may not translate directly or be able to follow along. + + In this tutorial we will look at how to setup a developer workflow using snapshots if they are provided on your machine. ## Setup diff --git a/install.py b/install.py index bbd91178..9886274e 100755 --- a/install.py +++ b/install.py @@ -12,20 +12,26 @@ """ import argparse -import llnl.util.tty as tty -import os -import spack.main import importlib.util +import os import sys -spec = importlib.util.spec_from_file_location("check", os.path.join(os.path.dirname(sys.argv[0]), "check.py")) +import llnl.util.tty as tty + +import spack.main + +spec = importlib.util.spec_from_file_location( + "check", os.path.join(os.path.dirname(sys.argv[0]), "check.py") +) check = importlib.util.module_from_spec(spec) sys.modules["check"] = check spec.loader.exec_module(check) parser = argparse.ArgumentParser() parser.add_argument("-s", "--scope", required=False, help="Spack scope to register spack-manager") -parser.add_argument("--test", action='store_true', help="Don't actually install but test installation process") +parser.add_argument( + "--test", action="store_true", help="Don't actually install but test installation process" +) if __name__ == "__main__": args = parser.parse_args() diff --git a/manager/cmd/manager.py b/manager/cmd/manager.py index 1d19bf79..e190a396 100644 --- a/manager/cmd/manager.py +++ b/manager/cmd/manager.py @@ -15,6 +15,7 @@ import spack.extensions.manager.manager_cmds.find_machine as find_machine import spack.extensions.manager.manager_cmds.include as include import spack.extensions.manager.manager_cmds.location as location +import spack.extensions.manager.manager_cmds.make as make # import spack.extensions.manager.manager_cmds.pin as pin # import spack.extensions.manager.manager_cmds.snapshot as snapshot @@ -44,6 +45,7 @@ def setup_parser(subparser): find_machine.add_command(sp, _subcommands) include.add_command(sp, _subcommands) location.add_command(sp, _subcommands) + make.add_command(sp, _subcommands) # pin.add_command(sp, _subcommands) # snapshot.add_command(sp, _subcommands) diff --git a/manager/manager_cmds/find_machine.py b/manager/manager_cmds/find_machine.py index af663ba8..505686ea 100644 --- a/manager/manager_cmds/find_machine.py +++ b/manager/manager_cmds/find_machine.py @@ -5,6 +5,8 @@ # This software is released under the BSD 3-clause license. See LICENSE file # for more details. +import os + import spack.extensions.manager.projects as m_proj @@ -45,10 +47,11 @@ def find_machine(verbose=False, projects=m_proj.get_projects()): def find_machine_cmd(parser, args): + projects = m_proj.get_projects(args.project) if args.list: print("Project:\t Machine:\t Detected: (+/-)") print("-" * 60) - for project in m_proj.get_projects(args.project): + for project in projects: for machine in project.machines: print( "{proj} \t {machine} \t {detected}".format( @@ -58,7 +61,16 @@ def find_machine_cmd(parser, args): ) ) return - find_machine(verbose=True, projects=m_proj.get_projects(args.project)) + elif args.config: + project, machine = find_machine(verbose=False, projects=projects) + if not project or machine == "NOT-FOUND": + return + else: + path = os.path.join(project.config_path, machine) + print(path) + return path + else: + find_machine(verbose=True, projects=projects) def setup_parser_args(sub_parser): @@ -71,6 +83,13 @@ def setup_parser_args(sub_parser): "index of the project" ), ) + sub_parser.add_argument( + "-c", + "--config", + action="store_true", + required=False, + help="location of the machine specific configs", + ) sub_parser.add_argument( "-l", diff --git a/manager/manager_cmds/make.py b/manager/manager_cmds/make.py new file mode 100644 index 00000000..9e90653a --- /dev/null +++ b/manager/manager_cmds/make.py @@ -0,0 +1,94 @@ +# Copyright (c) 2022, National Technology & Engineering Solutions of Sandia, +# LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +# Government retains certain rights in this software. +# +# This software is released under the BSD 3-clause license. See LICENSE file +# for more details. + +import os + +import llnl.util.tty as tty +from llnl.util.filesystem import working_dir + +import spack.build_environment as build_environment +import spack.builder +import spack.cmd +import spack.paths +from spack.util.executable import Executable + +""" +This was originally implemented by Tim Fuller @tjfulle +With his permission it is being published in the spack manager +subset of commands +""" +description = "make SPEC directly with `make` or `ninja`" +section = "manager" +level = "short" + + +def setup_parser(parser): + parser.add_argument( + "spec", metavar="SPEC", nargs="+", help="Spack package to build (must be a develop spec)" + ) + build_args = parser.add_mutually_exclusive_group() + build_args.add_argument( + "--args", + "-a", + default="", + required=False, + help="Additional arguments to pass to make as a string i.e. `--args='test -j16'`", + ) + build_args.add_argument( + "-j", + type=int, + required=False, + help="number of ranks to build with (specialized implementation of --args)", + ) + + +def make(parser, args): + env = spack.cmd.require_active_env(cmd_name="make") + specs = spack.cmd.parse_specs(args.spec) + if args.j: + extra_make_args = [f"-j{args.j}"] + else: + extra_make_args = args.args.split() + if not specs: + tty.die("You must supply a spec.") + if len(specs) != 1: + tty.die("Too many specs. Supply only one.") + spec = env.matching_spec(specs[0]) + if spec is None: + tty.die(f"{specs[0]}: spec not found in environment") + elif not spec.is_develop: + tty.die(f"{specs[0]}: must be a develop spec") + pkg = spec.package + builder = spack.builder.create(pkg) + if hasattr(builder, "build_directory"): + build_directory = os.path.normpath(os.path.join(pkg.stage.path, builder.build_directory)) + else: + build_directory = pkg.stage.source_path + try: + build_environment.setup_package(spec.package, False, "build") + except TypeError: + build_environment.setup_package(spec.package, False) + + if not os.path.isdir(build_directory): + tty.die( + ( + "Build directory does not exist. " + "Please run `spack install` to ensure the build is " + "configured properly" + ) + ) + + with working_dir(build_directory): + make_program = "ninja" if os.path.exists("build.ninja") else "make" + make = Executable(make_program) + make(*extra_make_args) + + +def add_command(parser, command_dict): + subparser = parser.add_parser("make", help=description) + setup_parser(subparser) + command_dict["make"] = make diff --git a/scripts/platform_configure_tool.py b/scripts/platform_configure_tool.py new file mode 100755 index 00000000..b65f5d96 --- /dev/null +++ b/scripts/platform_configure_tool.py @@ -0,0 +1,49 @@ +#! /usr/bin/env spack-python +import argparse +import os +import pathlib + +import spack.main +import spack.util.spack_yaml as syaml +from spack.util.module_cmd import module + +parser = argparse.ArgumentParser() +parser.add_argument("input", help="input file describing what to configure") +output_types = parser.add_mutually_exclusive_group(required=True) +output_types.add_argument("--output", help="location where the configs should get written") +output_types.add_argument("--scope", help="spack scope where the configs should get written") + +args = parser.parse_args() + +input_path = pathlib.PurePath(args.input) +if args.output: + output_path = pathlib.PurePath(args.output) + os.environ["SPACK_USER_CONFIG_PATH"] = str(output_path) + scope = "user" +else: + scope = args.scope + +exe_env = os.environ.copy() + +with open(input_path, "r") as f: + manifest = syaml.load(f) + +compiler = spack.main.SpackCommand("compiler", subprocess=True) +external_cmd = spack.main.SpackCommand("external", subprocess=True) + +if "compilers" in manifest: + for c in manifest["compilers"]: + module("load", c) + print(compiler("find", "--scope", scope, env=exe_env)) + module("unload", c) + +if "externals" in manifest: + print(external_cmd("find", "--scope", scope, *manifest["externals"], env=exe_env)) + +if "modules" in manifest: + for entry in manifest["modules"]: + m = entry["module"] + p = entry["packages"] + module("load", m) + print(external_cmd("find", "--scope", scope, *p, env=exe_env)) + module("unload", m) diff --git a/tests/test_find_machine.py b/tests/test_find_machine.py index 0fba26eb..53391d3c 100644 --- a/tests/test_find_machine.py +++ b/tests/test_find_machine.py @@ -35,3 +35,11 @@ def test_find_machine_filters_on_project(mock_manager_config_path): assert "moonlight" in out out = mgr_cmd("find-machine", "--project", "project_b", "--list") assert "moonlight" not in out + + +def test_find_machine_config_points_to_path(on_moonlight): + assert manager.config_path != manager._default_config_path + assert find_machine.machine_defined("moonlight") + out = mgr_cmd("find-machine", "--config") + assert "moonlight" in out + assert os.path.isdir(out.strip()) diff --git a/tests/test_make.py b/tests/test_make.py new file mode 100644 index 00000000..4df5ce67 --- /dev/null +++ b/tests/test_make.py @@ -0,0 +1,26 @@ +# Copyright (c) 2022, National Technology & Engineering Solutions of Sandia, +# LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +# Government retains certain rights in this software. +# +# This software is released under the BSD 3-clause license. See LICENSE file +# for more details. + +import pytest + +import spack.environment as ev +import spack.extensions +import spack.main + +env = spack.main.SpackCommand("env") +manager = spack.main.SpackCommand("manager") +add = spack.main.SpackCommand("add") + + +@pytest.mark.usefixtures("mutable_mock_env_path_wh_manager", "mock_packages", "mock_fetch") +def test_spackManagerMakeRequiresDevelopSpec(): + env("create", "test") + with ev.read("test"): + add("mpich") + with pytest.raises(spack.main.SpackCommandError): + out = manager("make", "mpich") + assert "must be a develop spec" in str(out)