Updates based on use for my dotfiles repo (#597)

* Refactor spack-manager to be a plugin (#577)

* Refactor spack-manager to be a plugin

* Update create-dev-env (#580)

* Add pytest compatible patch replacement

* Rework unit tests for create_dev_env

* Revamp develop tests (#581)

* Revamp develop tests

* Give env a more specific name

* Add fixture to register extension in mutable config

* Update tests and calling args

* Restore the external command (#582)

* Restore the external command

* Restore deleted line

* Remove some SPACK_MANAGER env variable deps

* Missed delete

* Add documentation for how to setup a new project (#583)

* Add introduction to projects concept

* Update the project creation docs
- Covers creating and populating configs

* Machine detection process explained.

* More updates, moving into repos

* clean any projects that are loaded for unit tests

* Fix doc reference and unit-test

* Really fix unit-tests this time

* Remove breakpoints

* Make code certain a project exists

* Remove unused problematic code

* Add installation procedure (#586)

* Add installation procedure

* Fix path

* Add include command (#587)

* Add support for quick-commands (#588)

* Add support for quick-commands

* Remove lingering env var

* Default view to false

* Fix unit-tests

* Style

* Retool projects core (#589)

* Retool projects core

Split projects object and concept out of the
manager module so they are created/updated each
use

* Clean up

* Add interface for adding and removing projects

* Style

* Make include relative again

* Add a `--project` filter for finding machines and creating environments (#590)

* Add --project filter

* Add to env creation

* Style

* Add a tool for configuring a new platform

* Update platform_configure_tool.py

* Fix scope

* Add config location to find-machine

* Style and test

* Fix missing import

* Fix unit test

* F/make (#601)

* Add 'make' command

* Updates for spack-manager parsing

* Add a -j arg for most common use case

* Style and docs

* Add first unit-test

* Style

check.py

@@ -6,16 +6,18 @@
 # for more details.
 
 """
-a script to make sure minimum requirements for spack-manager 
+a script to make sure minimum requirements for spack-manager
 are met.
 
 - python@3.8 or higher
 """
 import sys
 
+
 def check_spack_manager_requirements():
-    if sys.version_info < (3,8):
+    if sys.version_info < (3, 8):
         raise ValueError("Spack-Manager requires Python 3.8 or higher.")
 
+
 if __name__ == "__main__":
     check_spack_manager_requirements()

docs/conf.py

@@ -1,42 +1,37 @@
-extensions = ['myst_parser']
+extensions = ["myst_parser"]
 myst_heading_anchors = 3
-templates_path = ['_templates']
-master_doc = 'index'
-project = u'Spack-Manager'
-copyright = u'Phil Sakievich'
-author = u'Phil Sakievich'
-version = u'0.1'
-release = u'0.1'
-exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
-pygments_style = 'sphinx'
+templates_path = ["_templates"]
+master_doc = "index"
+project = "Spack-Manager"
+copyright = "Phil Sakievich"
+author = "Phil Sakievich"
+version = "0.1"
+release = "0.1"
+exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"]
+pygments_style = "sphinx"
 todo_include_todos = False
 numfig = True
-numfig_format = {'figure': '%s', 'table': '%s', 'code-block': '%s'}
-html_theme = 'sphinx_rtd_theme'
+numfig_format = {"figure": "%s", "table": "%s", "code-block": "%s"}
+html_theme = "sphinx_rtd_theme"
 html_static_path = []
-html_theme_options = {
-    'navigation_depth': 3
-}
+html_theme_options = {"navigation_depth": 3}
 html_show_sourcelink = True
 html_show_copyright = False
-htmlhelp_basename = 'spack-manager-doc'
-latex_elements = {
-}
+htmlhelp_basename = "spack-manager-doc"
+latex_elements = {}
 latex_documents = [
-    (master_doc, 'spack-manager.tex', u'Spack-Manager Documentation',
-     author, 'manual'),
-]
-man_pages = [
-    (master_doc, 'spack-manager', u'Spack-Manager Documentation',
-     [author], 1)
+    (master_doc, "spack-manager.tex", "Spack-Manager Documentation", author, "manual")
 ]
+man_pages = [(master_doc, "spack-manager", "Spack-Manager Documentation", [author], 1)]
 texinfo_documents = [
-    (master_doc, 'spack-manager', u'Spack-Manager Documentation',
-     author, 'Spack-Manager', 'One line description of project.',
-     'Miscellaneous'),
+    (
+        master_doc,
+        "spack-manager",
+        "Spack-Manager Documentation",
+        author,
+        "Spack-Manager",
+        "One line description of project.",
+        "Miscellaneous",
+    )
 ]
-source_suffix = {
-    '.rst': 'restructuredtext',
-    '.txt': 'markdown',
-    '.md': 'markdown',
-}
+source_suffix = {".rst": "restructuredtext", ".txt": "markdown", ".md": "markdown"}

docs/user_profiles/developers/developer_workflow.md

@@ -61,7 +61,6 @@ To see the options for the command we can run it with the `--help` command.
 
 ```console
 quick-create-dev -h
-+ spack-start
 *************************************************************
 HELP MESSAGE:
 quick-create-dev sets up a developer environment
@@ -143,7 +142,6 @@ git clone --recursive --branch main git@github.com:Exawind/exawind-driver.git ex
 git clone --recursive --branch master git@github.com:Exawind/nalu-wind.git
 git clone --recursive --branch main git@github.com:Exawind/amr-wind.git
 quick-create-dev -s exawind@main amr-wind@main nalu-wind@master
-+ spack-start
 + spack manager create-dev-env -s exawind@master amr-wind@main nalu-wind@master
 ==> Configuring spec exawind@master for development at path exawind
 ==> Warning: included configuration files should be updated manually [files=include.yaml]
@@ -181,13 +179,21 @@ If they are newer than the install time then it will trigger an incremental buil
 Any changes you make in a dependency will also trigger a rebuild of the upstream software too.
 In this environment if you make a change in `amr-wind` it will also trigger a rebuild of the `exawind` package as well.
 
+If you wish to just do a quick incremental build you can use the `spack manager make` command:
+```console
+spack manager make amr-wind -j=16
+```
 
 ## Running Tests and Coming Back
 
 To run tests in a one off manner you can use the `spack build-env` command to run commands in a sub-shell with the build environment.
 This is further documented [here](https://sandialabs.github.io/spack-manager/user_profiles/developers/snapshot_workflow.html#running).
 We also have a function `build-env-dive` which is a beta feature that launches this same subshell in your terminal and dives into it.
 It is further documented [here](https://sandialabs.github.io/spack-manager/user_profiles/developers/useful_commands.html#build-env-dive).
+Finally, if a `test` target is implementd for your software you can use `spack manager make`
+```console
+spack manager make --args="test -j16" amr-wind
+```
 
 If you wish to come back to an environment later, or in a new shell you can just run
 ```console

docs/user_profiles/developers/snapshot_workflow.md

@@ -1,5 +1,9 @@
 # Snapshot Developer Workflow Example
 
+**WARNING:** This documentation is fairly specific to ExaWind and has not been generalized for an arbitrary `Spack-Manager` project.
+THe information is still useful. However, you may not translate directly or be able to follow along.
+
+
 In this tutorial we will look at how to setup a developer workflow using snapshots if they are provided on your machine.
 
 ## Setup

install.py

@@ -12,20 +12,26 @@
 """
 
 import argparse
-import llnl.util.tty as tty
-import os
-import spack.main
 import importlib.util
+import os
 import sys
 
-spec = importlib.util.spec_from_file_location("check", os.path.join(os.path.dirname(sys.argv[0]), "check.py"))
+import llnl.util.tty as tty
+
+import spack.main
+
+spec = importlib.util.spec_from_file_location(
+    "check", os.path.join(os.path.dirname(sys.argv[0]), "check.py")
+)
 check = importlib.util.module_from_spec(spec)
 sys.modules["check"] = check
 spec.loader.exec_module(check)
 
 parser = argparse.ArgumentParser()
 parser.add_argument("-s", "--scope", required=False, help="Spack scope to register spack-manager")
-parser.add_argument("--test", action='store_true', help="Don't actually install but test installation process")
+parser.add_argument(
+    "--test", action="store_true", help="Don't actually install but test installation process"
+)
 
 if __name__ == "__main__":
     args = parser.parse_args()

manager/cmd/manager.py

@@ -15,6 +15,7 @@
 import spack.extensions.manager.manager_cmds.find_machine as find_machine
 import spack.extensions.manager.manager_cmds.include as include
 import spack.extensions.manager.manager_cmds.location as location
+import spack.extensions.manager.manager_cmds.make as make
 
 # import spack.extensions.manager.manager_cmds.pin as pin
 # import spack.extensions.manager.manager_cmds.snapshot as snapshot
@@ -44,6 +45,7 @@ def setup_parser(subparser):
     find_machine.add_command(sp, _subcommands)
     include.add_command(sp, _subcommands)
     location.add_command(sp, _subcommands)
+    make.add_command(sp, _subcommands)
     # pin.add_command(sp, _subcommands)
     # snapshot.add_command(sp, _subcommands)
 

manager/manager_cmds/find_machine.py

@@ -5,6 +5,8 @@
 # This software is released under the BSD 3-clause license. See LICENSE file
 # for more details.
 
+import os
+
 import spack.extensions.manager.projects as m_proj
 
 
@@ -45,10 +47,11 @@ def find_machine(verbose=False, projects=m_proj.get_projects()):
 
 
 def find_machine_cmd(parser, args):
+    projects = m_proj.get_projects(args.project)
     if args.list:
         print("Project:\t Machine:\t Detected: (+/-)")
         print("-" * 60)
-        for project in m_proj.get_projects(args.project):
+        for project in projects:
             for machine in project.machines:
                 print(
                     "{proj} \t {machine} \t {detected}".format(
@@ -58,7 +61,16 @@ def find_machine_cmd(parser, args):
                     )
                 )
         return
-    find_machine(verbose=True, projects=m_proj.get_projects(args.project))
+    elif args.config:
+        project, machine = find_machine(verbose=False, projects=projects)
+        if not project or machine == "NOT-FOUND":
+            return
+        else:
+            path = os.path.join(project.config_path, machine)
+            print(path)
+            return path
+    else:
+        find_machine(verbose=True, projects=projects)
 
 
 def setup_parser_args(sub_parser):
@@ -71,6 +83,13 @@ def setup_parser_args(sub_parser):
             "index of the project"
         ),
     )
+    sub_parser.add_argument(
+        "-c",
+        "--config",
+        action="store_true",
+        required=False,
+        help="location of the machine specific configs",
+    )
 
     sub_parser.add_argument(
         "-l",

manager/manager_cmds/make.py

@@ -0,0 +1,94 @@
+# Copyright (c) 2022, National Technology & Engineering Solutions of Sandia,
+# LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S.
+# Government retains certain rights in this software.
+#
+# This software is released under the BSD 3-clause license. See LICENSE file
+# for more details.
+
+import os
+
+import llnl.util.tty as tty
+from llnl.util.filesystem import working_dir
+
+import spack.build_environment as build_environment
+import spack.builder
+import spack.cmd
+import spack.paths
+from spack.util.executable import Executable
+
+"""
+This was originally implemented by Tim Fuller @tjfulle
+With his permission it is being published in the spack manager
+subset of commands
+"""
+description = "make SPEC directly with `make` or `ninja`"
+section = "manager"
+level = "short"
+
+
+def setup_parser(parser):
+    parser.add_argument(
+        "spec", metavar="SPEC", nargs="+", help="Spack package to build (must be a develop spec)"
+    )
+    build_args = parser.add_mutually_exclusive_group()
+    build_args.add_argument(
+        "--args",
+        "-a",
+        default="",
+        required=False,
+        help="Additional arguments to pass to make as a string i.e. `--args='test -j16'`",
+    )
+    build_args.add_argument(
+        "-j",
+        type=int,
+        required=False,
+        help="number of ranks to build with (specialized implementation of --args)",
+    )
+
+
+def make(parser, args):
+    env = spack.cmd.require_active_env(cmd_name="make")
+    specs = spack.cmd.parse_specs(args.spec)
+    if args.j:
+        extra_make_args = [f"-j{args.j}"]
+    else:
+        extra_make_args = args.args.split()
+    if not specs:
+        tty.die("You must supply a spec.")
+    if len(specs) != 1:
+        tty.die("Too many specs.  Supply only one.")
+    spec = env.matching_spec(specs[0])
+    if spec is None:
+        tty.die(f"{specs[0]}: spec not found in environment")
+    elif not spec.is_develop:
+        tty.die(f"{specs[0]}: must be a develop spec")
+    pkg = spec.package
+    builder = spack.builder.create(pkg)
+    if hasattr(builder, "build_directory"):
+        build_directory = os.path.normpath(os.path.join(pkg.stage.path, builder.build_directory))
+    else:
+        build_directory = pkg.stage.source_path
+    try:
+        build_environment.setup_package(spec.package, False, "build")
+    except TypeError:
+        build_environment.setup_package(spec.package, False)
+
+    if not os.path.isdir(build_directory):
+        tty.die(
+            (
+                "Build directory does not exist. "
+                "Please run `spack install` to ensure the build is "
+                "configured properly"
+            )
+        )
+
+    with working_dir(build_directory):
+        make_program = "ninja" if os.path.exists("build.ninja") else "make"
+        make = Executable(make_program)
+        make(*extra_make_args)
+
+
+def add_command(parser, command_dict):
+    subparser = parser.add_parser("make", help=description)
+    setup_parser(subparser)
+    command_dict["make"] = make

scripts/platform_configure_tool.py

@@ -0,0 +1,49 @@
+#! /usr/bin/env spack-python
+import argparse
+import os
+import pathlib
+
+import spack.main
+import spack.util.spack_yaml as syaml
+from spack.util.module_cmd import module
+
+parser = argparse.ArgumentParser()
+parser.add_argument("input", help="input file describing what to configure")
+output_types = parser.add_mutually_exclusive_group(required=True)
+output_types.add_argument("--output", help="location where the configs should get written")
+output_types.add_argument("--scope", help="spack scope where the configs should get written")
+
+args = parser.parse_args()
+
+input_path = pathlib.PurePath(args.input)
+if args.output:
+    output_path = pathlib.PurePath(args.output)
+    os.environ["SPACK_USER_CONFIG_PATH"] = str(output_path)
+    scope = "user"
+else:
+    scope = args.scope
+
+exe_env = os.environ.copy()
+
+with open(input_path, "r") as f:
+    manifest = syaml.load(f)
+
+compiler = spack.main.SpackCommand("compiler", subprocess=True)
+external_cmd = spack.main.SpackCommand("external", subprocess=True)
+
+if "compilers" in manifest:
+    for c in manifest["compilers"]:
+        module("load", c)
+        print(compiler("find", "--scope", scope, env=exe_env))
+        module("unload", c)
+
+if "externals" in manifest:
+    print(external_cmd("find", "--scope", scope, *manifest["externals"], env=exe_env))
+
+if "modules" in manifest:
+    for entry in manifest["modules"]:
+        m = entry["module"]
+        p = entry["packages"]
+        module("load", m)
+        print(external_cmd("find", "--scope", scope, *p, env=exe_env))
+        module("unload", m)

tests/test_find_machine.py

@@ -35,3 +35,11 @@ def test_find_machine_filters_on_project(mock_manager_config_path):
     assert "moonlight" in out
     out = mgr_cmd("find-machine", "--project", "project_b", "--list")
     assert "moonlight" not in out
+
+
+def test_find_machine_config_points_to_path(on_moonlight):
+    assert manager.config_path != manager._default_config_path
+    assert find_machine.machine_defined("moonlight")
+    out = mgr_cmd("find-machine", "--config")
+    assert "moonlight" in out
+    assert os.path.isdir(out.strip())