From 50c6463730df8150e1a16daa734a353edf230059 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Mon, 28 Oct 2024 16:08:26 -0400 Subject: [PATCH 1/2] feat: support PEP 723 directly fix: version based on pipx fix: subprocess on Windows fix: uv or virtualenv, test fix: windows fix: ignore errors in rmtree (3.8+ should be fine on Windows now) fix: resolve nox path on Windows chore: update for recent linting/typing additions Signed-off-by: Henry Schreiner --- docs/tutorial.rst | 11 ++- nox/_cli.py | 112 ++++++++++++++++++++++++- nox/_options.py | 7 ++ nox/project.py | 13 ++- nox/sessions.py | 9 +- nox/virtualenv.py | 28 ++++++- tests/resources/noxfile_script_mode.py | 12 +++ tests/test__cli.py | 71 ++++++++++++++++ tests/test_main.py | 43 ++++++++++ 9 files changed, 288 insertions(+), 18 deletions(-) create mode 100644 tests/resources/noxfile_script_mode.py create mode 100644 tests/test__cli.py diff --git a/docs/tutorial.rst b/docs/tutorial.rst index bb4e20e3..3a094dd3 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -597,8 +597,8 @@ the tags, so all three sessions: * flake8 -Running without the nox command -------------------------------- +Running without the nox command or adding dependencies +------------------------------------------------------ With a few small additions to your noxfile, you can support running using only a generalized Python runner, such as ``pipx run noxfile.py``, ``uv run @@ -618,6 +618,13 @@ And the following block of code: if __name__ == "__main__": nox.main() +If this comment block is present, nox will also read it, and run a custom +environment (``_nox_script_mode``) if the dependencies are not met in the +current environment. This allows you to specify dependencies for your noxfile +or a minimum version of nox here (``requires-python`` version setting not +supported yet, but planned). You can control this with +``--script-mode``/``NOX_SCRIPT_MODE``; ``none`` will deactivate it, and +``fresh`` will rebuild it; the default is ``reuse``. Next steps ---------- diff --git a/nox/_cli.py b/nox/_cli.py index bc69fc39..67b03fdc 100644 --- a/nox/_cli.py +++ b/nox/_cli.py @@ -16,12 +16,26 @@ from __future__ import annotations +import importlib.metadata +import os +import shutil +import subprocess import sys -from typing import Any +from pathlib import Path +from typing import TYPE_CHECKING, Any, NoReturn +import packaging.requirements +import packaging.utils + +import nox.command +import nox.virtualenv from nox import _options, tasks, workflow from nox._version import get_nox_version from nox.logger import setup_logging +from nox.project import load_toml + +if TYPE_CHECKING: + from collections.abc import Generator __all__ = ["execute_workflow", "main"] @@ -51,6 +65,87 @@ def execute_workflow(args: Any) -> int: ) +def get_dependencies( + req: packaging.requirements.Requirement, +) -> Generator[packaging.requirements.Requirement, None, None]: + """ + Gets all dependencies. Raises ModuleNotFoundError if a package is not installed. + """ + info = importlib.metadata.metadata(req.name) + yield req + + dist_list = info.get_all("requires-dist") or [] + extra_list = [packaging.requirements.Requirement(mk) for mk in dist_list] + for extra in req.extras: + for ireq in extra_list: + if ireq.marker and not ireq.marker.evaluate({"extra": extra}): + continue + yield from get_dependencies(ireq) + + +def check_dependencies(dependencies: list[str]) -> bool: + """ + Checks to see if a list of dependencies is currently installed. + """ + itr_deps = (packaging.requirements.Requirement(d) for d in dependencies) + deps = [d for d in itr_deps if not d.marker or d.marker.evaluate()] + + # Select the one nox dependency (required) + nox_dep = [d for d in deps if packaging.utils.canonicalize_name(d.name) == "nox"] + if not nox_dep: + msg = "Must have a nox dependency in TOML script dependencies" + raise ValueError(msg) + + try: + expanded_deps = {d for req in deps for d in get_dependencies(req)} + except ModuleNotFoundError: + return False + + for dep in expanded_deps: + if dep.specifier: + version = importlib.metadata.version(dep.name) + if not dep.specifier.contains(version): + return False + + return True + + +def run_script_mode(envdir: Path, *, reuse: bool, dependencies: list[str]) -> NoReturn: + envdir.mkdir(exist_ok=True) + noxenv = envdir.joinpath("_nox_script_mode") + venv = nox.virtualenv.get_virtualenv( + "uv", + "virtualenv", + reuse_existing=reuse, + envdir=str(noxenv), + ) + venv.create() + env = venv.get_env() + env["NOX_SCRIPT_MODE"] = "none" + cmd = ( + [nox.virtualenv.UV, "pip", "install"] + if venv.venv_backend == "uv" + else ["pip", "install"] + ) + subprocess.run([*cmd, *dependencies], env=env, check=True) + nox_cmd = shutil.which("nox", path=env["PATH"]) + assert nox_cmd is not None, "Nox must be discoverable when installed" + # The os.exec functions don't work properly on Windows + if sys.platform.startswith("win"): + raise SystemExit( + subprocess.run( + [nox_cmd, *sys.argv[1:]], + env=env, + stdout=None, + stderr=None, + encoding="utf-8", + text=True, + check=False, + ).returncode + ) + os.execle(nox_cmd, nox_cmd, *sys.argv[1:], env) # pragma: nocover # noqa: S606 + + def main() -> None: args = _options.options.parse_args() @@ -65,6 +160,21 @@ def main() -> None: setup_logging( color=args.color, verbose=args.verbose, add_timestamp=args.add_timestamp ) + nox_script_mode = os.environ.get("NOX_SCRIPT_MODE", "") or args.script_mode + if nox_script_mode not in {"none", "reuse", "fresh"}: + msg = f"Invalid NOX_SCRIPT_MODE: {nox_script_mode!r}, must be one of 'none', 'reuse', or 'fresh'" + raise SystemExit(msg) + if nox_script_mode != "none": + toml_config = load_toml(os.path.expandvars(args.noxfile), missing_ok=True) + dependencies = toml_config.get("dependencies") + if dependencies is not None: + valid_env = check_dependencies(dependencies) + # Coverage misses this, but it's covered via subprocess call + if not valid_env: # pragma: nocover + envdir = Path(args.envdir or ".nox") + run_script_mode( + envdir, reuse=nox_script_mode == "reuse", dependencies=dependencies + ) exit_code = execute_workflow(args) diff --git a/nox/_options.py b/nox/_options.py index 10c9515e..a9689dac 100644 --- a/nox/_options.py +++ b/nox/_options.py @@ -338,6 +338,13 @@ def _tag_completer( action="store_true", help="Show the Nox version and exit.", ), + _option_set.Option( + "script_mode", + "--script-mode", + group=options.groups["general"], + choices=["none", "fresh", "reuse"], + default="reuse", + ), _option_set.Option( "list_sessions", "-l", diff --git a/nox/project.py b/nox/project.py index d808c9a9..627d1e6a 100644 --- a/nox/project.py +++ b/nox/project.py @@ -34,7 +34,9 @@ def __dir__() -> list[str]: ) -def load_toml(filename: os.PathLike[str] | str) -> dict[str, Any]: +def load_toml( + filename: os.PathLike[str] | str, *, missing_ok: bool = False +) -> dict[str, Any]: """ Load a toml file or a script with a PEP 723 script block. @@ -42,6 +44,9 @@ def load_toml(filename: os.PathLike[str] | str) -> dict[str, Any]: ``.py`` extension / no extension to be considered a script. Other file extensions are not valid in this function. + If ``missing_ok``, this will return an empty dict if a script block was not + found, otherwise it will raise a error. + Example: .. code-block:: python @@ -55,7 +60,7 @@ def myscript(session): if filepath.suffix == ".toml": return _load_toml_file(filepath) if filepath.suffix in {".py", ""}: - return _load_script_block(filepath) + return _load_script_block(filepath, missing_ok=missing_ok) msg = f"Extension must be .py or .toml, got {filepath.suffix}" raise ValueError(msg) @@ -65,12 +70,14 @@ def _load_toml_file(filepath: Path) -> dict[str, Any]: return tomllib.load(f) -def _load_script_block(filepath: Path) -> dict[str, Any]: +def _load_script_block(filepath: Path, *, missing_ok: bool) -> dict[str, Any]: name = "script" script = filepath.read_text(encoding="utf-8") matches = list(filter(lambda m: m.group("type") == name, REGEX.finditer(script))) if not matches: + if missing_ok: + return {} msg = f"No {name} block found in {filepath}" raise ValueError(msg) if len(matches) > 1: diff --git a/nox/sessions.py b/nox/sessions.py index 07a78811..f1d3c6b6 100644 --- a/nox/sessions.py +++ b/nox/sessions.py @@ -618,14 +618,7 @@ def _run( args = (nox.virtualenv.UV, *args[1:]) # Combine the env argument with our virtualenv's env vars. - env = env or {} - env = {**self.env, **env} - if include_outer_env: - env = {**os.environ, **env} - if self.virtualenv.bin_paths: - env["PATH"] = os.pathsep.join( - [*self.virtualenv.bin_paths, env.get("PATH") or ""] - ) + env = self.virtualenv.get_env(env=env, include_outer_env=include_outer_env) # If --error-on-external-run is specified, error on external programs. if self._runner.global_config.error_on_external_run and external is None: diff --git a/nox/virtualenv.py b/nox/virtualenv.py index 2d32628e..96f74305 100644 --- a/nox/virtualenv.py +++ b/nox/virtualenv.py @@ -191,6 +191,27 @@ def venv_backend(self) -> str: Returns the string used to select this environment. """ + def get_env( + self, + *, + env: Mapping[str, str | None] | None = None, + include_outer_env: bool = True, + ) -> dict[str, str]: + """ + Get the computed environment, with bin paths added and exclusions + applied. You can request the outer environment be excluded, and/or pass + in an env to add. + """ + + computed_env = {**self.env, **(env or {})} + if include_outer_env: + computed_env = {**os.environ, **computed_env} + if self.bin_paths: + computed_env["PATH"] = os.pathsep.join( + [*self.bin_paths, computed_env.get("PATH") or ""] + ) + return {k: v for k, v in computed_env.items() if v is not None} + def locate_via_py(version: str) -> str | None: """Find the Python executable using the Windows Launcher. @@ -331,7 +352,7 @@ def _clean_location(self) -> bool: if self.reuse_existing and is_conda: return False if not is_conda: - shutil.rmtree(self.location) + shutil.rmtree(self.location, ignore_errors=True) else: cmd = [ self.conda_cmd, @@ -343,8 +364,7 @@ def _clean_location(self) -> bool: ] nox.command.run(cmd, silent=True, log=False) # Make sure that location is clean - with contextlib.suppress(FileNotFoundError): - shutil.rmtree(self.location) + shutil.rmtree(self.location, ignore_errors=True) return True @@ -471,7 +491,7 @@ def _clean_location(self) -> bool: and self._check_reused_environment_interpreter() ): return False - shutil.rmtree(self.location) + shutil.rmtree(self.location, ignore_errors=True) return True def _read_pyvenv_cfg(self) -> dict[str, str] | None: diff --git a/tests/resources/noxfile_script_mode.py b/tests/resources/noxfile_script_mode.py new file mode 100644 index 00000000..d95d0bb2 --- /dev/null +++ b/tests/resources/noxfile_script_mode.py @@ -0,0 +1,12 @@ +# /// script +# dependencies = ["nox", "cowsay"] +# /// + +import cowsay + +import nox + + +@nox.session +def example(session: nox.Session) -> None: + print(cowsay.cow("hello_world")) diff --git a/tests/test__cli.py b/tests/test__cli.py new file mode 100644 index 00000000..37a3e0de --- /dev/null +++ b/tests/test__cli.py @@ -0,0 +1,71 @@ +import importlib.metadata +import importlib.util +import sys + +import packaging.requirements +import packaging.version +import pytest + +import nox._cli + + +def test_get_dependencies() -> None: + if importlib.util.find_spec("tox") is None: + with pytest.raises(ModuleNotFoundError): + list( + nox._cli.get_dependencies( + packaging.requirements.Requirement("nox[tox_to_nox]") + ) + ) + else: + deps = nox._cli.get_dependencies( + packaging.requirements.Requirement("nox[tox_to_nox]") + ) + dep_list = { + "argcomplete", + "attrs", + "colorlog", + "dependency-groups", + "jinja2", + "nox", + "packaging", + "tox", + "virtualenv", + } + if sys.version_info < (3, 9): + dep_list.add("importlib-resources") + if sys.version_info < (3, 11): + dep_list.add("tomli") + assert {d.name for d in deps} == dep_list + + +def test_version_check() -> None: + current_version = packaging.version.Version(importlib.metadata.version("nox")) + + assert nox._cli.check_dependencies([f"nox>={current_version}"]) + assert not nox._cli.check_dependencies([f"nox>{current_version}"]) + + plus_one = packaging.version.Version( + f"{current_version.major}.{current_version.minor}.{current_version.micro + 1}" + ) + assert not nox._cli.check_dependencies([f"nox>={plus_one}"]) + + +def test_nox_check() -> None: + with pytest.raises(ValueError, match="Must have a nox"): + nox._cli.check_dependencies(["packaging"]) + + with pytest.raises(ValueError, match="Must have a nox"): + nox._cli.check_dependencies([]) + + +def test_unmatched_specifier() -> None: + assert not nox._cli.check_dependencies(["packaging<1", "nox"]) + + +def test_invalid_mode(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("NOX_SCRIPT_MODE", "invalid") + monkeypatch.setattr(sys, "argv", ["nox"]) + + with pytest.raises(SystemExit, match="Invalid NOX_SCRIPT_MODE"): + nox._cli.main() diff --git a/tests/test_main.py b/tests/test_main.py index f1bda816..0cfcb748 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -1020,3 +1020,46 @@ def test_symlink_sym_not(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.chdir(Path(RESOURCES) / "sym_dir") res = subprocess.run([sys.executable, "-m", "nox", "-s", "orig"], check=False) assert res.returncode == 1 + + +def test_noxfile_script_mode() -> None: + job = subprocess.run( + [ + sys.executable, + "-m", + "nox", + "-f", + Path(RESOURCES) / "noxfile_script_mode.py", + "-s", + "example", + ], + check=False, + capture_output=True, + text=True, + ) + print(job.stdout) + print(job.stderr) + assert job.returncode == 0 + assert "hello_world" in job.stdout + + +def test_noxfile_no_script_mode() -> None: + env = os.environ.copy() + env["NOX_SCRIPT_MODE"] = "none" + job = subprocess.run( + [ + sys.executable, + "-m", + "nox", + "-f", + Path(RESOURCES) / "noxfile_script_mode.py", + "-s", + "example", + ], + env=env, + check=False, + capture_output=True, + text=True, + ) + assert job.returncode == 1 + assert "No module named 'cowsay'" in job.stderr From 9908e0f55045ec452c5fd2b066e402f29334b1ab Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Fri, 17 Jan 2025 18:32:45 -0500 Subject: [PATCH 2/2] fix: keep Nones Signed-off-by: Henry Schreiner --- nox/_cli.py | 2 +- nox/virtualenv.py | 9 ++++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/nox/_cli.py b/nox/_cli.py index 67b03fdc..87818232 100644 --- a/nox/_cli.py +++ b/nox/_cli.py @@ -120,7 +120,7 @@ def run_script_mode(envdir: Path, *, reuse: bool, dependencies: list[str]) -> No envdir=str(noxenv), ) venv.create() - env = venv.get_env() + env = {k: v for k, v in venv.get_env().items() if v is not None} env["NOX_SCRIPT_MODE"] = "none" cmd = ( [nox.virtualenv.UV, "pip", "install"] diff --git a/nox/virtualenv.py b/nox/virtualenv.py index 96f74305..605a99d9 100644 --- a/nox/virtualenv.py +++ b/nox/virtualenv.py @@ -196,11 +196,10 @@ def get_env( *, env: Mapping[str, str | None] | None = None, include_outer_env: bool = True, - ) -> dict[str, str]: + ) -> dict[str, str | None]: """ - Get the computed environment, with bin paths added and exclusions - applied. You can request the outer environment be excluded, and/or pass - in an env to add. + Get the computed environment, with bin paths added. You can request + the outer environment be excluded, and/or pass in an env to add. """ computed_env = {**self.env, **(env or {})} @@ -210,7 +209,7 @@ def get_env( computed_env["PATH"] = os.pathsep.join( [*self.bin_paths, computed_env.get("PATH") or ""] ) - return {k: v for k, v in computed_env.items() if v is not None} + return computed_env def locate_via_py(version: str) -> str | None: