From 505b59a519f8dc3c2d1b7d8f11b23bf13f5dc6eb Mon Sep 17 00:00:00 2001 From: Stephen Rosen Date: Thu, 31 Oct 2024 19:43:49 -0500 Subject: [PATCH 1/9] Add 'dependency-groups==1.3.0' to vendored libs Steps taken: - add `dependency-groups==1.3.0` to vendor.txt - add dependency-groups to vendor __init__.py - run vendoring sync - examine results to confirm apparent correctness (rewritten tomli imports) --- src/pip/_vendor/__init__.py | 1 + src/pip/_vendor/dependency_groups/LICENSE.txt | 9 + src/pip/_vendor/dependency_groups/__init__.py | 13 ++ src/pip/_vendor/dependency_groups/__main__.py | 65 ++++++ .../dependency_groups/_implementation.py | 213 ++++++++++++++++++ .../_lint_dependency_groups.py | 59 +++++ .../_vendor/dependency_groups/_pip_wrapper.py | 62 +++++ .../_vendor/dependency_groups/_toml_compat.py | 9 + src/pip/_vendor/dependency_groups/py.typed | 0 src/pip/_vendor/vendor.txt | 1 + 10 files changed, 432 insertions(+) create mode 100644 src/pip/_vendor/dependency_groups/LICENSE.txt create mode 100644 src/pip/_vendor/dependency_groups/__init__.py create mode 100644 src/pip/_vendor/dependency_groups/__main__.py create mode 100644 src/pip/_vendor/dependency_groups/_implementation.py create mode 100644 src/pip/_vendor/dependency_groups/_lint_dependency_groups.py create mode 100644 src/pip/_vendor/dependency_groups/_pip_wrapper.py create mode 100644 src/pip/_vendor/dependency_groups/_toml_compat.py create mode 100644 src/pip/_vendor/dependency_groups/py.typed diff --git a/src/pip/_vendor/__init__.py b/src/pip/_vendor/__init__.py index 561089ccc0c..34ccb990791 100644 --- a/src/pip/_vendor/__init__.py +++ b/src/pip/_vendor/__init__.py @@ -60,6 +60,7 @@ def vendored(modulename): # Actually alias all of our vendored dependencies. vendored("cachecontrol") vendored("certifi") + vendored("dependency-groups") vendored("distlib") vendored("distro") vendored("packaging") diff --git a/src/pip/_vendor/dependency_groups/LICENSE.txt b/src/pip/_vendor/dependency_groups/LICENSE.txt new file mode 100644 index 00000000000..b9723b85ed7 --- /dev/null +++ b/src/pip/_vendor/dependency_groups/LICENSE.txt @@ -0,0 +1,9 @@ +MIT License + +Copyright (c) 2024-present Stephen Rosen + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/src/pip/_vendor/dependency_groups/__init__.py b/src/pip/_vendor/dependency_groups/__init__.py new file mode 100644 index 00000000000..9fec2029949 --- /dev/null +++ b/src/pip/_vendor/dependency_groups/__init__.py @@ -0,0 +1,13 @@ +from ._implementation import ( + CyclicDependencyError, + DependencyGroupInclude, + DependencyGroupResolver, + resolve, +) + +__all__ = ( + "CyclicDependencyError", + "DependencyGroupInclude", + "DependencyGroupResolver", + "resolve", +) diff --git a/src/pip/_vendor/dependency_groups/__main__.py b/src/pip/_vendor/dependency_groups/__main__.py new file mode 100644 index 00000000000..48ebb0d41cf --- /dev/null +++ b/src/pip/_vendor/dependency_groups/__main__.py @@ -0,0 +1,65 @@ +import argparse +import sys + +from ._implementation import resolve +from ._toml_compat import tomllib + + +def main() -> None: + if tomllib is None: + print( + "Usage error: dependency-groups CLI requires tomli or Python 3.11+", + file=sys.stderr, + ) + raise SystemExit(2) + + parser = argparse.ArgumentParser( + description=( + "A dependency-groups CLI. Prints out a resolved group, newline-delimited." + ) + ) + parser.add_argument( + "GROUP_NAME", nargs="*", help="The dependency group(s) to resolve." + ) + parser.add_argument( + "-f", + "--pyproject-file", + default="pyproject.toml", + help="The pyproject.toml file. Defaults to trying in the current directory.", + ) + parser.add_argument( + "-o", + "--output", + help="An output file. Defaults to stdout.", + ) + parser.add_argument( + "-l", + "--list", + action="store_true", + help="List the available dependency groups", + ) + args = parser.parse_args() + + with open(args.pyproject_file, "rb") as fp: + pyproject = tomllib.load(fp) + + dependency_groups_raw = pyproject.get("dependency-groups", {}) + + if args.list: + print(*dependency_groups_raw.keys()) + return + if not args.GROUP_NAME: + print("A GROUP_NAME is required", file=sys.stderr) + raise SystemExit(3) + + content = "\n".join(resolve(dependency_groups_raw, *args.GROUP_NAME)) + + if args.output is None or args.output == "-": + print(content) + else: + with open(args.output, "w", encoding="utf-8") as fp: + print(content, file=fp) + + +if __name__ == "__main__": + main() diff --git a/src/pip/_vendor/dependency_groups/_implementation.py b/src/pip/_vendor/dependency_groups/_implementation.py new file mode 100644 index 00000000000..80d91693820 --- /dev/null +++ b/src/pip/_vendor/dependency_groups/_implementation.py @@ -0,0 +1,213 @@ +from __future__ import annotations + +import dataclasses +import re +from collections.abc import Mapping + +from pip._vendor.packaging.requirements import Requirement + + +def _normalize_name(name: str) -> str: + return re.sub(r"[-_.]+", "-", name).lower() + + +def _normalize_group_names( + dependency_groups: Mapping[str, str | Mapping[str, str]] +) -> Mapping[str, str | Mapping[str, str]]: + original_names: dict[str, list[str]] = {} + normalized_groups = {} + + for group_name, value in dependency_groups.items(): + normed_group_name = _normalize_name(group_name) + original_names.setdefault(normed_group_name, []).append(group_name) + normalized_groups[normed_group_name] = value + + errors = [] + for normed_name, names in original_names.items(): + if len(names) > 1: + errors.append(f"{normed_name} ({', '.join(names)})") + if errors: + raise ValueError(f"Duplicate dependency group names: {', '.join(errors)}") + + return normalized_groups + + +@dataclasses.dataclass +class DependencyGroupInclude: + include_group: str + + +class CyclicDependencyError(ValueError): + """ + An error representing the detection of a cycle. + """ + + def __init__(self, requested_group: str, group: str, include_group: str) -> None: + self.requested_group = requested_group + self.group = group + self.include_group = include_group + + if include_group == group: + reason = f"{group} includes itself" + else: + reason = f"{include_group} -> {group}, {group} -> {include_group}" + super().__init__( + "Cyclic dependency group include while resolving " + f"{requested_group}: {reason}" + ) + + +class DependencyGroupResolver: + """ + A resolver for Dependency Group data. + + This class handles caching, name normalization, cycle detection, and other + parsing requirements. There are only two public methods for exploring the data: + ``lookup()`` and ``resolve()``. + + :param dependency_groups: A mapping, as provided via pyproject + ``[dependency-groups]``. + """ + + def __init__( + self, + dependency_groups: Mapping[str, str | Mapping[str, str]], + ) -> None: + if not isinstance(dependency_groups, Mapping): + raise TypeError("Dependency Groups table is not a mapping") + self.dependency_groups = _normalize_group_names(dependency_groups) + # a map of group names to parsed data + self._parsed_groups: dict[ + str, tuple[Requirement | DependencyGroupInclude, ...] + ] = {} + # a map of group names to their ancestors, used for cycle detection + self._include_graph_ancestors: dict[str, tuple[str, ...]] = {} + # a cache of completed resolutions to Requirement lists + self._resolve_cache: dict[str, tuple[Requirement, ...]] = {} + + def lookup(self, group: str) -> tuple[Requirement | DependencyGroupInclude, ...]: + """ + Lookup a group name, returning the parsed dependency data for that group. + This will not resolve includes. + + :param group: the name of the group to lookup + + :raises ValueError: if the data does not appear to be valid dependency group + data + :raises TypeError: if the data is not a string + :raises LookupError: if group name is absent + :raises packaging.requirements.InvalidRequirement: if a specifier is not valid + """ + if not isinstance(group, str): + raise TypeError("Dependency group name is not a str") + group = _normalize_name(group) + return self._parse_group(group) + + def resolve(self, group: str) -> tuple[Requirement, ...]: + """ + Resolve a dependency group to a list of requirements. + + :param group: the name of the group to resolve + + :raises TypeError: if the inputs appear to be the wrong types + :raises ValueError: if the data does not appear to be valid dependency group + data + :raises LookupError: if group name is absent + :raises packaging.requirements.InvalidRequirement: if a specifier is not valid + """ + if not isinstance(group, str): + raise TypeError("Dependency group name is not a str") + group = _normalize_name(group) + return self._resolve(group, group) + + def _parse_group( + self, group: str + ) -> tuple[Requirement | DependencyGroupInclude, ...]: + # short circuit -- never do the work twice + if group in self._parsed_groups: + return self._parsed_groups[group] + + if group not in self.dependency_groups: + raise LookupError(f"Dependency group '{group}' not found") + + raw_group = self.dependency_groups[group] + if not isinstance(raw_group, list): + raise TypeError(f"Dependency group '{group}' is not a list") + + elements: list[Requirement | DependencyGroupInclude] = [] + for item in raw_group: + if isinstance(item, str): + # packaging.requirements.Requirement parsing ensures that this is a + # valid PEP 508 Dependency Specifier + # raises InvalidRequirement on failure + elements.append(Requirement(item)) + elif isinstance(item, dict): + if tuple(item.keys()) != ("include-group",): + raise ValueError(f"Invalid dependency group item: {item}") + + include_group = next(iter(item.values())) + elements.append(DependencyGroupInclude(include_group=include_group)) + else: + raise ValueError(f"Invalid dependency group item: {item}") + + self._parsed_groups[group] = tuple(elements) + return self._parsed_groups[group] + + def _resolve(self, group: str, requested_group: str) -> tuple[Requirement, ...]: + """ + This is a helper for cached resolution to strings. + + :param group: The name of the group to resolve. + :param requested_group: The group which was used in the original, user-facing + request. + """ + if group in self._resolve_cache: + return self._resolve_cache[group] + + parsed = self._parse_group(group) + + resolved_group = [] + for item in parsed: + if isinstance(item, Requirement): + resolved_group.append(item) + elif isinstance(item, DependencyGroupInclude): + if item.include_group in self._include_graph_ancestors.get(group, ()): + raise CyclicDependencyError( + requested_group, group, item.include_group + ) + self._include_graph_ancestors[item.include_group] = ( + *self._include_graph_ancestors.get(group, ()), + group, + ) + resolved_group.extend( + self._resolve(item.include_group, requested_group) + ) + else: # unreachable + raise NotImplementedError( + f"Invalid dependency group item after parse: {item}" + ) + + self._resolve_cache[group] = tuple(resolved_group) + return self._resolve_cache[group] + + +def resolve( + dependency_groups: Mapping[str, str | Mapping[str, str]], /, *groups: str +) -> tuple[str, ...]: + """ + Resolve a dependency group to a tuple of requirements, as strings. + + :param dependency_groups: the parsed contents of the ``[dependency-groups]`` table + from ``pyproject.toml`` + :param groups: the name of the group(s) to resolve + + :raises TypeError: if the inputs appear to be the wrong types + :raises ValueError: if the data does not appear to be valid dependency group data + :raises LookupError: if group name is absent + :raises packaging.requirements.InvalidRequirement: if a specifier is not valid + """ + return tuple( + str(r) + for group in groups + for r in DependencyGroupResolver(dependency_groups).resolve(group) + ) diff --git a/src/pip/_vendor/dependency_groups/_lint_dependency_groups.py b/src/pip/_vendor/dependency_groups/_lint_dependency_groups.py new file mode 100644 index 00000000000..09454bdc280 --- /dev/null +++ b/src/pip/_vendor/dependency_groups/_lint_dependency_groups.py @@ -0,0 +1,59 @@ +from __future__ import annotations + +import argparse +import sys + +from ._implementation import DependencyGroupResolver +from ._toml_compat import tomllib + + +def main(*, argv: list[str] | None = None) -> None: + if tomllib is None: + print( + "Usage error: dependency-groups CLI requires tomli or Python 3.11+", + file=sys.stderr, + ) + raise SystemExit(2) + + parser = argparse.ArgumentParser( + description=( + "Lint Dependency Groups for validity. " + "This will eagerly load and check all of your Dependency Groups." + ) + ) + parser.add_argument( + "-f", + "--pyproject-file", + default="pyproject.toml", + help="The pyproject.toml file. Defaults to trying in the current directory.", + ) + args = parser.parse_args(argv if argv is not None else sys.argv[1:]) + + with open(args.pyproject_file, "rb") as fp: + pyproject = tomllib.load(fp) + dependency_groups_raw = pyproject.get("dependency-groups", {}) + + errors: list[str] = [] + try: + resolver = DependencyGroupResolver(dependency_groups_raw) + except (ValueError, TypeError) as e: + errors.append(f"{type(e).__name__}: {e}") + else: + for groupname in resolver.dependency_groups: + try: + resolver.resolve(groupname) + except (LookupError, ValueError, TypeError) as e: + errors.append(f"{type(e).__name__}: {e}") + + if errors: + print("errors encountered while examining dependency groups:") + for msg in errors: + print(f" {msg}") + sys.exit(1) + else: + print("ok") + sys.exit(0) + + +if __name__ == "__main__": + main() diff --git a/src/pip/_vendor/dependency_groups/_pip_wrapper.py b/src/pip/_vendor/dependency_groups/_pip_wrapper.py new file mode 100644 index 00000000000..f86d8961ba2 --- /dev/null +++ b/src/pip/_vendor/dependency_groups/_pip_wrapper.py @@ -0,0 +1,62 @@ +from __future__ import annotations + +import argparse +import subprocess +import sys + +from ._implementation import DependencyGroupResolver +from ._toml_compat import tomllib + + +def _invoke_pip(deps: list[str]) -> None: + subprocess.check_call([sys.executable, "-m", "pip", "install", *deps]) + + +def main(*, argv: list[str] | None = None) -> None: + if tomllib is None: + print( + "Usage error: dependency-groups CLI requires tomli or Python 3.11+", + file=sys.stderr, + ) + raise SystemExit(2) + + parser = argparse.ArgumentParser(description="Install Dependency Groups.") + parser.add_argument( + "DEPENDENCY_GROUP", nargs="+", help="The dependency groups to install." + ) + parser.add_argument( + "-f", + "--pyproject-file", + default="pyproject.toml", + help="The pyproject.toml file. Defaults to trying in the current directory.", + ) + args = parser.parse_args(argv if argv is not None else sys.argv[1:]) + + with open(args.pyproject_file, "rb") as fp: + pyproject = tomllib.load(fp) + dependency_groups_raw = pyproject.get("dependency-groups", {}) + + errors: list[str] = [] + resolved: list[str] = [] + try: + resolver = DependencyGroupResolver(dependency_groups_raw) + except (ValueError, TypeError) as e: + errors.append(f"{type(e).__name__}: {e}") + else: + for groupname in args.DEPENDENCY_GROUP: + try: + resolved.extend(str(r) for r in resolver.resolve(groupname)) + except (LookupError, ValueError, TypeError) as e: + errors.append(f"{type(e).__name__}: {e}") + + if errors: + print("errors encountered while examining dependency groups:") + for msg in errors: + print(f" {msg}") + sys.exit(1) + + _invoke_pip(resolved) + + +if __name__ == "__main__": + main() diff --git a/src/pip/_vendor/dependency_groups/_toml_compat.py b/src/pip/_vendor/dependency_groups/_toml_compat.py new file mode 100644 index 00000000000..8d6f921c2a5 --- /dev/null +++ b/src/pip/_vendor/dependency_groups/_toml_compat.py @@ -0,0 +1,9 @@ +try: + import tomllib +except ImportError: + try: + from pip._vendor import tomli as tomllib # type: ignore[no-redef, unused-ignore] + except ModuleNotFoundError: # pragma: no cover + tomllib = None # type: ignore[assignment, unused-ignore] + +__all__ = ("tomllib",) diff --git a/src/pip/_vendor/dependency_groups/py.typed b/src/pip/_vendor/dependency_groups/py.typed new file mode 100644 index 00000000000..e69de29bb2d diff --git a/src/pip/_vendor/vendor.txt b/src/pip/_vendor/vendor.txt index 2ba053a6e54..27396ca6dbb 100644 --- a/src/pip/_vendor/vendor.txt +++ b/src/pip/_vendor/vendor.txt @@ -16,3 +16,4 @@ resolvelib==1.0.1 setuptools==70.3.0 tomli==2.0.1 truststore==0.10.0 +dependency-groups==1.3.0 From 38aec3aa8ec8a92a898adf358e5d0091d8dd6b15 Mon Sep 17 00:00:00 2001 From: Stephen Rosen Date: Thu, 31 Oct 2024 20:07:06 -0500 Subject: [PATCH 2/9] Implement Dependency Group option: `--group` `--group` is supported on `download` and `install` commands. The option is parsed into the more verbose and explicit `dependency_groups` name on the parsed args. Both of these commands invoke the same processor for resolving dependency groups, which loads `pyproject.toml` and resolves the list of provided groups against the `[dependency-groups]` table. A small alteration is made to `pip wheel` to initialize `dependency_groups = []`, as this allows for some lower-level consistency in the handling of the commands. --- src/pip/_internal/cli/cmdoptions.py | 11 ++++ src/pip/_internal/cli/req_command.py | 20 ++++++- src/pip/_internal/commands/download.py | 1 + src/pip/_internal/commands/install.py | 1 + src/pip/_internal/commands/wheel.py | 4 ++ src/pip/_internal/req/req_dependency_group.py | 59 +++++++++++++++++++ 6 files changed, 95 insertions(+), 1 deletion(-) create mode 100644 src/pip/_internal/req/req_dependency_group.py diff --git a/src/pip/_internal/cli/cmdoptions.py b/src/pip/_internal/cli/cmdoptions.py index 0b7cff77bdd..c36a63d409a 100644 --- a/src/pip/_internal/cli/cmdoptions.py +++ b/src/pip/_internal/cli/cmdoptions.py @@ -733,6 +733,17 @@ def _handle_no_cache_dir( help="Don't install package dependencies.", ) +dependency_groups: Callable[..., Option] = partial( + Option, + "--group", + dest="dependency_groups", + default=[], + action="append", + metavar="group", + help="Install a named dependency-group from `pyproject.toml` " + "in the current directory.", +) + ignore_requires_python: Callable[..., Option] = partial( Option, "--ignore-requires-python", diff --git a/src/pip/_internal/cli/req_command.py b/src/pip/_internal/cli/req_command.py index 92900f94ff4..5354194d9e8 100644 --- a/src/pip/_internal/cli/req_command.py +++ b/src/pip/_internal/cli/req_command.py @@ -28,6 +28,7 @@ install_req_from_parsed_requirement, install_req_from_req_string, ) +from pip._internal.req.req_dependency_group import parse_dependency_groups from pip._internal.req.req_file import parse_requirements from pip._internal.req.req_install import InstallRequirement from pip._internal.resolution.base import BaseResolver @@ -240,6 +241,18 @@ def get_requirements( ) requirements.append(req_to_add) + if options.dependency_groups: + for req in parse_dependency_groups( + options.dependency_groups, session, finder=finder, options=options + ): + req_to_add = install_req_from_req_string( + req, + isolated=options.isolated_mode, + use_pep517=options.use_pep517, + user_supplied=True, + ) + requirements.append(req_to_add) + for req in options.editables: req_to_add = install_req_from_editable( req, @@ -272,7 +285,12 @@ def get_requirements( if any(req.has_hash_options for req in requirements): options.require_hashes = True - if not (args or options.editables or options.requirements): + if not ( + args + or options.editables + or options.requirements + or options.dependency_groups + ): opts = {"name": self.name} if options.find_links: raise CommandError( diff --git a/src/pip/_internal/commands/download.py b/src/pip/_internal/commands/download.py index 917bbb91d83..51277b0d6a5 100644 --- a/src/pip/_internal/commands/download.py +++ b/src/pip/_internal/commands/download.py @@ -38,6 +38,7 @@ class DownloadCommand(RequirementCommand): def add_options(self) -> None: self.cmd_opts.add_option(cmdoptions.constraints()) self.cmd_opts.add_option(cmdoptions.requirements()) + self.cmd_opts.add_option(cmdoptions.dependency_groups()) self.cmd_opts.add_option(cmdoptions.no_deps()) self.cmd_opts.add_option(cmdoptions.global_options()) self.cmd_opts.add_option(cmdoptions.no_binary()) diff --git a/src/pip/_internal/commands/install.py b/src/pip/_internal/commands/install.py index ad45a2f2a57..ba6d6da3000 100644 --- a/src/pip/_internal/commands/install.py +++ b/src/pip/_internal/commands/install.py @@ -77,6 +77,7 @@ def add_options(self) -> None: self.cmd_opts.add_option(cmdoptions.pre()) self.cmd_opts.add_option(cmdoptions.editable()) + self.cmd_opts.add_option(cmdoptions.dependency_groups()) self.cmd_opts.add_option( "--dry-run", action="store_true", diff --git a/src/pip/_internal/commands/wheel.py b/src/pip/_internal/commands/wheel.py index 278719f4e0c..f45e1692207 100644 --- a/src/pip/_internal/commands/wheel.py +++ b/src/pip/_internal/commands/wheel.py @@ -102,6 +102,10 @@ def add_options(self) -> None: @with_cleanup def run(self, options: Values, args: List[str]) -> int: + # dependency-groups aren't desirable with `pip wheel`, but providing it + # consistently allows RequirementCommand to expect it to be present + options.dependency_groups = [] + session = self.get_default_session(options) finder = self._build_package_finder(options, session) diff --git a/src/pip/_internal/req/req_dependency_group.py b/src/pip/_internal/req/req_dependency_group.py new file mode 100644 index 00000000000..1750d4dd06c --- /dev/null +++ b/src/pip/_internal/req/req_dependency_group.py @@ -0,0 +1,59 @@ +import optparse +from typing import TYPE_CHECKING, Any, Dict, List, Optional + +from pip._vendor import tomli +from pip._vendor.dependency_groups import resolve as resolve_dependency_group + +from pip._internal.exceptions import InstallationError +from pip._internal.network.session import PipSession + +if TYPE_CHECKING: + from pip._internal.index.package_finder import PackageFinder + + +def parse_dependency_groups( + groups: List[str], + session: PipSession, + finder: Optional["PackageFinder"] = None, + options: Optional[optparse.Values] = None, +) -> List[str]: + """ + Parse dependency groups data in a way which is sensitive to the `pip` context and + raises InstallationErrors if anything goes wrong. + """ + pyproject = _load_pyproject() + + if "dependency-groups" not in pyproject: + raise InstallationError( + "[dependency-groups] table was missing. Cannot resolve '--group' options." + ) + raw_dependency_groups = pyproject["dependency-groups"] + if not isinstance(raw_dependency_groups, dict): + raise InstallationError( + "[dependency-groups] table was malformed. Cannot resolve '--group' options." + ) + + try: + return list(resolve_dependency_group(raw_dependency_groups, *groups)) + except (ValueError, TypeError, LookupError) as e: + raise InstallationError("[dependency-groups] resolution failed: {e}") from e + + +def _load_pyproject() -> Dict[str, Any]: + """ + This helper loads pyproject.toml from the current working directory. + + It does not allow specification of the path to be used and raises an + InstallationError if the operation fails. + """ + try: + with open("pyproject.toml", "rb") as fp: + return tomli.load(fp) + except FileNotFoundError: + raise InstallationError( + "pyproject.toml not found. Cannot resolve '--group' options." + ) + except tomli.TOMLDecodeError as e: + raise InstallationError(f"Error parsing pyproject.toml: {e}") from e + except OSError as e: + raise InstallationError(f"Error reading pyproject.toml: {e}") from e From 164e731bd2f222997544678f32a8c4d67a99d0f8 Mon Sep 17 00:00:00 2001 From: Stephen Rosen Date: Mon, 4 Nov 2024 18:21:30 -0600 Subject: [PATCH 3/9] Add unit tests for dependency group loading A new unit test module is added for parsing dependency groups and used to verify all of the pip-defined behaviors for handling dependency-groups. In one path, the underlying exception message from `dependency-groups` is exposed to users, where it should offer some explanation of why parsing failed, and this is therefore tested. Some related changes are applied to the dependency groups usage sites in the src tree. The signature of the dependency group requirement parse function is simplified, and its usage is therefore updated. A bugfix is applied to add a missing `f` on an intended f-string. --- src/pip/_internal/cli/req_command.py | 4 +- src/pip/_internal/req/req_dependency_group.py | 16 +-- tests/unit/test_req_dependency_group.py | 116 ++++++++++++++++++ 3 files changed, 120 insertions(+), 16 deletions(-) create mode 100644 tests/unit/test_req_dependency_group.py diff --git a/src/pip/_internal/cli/req_command.py b/src/pip/_internal/cli/req_command.py index 5354194d9e8..26bb8159fa8 100644 --- a/src/pip/_internal/cli/req_command.py +++ b/src/pip/_internal/cli/req_command.py @@ -242,9 +242,7 @@ def get_requirements( requirements.append(req_to_add) if options.dependency_groups: - for req in parse_dependency_groups( - options.dependency_groups, session, finder=finder, options=options - ): + for req in parse_dependency_groups(options.dependency_groups): req_to_add = install_req_from_req_string( req, isolated=options.isolated_mode, diff --git a/src/pip/_internal/req/req_dependency_group.py b/src/pip/_internal/req/req_dependency_group.py index 1750d4dd06c..aa9839ead59 100644 --- a/src/pip/_internal/req/req_dependency_group.py +++ b/src/pip/_internal/req/req_dependency_group.py @@ -1,22 +1,12 @@ -import optparse -from typing import TYPE_CHECKING, Any, Dict, List, Optional +from typing import Any, Dict, List from pip._vendor import tomli from pip._vendor.dependency_groups import resolve as resolve_dependency_group from pip._internal.exceptions import InstallationError -from pip._internal.network.session import PipSession -if TYPE_CHECKING: - from pip._internal.index.package_finder import PackageFinder - -def parse_dependency_groups( - groups: List[str], - session: PipSession, - finder: Optional["PackageFinder"] = None, - options: Optional[optparse.Values] = None, -) -> List[str]: +def parse_dependency_groups(groups: List[str]) -> List[str]: """ Parse dependency groups data in a way which is sensitive to the `pip` context and raises InstallationErrors if anything goes wrong. @@ -36,7 +26,7 @@ def parse_dependency_groups( try: return list(resolve_dependency_group(raw_dependency_groups, *groups)) except (ValueError, TypeError, LookupError) as e: - raise InstallationError("[dependency-groups] resolution failed: {e}") from e + raise InstallationError(f"[dependency-groups] resolution failed: {e}") from e def _load_pyproject() -> Dict[str, Any]: diff --git a/tests/unit/test_req_dependency_group.py b/tests/unit/test_req_dependency_group.py new file mode 100644 index 00000000000..dd78fd031ac --- /dev/null +++ b/tests/unit/test_req_dependency_group.py @@ -0,0 +1,116 @@ +import errno +from pathlib import Path +from typing import Any + +import pytest + +from pip._internal.exceptions import InstallationError +from pip._internal.req.req_dependency_group import parse_dependency_groups + + +def test_parse_simple_dependency_groups( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + pyproject = tmp_path.joinpath("pyproject.toml") + pyproject.write_text( + """\ +[dependency-groups] +foo = ["bar"] +""" + ) + monkeypatch.chdir(tmp_path) + + result = list(parse_dependency_groups(["foo"])) + + assert len(result) == 1, result + assert result[0] == "bar" + + +def test_parse_cyclic_dependency_groups( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + pyproject = tmp_path.joinpath("pyproject.toml") + pyproject.write_text( + """\ +[dependency-groups] +foo = [{include-group="bar"}] +bar = [{include-group="foo"}] +""" + ) + monkeypatch.chdir(tmp_path) + + with pytest.raises( + InstallationError, match=r"\[dependency-groups\] resolution failed:" + ) as excinfo: + parse_dependency_groups(["foo"]) + + exception = excinfo.value + assert ( + "Cyclic dependency group include while resolving foo: foo -> bar, bar -> foo" + ) in str(exception) + + +def test_parse_with_no_dependency_groups_defined( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + pyproject = tmp_path.joinpath("pyproject.toml") + pyproject.write_text( + """\ +""" + ) + monkeypatch.chdir(tmp_path) + + with pytest.raises( + InstallationError, match=r"\[dependency-groups\] table was missing\." + ): + parse_dependency_groups(["foo"]) + + +def test_parse_with_no_pyproject_file( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.chdir(tmp_path) + + with pytest.raises(InstallationError, match=r"pyproject\.toml not found\."): + parse_dependency_groups(["foo"]) + + +def test_parse_with_malformed_pyproject_file( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + pyproject = tmp_path.joinpath("pyproject.toml") + pyproject.write_text( + """\ +[dependency-groups # no closing bracket +foo = ["bar"] +""" + ) + monkeypatch.chdir(tmp_path) + + with pytest.raises(InstallationError, match=r"Error parsing pyproject\.toml"): + parse_dependency_groups(["foo"]) + + +def test_parse_gets_unexpected_oserror( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + pyproject = tmp_path.joinpath("pyproject.toml") + pyproject.write_text( + """\ +[dependency-groups] +foo = ["bar"] +""" + ) + monkeypatch.chdir(tmp_path) + + # inject an implementation of `tomli.load()` which emits an 'OSError(EPIPE, ...)' + # as though we were loading from a fifo or other unusual filetype + def epipe_toml_load(*args: Any, **kwargs: Any) -> None: + raise OSError(errno.EPIPE, "Broken pipe") + + monkeypatch.setattr( + "pip._internal.req.req_dependency_group.tomli.load", epipe_toml_load + ) + + with pytest.raises(InstallationError, match=r"Error reading pyproject\.toml"): + parse_dependency_groups(["foo"]) From 1d80247096388b2ae1aa62daf61f6bc95a7493c0 Mon Sep 17 00:00:00 2001 From: Stephen Rosen Date: Mon, 4 Nov 2024 18:47:19 -0600 Subject: [PATCH 4/9] Add initial functional tests for dependency-groups This initial suite of tests is modeled fairly closely on existing tests for requirements files. Tests cover the following cases: - installing an empty dependency group (and nothing else) - installing from a simple / straightforward group - installing from multiple groups in a single command - normalizing names from the CLI and pyproject.toml to match - applying a constraints file to a dependency-group install --- tests/functional/test_install.py | 15 +++++ tests/functional/test_install_reqs.py | 95 +++++++++++++++++++++++++++ 2 files changed, 110 insertions(+) diff --git a/tests/functional/test_install.py b/tests/functional/test_install.py index 4718beb948c..40ef585807b 100644 --- a/tests/functional/test_install.py +++ b/tests/functional/test_install.py @@ -318,6 +318,21 @@ def test_install_exit_status_code_when_blank_requirements_file( script.pip("install", "-r", "blank.txt") +def test_install_exit_status_code_when_empty_dependency_group( + script: PipTestEnvironment, +) -> None: + """ + Test install exit status code is 0 when empty dependency group specified + """ + script.scratch_path.joinpath("pyproject.toml").write_text( + """\ +[dependency-groups] +empty = [] +""" + ) + script.pip("install", "--group", "empty") + + @pytest.mark.network def test_basic_install_from_pypi(script: PipTestEnvironment) -> None: """ diff --git a/tests/functional/test_install_reqs.py b/tests/functional/test_install_reqs.py index b1aed6ad3f4..2d5ef8b0304 100644 --- a/tests/functional/test_install_reqs.py +++ b/tests/functional/test_install_reqs.py @@ -93,6 +93,75 @@ def test_requirements_file(script: PipTestEnvironment) -> None: assert result.files_created[script.site_packages / fn].dir +@pytest.mark.network +def test_dependency_group(script: PipTestEnvironment) -> None: + """ + Test installing from a dependency group. + + """ + pyproject = script.scratch_path / "pyproject.toml" + pyproject.write_text( + textwrap.dedent( + """\ + [dependency-groups] + initools = [ + "INITools==0.2", + "peppercorn<=0.6", + ] + """ + ) + ) + result = script.pip("install", "--group", "initools") + result.did_create(script.site_packages / "INITools-0.2.dist-info") + result.did_create(script.site_packages / "initools") + assert result.files_created[script.site_packages / "peppercorn"].dir + assert result.files_created[script.site_packages / "peppercorn-0.6.dist-info"].dir + + +@pytest.mark.network +def test_multiple_dependency_groups(script: PipTestEnvironment) -> None: + """ + Test installing from two dependency groups simultaneously. + + """ + pyproject = script.scratch_path / "pyproject.toml" + pyproject.write_text( + textwrap.dedent( + """\ + [dependency-groups] + initools = ["INITools==0.2"] + peppercorn = ["peppercorn<=0.6"] + """ + ) + ) + result = script.pip("install", "--group", "initools", "--group", "peppercorn") + result.did_create(script.site_packages / "INITools-0.2.dist-info") + result.did_create(script.site_packages / "initools") + assert result.files_created[script.site_packages / "peppercorn"].dir + assert result.files_created[script.site_packages / "peppercorn-0.6.dist-info"].dir + + +@pytest.mark.network +def test_dependency_group_with_non_normalized_name(script: PipTestEnvironment) -> None: + """ + Test installing from a dependency group with a non-normalized name, verifying that + the pyproject.toml content and CLI arg are normalized to match. + + """ + pyproject = script.scratch_path / "pyproject.toml" + pyproject.write_text( + textwrap.dedent( + """\ + [dependency-groups] + INITOOLS = ["INITools==0.2"] + """ + ) + ) + result = script.pip("install", "--group", "IniTools") + result.did_create(script.site_packages / "INITools-0.2.dist-info") + result.did_create(script.site_packages / "initools") + + def test_schema_check_in_requirements_file(script: PipTestEnvironment) -> None: """ Test installing from a requirements file with an invalid vcs schema.. @@ -212,6 +281,32 @@ def test_package_in_constraints_and_dependencies( assert "installed TopoRequires-0.0.1" in result.stdout +def test_constraints_apply_to_dependency_groups( + script: PipTestEnvironment, data: TestData +) -> None: + script.scratch_path.joinpath("constraints.txt").write_text("TopoRequires==0.0.1") + pyproject = script.scratch_path / "pyproject.toml" + pyproject.write_text( + textwrap.dedent( + """\ + [dependency-groups] + mylibs = ["TopoRequires2"] + """ + ) + ) + result = script.pip( + "install", + "--no-index", + "-f", + data.find_links, + "-c", + script.scratch_path / "constraints.txt", + "--group", + "mylibs", + ) + assert "installed TopoRequires-0.0.1" in result.stdout + + def test_multiple_constraints_files(script: PipTestEnvironment, data: TestData) -> None: script.scratch_path.joinpath("outer.txt").write_text("-c inner.txt") script.scratch_path.joinpath("inner.txt").write_text("Upper==1.0") From 282f69ec08f427b715b0ba246e2fd00a026b2e83 Mon Sep 17 00:00:00 2001 From: Stephen Rosen Date: Mon, 4 Nov 2024 18:54:25 -0600 Subject: [PATCH 5/9] Add a news fragment for `--group` support --- news/12963.feature.rst | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 news/12963.feature.rst diff --git a/news/12963.feature.rst b/news/12963.feature.rst new file mode 100644 index 00000000000..232dc3522e8 --- /dev/null +++ b/news/12963.feature.rst @@ -0,0 +1,3 @@ +- Add a ``--group`` option which allows installation from PEP 735 Dependency + Groups. Only ``pyproject.toml`` files in the current working directory are + supported. From 6238f2c610f40e8f3c63c25eab70a8a22220927a Mon Sep 17 00:00:00 2001 From: Stephen Rosen Date: Fri, 8 Nov 2024 09:27:04 -0600 Subject: [PATCH 6/9] Support `--group` on `pip wheel` Per review, support on `pip wheel` is desirable. This is net-net simpler, since we don't need any trickery to "dodge" the fact that it is a `RequirementCommand` but wasn't supporting `--group`. The desire to *not* support `--group` here was based on a mistaken idea about what `pip wheel` does. --- src/pip/_internal/cli/req_command.py | 1 + src/pip/_internal/commands/download.py | 1 - src/pip/_internal/commands/install.py | 1 - src/pip/_internal/commands/wheel.py | 4 --- tests/functional/test_wheel.py | 36 ++++++++++++++++++++++++++ 5 files changed, 37 insertions(+), 6 deletions(-) diff --git a/src/pip/_internal/cli/req_command.py b/src/pip/_internal/cli/req_command.py index 26bb8159fa8..82164e8a720 100644 --- a/src/pip/_internal/cli/req_command.py +++ b/src/pip/_internal/cli/req_command.py @@ -80,6 +80,7 @@ class RequirementCommand(IndexGroupCommand): def __init__(self, *args: Any, **kw: Any) -> None: super().__init__(*args, **kw) + self.cmd_opts.add_option(cmdoptions.dependency_groups()) self.cmd_opts.add_option(cmdoptions.no_clean()) @staticmethod diff --git a/src/pip/_internal/commands/download.py b/src/pip/_internal/commands/download.py index 51277b0d6a5..917bbb91d83 100644 --- a/src/pip/_internal/commands/download.py +++ b/src/pip/_internal/commands/download.py @@ -38,7 +38,6 @@ class DownloadCommand(RequirementCommand): def add_options(self) -> None: self.cmd_opts.add_option(cmdoptions.constraints()) self.cmd_opts.add_option(cmdoptions.requirements()) - self.cmd_opts.add_option(cmdoptions.dependency_groups()) self.cmd_opts.add_option(cmdoptions.no_deps()) self.cmd_opts.add_option(cmdoptions.global_options()) self.cmd_opts.add_option(cmdoptions.no_binary()) diff --git a/src/pip/_internal/commands/install.py b/src/pip/_internal/commands/install.py index ba6d6da3000..ad45a2f2a57 100644 --- a/src/pip/_internal/commands/install.py +++ b/src/pip/_internal/commands/install.py @@ -77,7 +77,6 @@ def add_options(self) -> None: self.cmd_opts.add_option(cmdoptions.pre()) self.cmd_opts.add_option(cmdoptions.editable()) - self.cmd_opts.add_option(cmdoptions.dependency_groups()) self.cmd_opts.add_option( "--dry-run", action="store_true", diff --git a/src/pip/_internal/commands/wheel.py b/src/pip/_internal/commands/wheel.py index f45e1692207..278719f4e0c 100644 --- a/src/pip/_internal/commands/wheel.py +++ b/src/pip/_internal/commands/wheel.py @@ -102,10 +102,6 @@ def add_options(self) -> None: @with_cleanup def run(self, options: Values, args: List[str]) -> int: - # dependency-groups aren't desirable with `pip wheel`, but providing it - # consistently allows RequirementCommand to expect it to be present - options.dependency_groups = [] - session = self.get_default_session(options) finder = self._build_package_finder(options, session) diff --git a/tests/functional/test_wheel.py b/tests/functional/test_wheel.py index da2bd2d7904..e1ede880496 100644 --- a/tests/functional/test_wheel.py +++ b/tests/functional/test_wheel.py @@ -3,6 +3,7 @@ import os import re import sys +import textwrap from pathlib import Path import pytest @@ -69,6 +70,41 @@ def test_pip_wheel_success(script: PipTestEnvironment, data: TestData) -> None: assert "Successfully built simple" in result.stdout, result.stdout +def test_pip_wheel_success_with_dependency_group( + script: PipTestEnvironment, data: TestData +) -> None: + """ + Test 'pip wheel' success. + """ + pyproject = script.scratch_path / "pyproject.toml" + pyproject.write_text( + textwrap.dedent( + """\ + [dependency-groups] + simple = ["simple==3.0"] + """ + ) + ) + result = script.pip( + "wheel", + "--no-index", + "-f", + data.find_links, + "--group", + "simple", + ) + wheel_file_name = f"simple-3.0-py{pyversion[0]}-none-any.whl" + wheel_file_path = script.scratch / wheel_file_name + assert re.search( + r"Created wheel for simple: " + rf"filename={re.escape(wheel_file_name)} size=\d+ sha256=[A-Fa-f0-9]{{64}}", + result.stdout, + ) + assert re.search(r"^\s+Stored in directory: ", result.stdout, re.M) + result.did_create(wheel_file_path) + assert "Successfully built simple" in result.stdout, result.stdout + + def test_pip_wheel_build_cache(script: PipTestEnvironment, data: TestData) -> None: """ Test 'pip wheel' builds and caches. From 9dbdabb0d9f592e3945eff1033c35ab629645886 Mon Sep 17 00:00:00 2001 From: Stephen Rosen Date: Fri, 8 Nov 2024 09:35:36 -0600 Subject: [PATCH 7/9] Review feedback: use dedent in tests --- tests/unit/test_req_dependency_group.py | 48 ++++++++++++++----------- 1 file changed, 27 insertions(+), 21 deletions(-) diff --git a/tests/unit/test_req_dependency_group.py b/tests/unit/test_req_dependency_group.py index dd78fd031ac..48caf6b211f 100644 --- a/tests/unit/test_req_dependency_group.py +++ b/tests/unit/test_req_dependency_group.py @@ -1,4 +1,5 @@ import errno +import textwrap from pathlib import Path from typing import Any @@ -13,10 +14,12 @@ def test_parse_simple_dependency_groups( ) -> None: pyproject = tmp_path.joinpath("pyproject.toml") pyproject.write_text( - """\ -[dependency-groups] -foo = ["bar"] -""" + textwrap.dedent( + """\ + [dependency-groups] + foo = ["bar"] + """ + ) ) monkeypatch.chdir(tmp_path) @@ -31,11 +34,13 @@ def test_parse_cyclic_dependency_groups( ) -> None: pyproject = tmp_path.joinpath("pyproject.toml") pyproject.write_text( - """\ -[dependency-groups] -foo = [{include-group="bar"}] -bar = [{include-group="foo"}] -""" + textwrap.dedent( + """\ + [dependency-groups] + foo = [{include-group="bar"}] + bar = [{include-group="foo"}] + """ + ) ) monkeypatch.chdir(tmp_path) @@ -54,10 +59,7 @@ def test_parse_with_no_dependency_groups_defined( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: pyproject = tmp_path.joinpath("pyproject.toml") - pyproject.write_text( - """\ -""" - ) + pyproject.write_text("") monkeypatch.chdir(tmp_path) with pytest.raises( @@ -80,10 +82,12 @@ def test_parse_with_malformed_pyproject_file( ) -> None: pyproject = tmp_path.joinpath("pyproject.toml") pyproject.write_text( - """\ -[dependency-groups # no closing bracket -foo = ["bar"] -""" + textwrap.dedent( + """\ + [dependency-groups # no closing bracket + foo = ["bar"] + """ + ) ) monkeypatch.chdir(tmp_path) @@ -96,10 +100,12 @@ def test_parse_gets_unexpected_oserror( ) -> None: pyproject = tmp_path.joinpath("pyproject.toml") pyproject.write_text( - """\ -[dependency-groups] -foo = ["bar"] -""" + textwrap.dedent( + """\ + [dependency-groups] + foo = ["bar"] + """ + ) ) monkeypatch.chdir(tmp_path) From cb10ef88de26a9d9efad1dba1d1ef1e64f5d9ced Mon Sep 17 00:00:00 2001 From: Stephen Rosen Date: Sat, 14 Dec 2024 22:00:18 -0600 Subject: [PATCH 8/9] Update `--group` option to take `[path:]name` In discussions about the correct interface for `pip` to use [dependency-groups], no strong consensus arose. However, the option with the most support appears to be to make it possible to pass a file path plus a group name. This change converts the `--group` option to take colon-separated path:groupname pairs, with the path part optional. The CLI parsing code is responsible for handling the syntax and for filling in a default path of `"pyproject.toml"`. If a path is provided, it must have a basename of `pyproject.toml`. Failing to meet this constraint is an error at arg parsing time. The `dependency_groups` usage is updated to create a DependencyGroupResolver per `pyproject.toml` file provided. This ensures that we only parse each file once, and we keep the results of previous resolutions when resolving multiple dependency groups from the same file. (Technically, the implementation is a resolver per path, which is subtly different from per-file, in that it doesn't account for symlinks, hardlinks, etc.) --- src/pip/_internal/cli/cmdoptions.py | 38 ++++++++- src/pip/_internal/req/req_dependency_group.py | 85 ++++++++++++------- tests/functional/test_install.py | 24 ++++++ tests/functional/test_install_reqs.py | 24 +++++- tests/unit/test_req_dependency_group.py | 21 +++-- 5 files changed, 147 insertions(+), 45 deletions(-) diff --git a/src/pip/_internal/cli/cmdoptions.py b/src/pip/_internal/cli/cmdoptions.py index c36a63d409a..6ddb318f5f6 100644 --- a/src/pip/_internal/cli/cmdoptions.py +++ b/src/pip/_internal/cli/cmdoptions.py @@ -13,6 +13,7 @@ import importlib.util import logging import os +import pathlib import textwrap from functools import partial from optparse import SUPPRESS_HELP, Option, OptionGroup, OptionParser, Values @@ -733,15 +734,44 @@ def _handle_no_cache_dir( help="Don't install package dependencies.", ) + +def _handle_dependency_group( + option: Option, opt: str, value: str, parser: OptionParser +) -> None: + """ + Process a value provided for the --group option. + + Splits on the rightmost ":", and validates that the path (if present) ends + in `pyproject.toml`. Defaults the path to `pyproject.toml` when one is not given. + + `:` cannot appear in dependency group names, so this is a safe and simple parse. + + This is an optparse.Option callback for the dependency_groups option. + """ + path, sep, groupname = value.rpartition(":") + if not sep: + path = "pyproject.toml" + else: + # check for 'pyproject.toml' filenames using pathlib + if pathlib.PurePath(path).name != "pyproject.toml": + msg = "group paths use 'pyproject.toml' filenames" + raise_option_error(parser, option=option, msg=msg) + + parser.values.dependency_groups.append((path, groupname)) + + dependency_groups: Callable[..., Option] = partial( Option, "--group", dest="dependency_groups", default=[], - action="append", - metavar="group", - help="Install a named dependency-group from `pyproject.toml` " - "in the current directory.", + type=str, + action="callback", + callback=_handle_dependency_group, + metavar="[path:]group", + help='Install a named dependency-group from a "pyproject.toml" file. ' + 'If a path is given, it must end in "pyproject.toml:". ' + 'Defaults to using "pyproject.toml" in the current directory.', ) ignore_requires_python: Callable[..., Option] = partial( diff --git a/src/pip/_internal/req/req_dependency_group.py b/src/pip/_internal/req/req_dependency_group.py index aa9839ead59..8f124de5b81 100644 --- a/src/pip/_internal/req/req_dependency_group.py +++ b/src/pip/_internal/req/req_dependency_group.py @@ -1,49 +1,74 @@ -from typing import Any, Dict, List +from typing import Any, Dict, Iterable, Iterator, List, Tuple from pip._vendor import tomli -from pip._vendor.dependency_groups import resolve as resolve_dependency_group +from pip._vendor.dependency_groups import DependencyGroupResolver from pip._internal.exceptions import InstallationError -def parse_dependency_groups(groups: List[str]) -> List[str]: +def parse_dependency_groups(groups: List[Tuple[str, str]]) -> List[str]: """ - Parse dependency groups data in a way which is sensitive to the `pip` context and - raises InstallationErrors if anything goes wrong. + Parse dependency groups data as provided via the CLI, in a `[path:]group` syntax. + + Raises InstallationErrors if anything goes wrong. """ - pyproject = _load_pyproject() - - if "dependency-groups" not in pyproject: - raise InstallationError( - "[dependency-groups] table was missing. Cannot resolve '--group' options." - ) - raw_dependency_groups = pyproject["dependency-groups"] - if not isinstance(raw_dependency_groups, dict): - raise InstallationError( - "[dependency-groups] table was malformed. Cannot resolve '--group' options." - ) + resolvers = _build_resolvers(path for (path, _) in groups) + return list(_resolve_all_groups(resolvers, groups)) - try: - return list(resolve_dependency_group(raw_dependency_groups, *groups)) - except (ValueError, TypeError, LookupError) as e: - raise InstallationError(f"[dependency-groups] resolution failed: {e}") from e + +def _resolve_all_groups( + resolvers: Dict[str, DependencyGroupResolver], groups: List[Tuple[str, str]] +) -> Iterator[str]: + """ + Run all resolution, converting any error from `DependencyGroupResolver` into + an InstallationError. + """ + for path, groupname in groups: + resolver = resolvers[path] + try: + yield from (str(req) for req in resolver.resolve(groupname)) + except (ValueError, TypeError, LookupError) as e: + raise InstallationError( + f"[dependency-groups] resolution failed for '{groupname}' " + f"from '{path}': {e}" + ) from e + + +def _build_resolvers(paths: Iterable[str]) -> Dict[str, Any]: + resolvers = {} + for path in paths: + if path in resolvers: + continue + + pyproject = _load_pyproject(path) + if "dependency-groups" not in pyproject: + raise InstallationError( + f"[dependency-groups] table was missing from '{path}'. " + "Cannot resolve '--group' option." + ) + raw_dependency_groups = pyproject["dependency-groups"] + if not isinstance(raw_dependency_groups, dict): + raise InstallationError( + f"[dependency-groups] table was malformed in {path}. " + "Cannot resolve '--group' option." + ) + + resolvers[path] = DependencyGroupResolver(raw_dependency_groups) + return resolvers -def _load_pyproject() -> Dict[str, Any]: +def _load_pyproject(path: str) -> Dict[str, Any]: """ - This helper loads pyproject.toml from the current working directory. + This helper loads a pyproject.toml as TOML. - It does not allow specification of the path to be used and raises an - InstallationError if the operation fails. + It raises an InstallationError if the operation fails. """ try: - with open("pyproject.toml", "rb") as fp: + with open(path, "rb") as fp: return tomli.load(fp) except FileNotFoundError: - raise InstallationError( - "pyproject.toml not found. Cannot resolve '--group' options." - ) + raise InstallationError(f"{path} not found. Cannot resolve '--group' option.") except tomli.TOMLDecodeError as e: - raise InstallationError(f"Error parsing pyproject.toml: {e}") from e + raise InstallationError(f"Error parsing {path}: {e}") from e except OSError as e: - raise InstallationError(f"Error reading pyproject.toml: {e}") from e + raise InstallationError(f"Error reading {path}: {e}") from e diff --git a/tests/functional/test_install.py b/tests/functional/test_install.py index 40ef585807b..e3f04260de5 100644 --- a/tests/functional/test_install.py +++ b/tests/functional/test_install.py @@ -333,6 +333,30 @@ def test_install_exit_status_code_when_empty_dependency_group( script.pip("install", "--group", "empty") +@pytest.mark.parametrize("file_exists", [True, False]) +def test_install_dependency_group_bad_filename_error( + script: PipTestEnvironment, file_exists: bool +) -> None: + """ + Test install exit status code is 2 (usage error) when a dependency group path is + specified which isn't a `pyproject.toml` + """ + if file_exists: + script.scratch_path.joinpath("not-pyproject.toml").write_text( + textwrap.dedent( + """ + [dependency-groups] + publish = ["twine"] + """ + ) + ) + result = script.pip( + "install", "--group", "not-pyproject.toml:publish", expect_error=True + ) + assert "group paths use 'pyproject.toml' filenames" in result.stderr + assert result.returncode == 2 + + @pytest.mark.network def test_basic_install_from_pypi(script: PipTestEnvironment) -> None: """ diff --git a/tests/functional/test_install_reqs.py b/tests/functional/test_install_reqs.py index 2d5ef8b0304..45103dc7c74 100644 --- a/tests/functional/test_install_reqs.py +++ b/tests/functional/test_install_reqs.py @@ -94,10 +94,22 @@ def test_requirements_file(script: PipTestEnvironment) -> None: @pytest.mark.network -def test_dependency_group(script: PipTestEnvironment) -> None: +@pytest.mark.parametrize( + "path, groupname", + [ + (None, "initools"), + ("pyproject.toml", "initools"), + ("./pyproject.toml", "initools"), + (lambda path: path.absolute(), "initools"), + ], +) +def test_dependency_group( + script: PipTestEnvironment, + path: Any, + groupname: str, +) -> None: """ Test installing from a dependency group. - """ pyproject = script.scratch_path / "pyproject.toml" pyproject.write_text( @@ -111,7 +123,13 @@ def test_dependency_group(script: PipTestEnvironment) -> None: """ ) ) - result = script.pip("install", "--group", "initools") + if path is None: + arg = groupname + else: + if callable(path): + path = path(pyproject) + arg = f"{path}:{groupname}" + result = script.pip("install", "--group", arg) result.did_create(script.site_packages / "INITools-0.2.dist-info") result.did_create(script.site_packages / "initools") assert result.files_created[script.site_packages / "peppercorn"].dir diff --git a/tests/unit/test_req_dependency_group.py b/tests/unit/test_req_dependency_group.py index 48caf6b211f..b596f6fc5d7 100644 --- a/tests/unit/test_req_dependency_group.py +++ b/tests/unit/test_req_dependency_group.py @@ -23,7 +23,7 @@ def test_parse_simple_dependency_groups( ) monkeypatch.chdir(tmp_path) - result = list(parse_dependency_groups(["foo"])) + result = list(parse_dependency_groups([("pyproject.toml", "foo")])) assert len(result) == 1, result assert result[0] == "bar" @@ -45,9 +45,13 @@ def test_parse_cyclic_dependency_groups( monkeypatch.chdir(tmp_path) with pytest.raises( - InstallationError, match=r"\[dependency-groups\] resolution failed:" + InstallationError, + match=( + r"\[dependency-groups\] resolution failed for " + r"'foo' from 'pyproject\.toml':" + ), ) as excinfo: - parse_dependency_groups(["foo"]) + parse_dependency_groups([("pyproject.toml", "foo")]) exception = excinfo.value assert ( @@ -63,9 +67,10 @@ def test_parse_with_no_dependency_groups_defined( monkeypatch.chdir(tmp_path) with pytest.raises( - InstallationError, match=r"\[dependency-groups\] table was missing\." + InstallationError, + match=(r"\[dependency-groups\] table was missing from 'pyproject\.toml'\."), ): - parse_dependency_groups(["foo"]) + parse_dependency_groups([("pyproject.toml", "foo")]) def test_parse_with_no_pyproject_file( @@ -74,7 +79,7 @@ def test_parse_with_no_pyproject_file( monkeypatch.chdir(tmp_path) with pytest.raises(InstallationError, match=r"pyproject\.toml not found\."): - parse_dependency_groups(["foo"]) + parse_dependency_groups([("pyproject.toml", "foo")]) def test_parse_with_malformed_pyproject_file( @@ -92,7 +97,7 @@ def test_parse_with_malformed_pyproject_file( monkeypatch.chdir(tmp_path) with pytest.raises(InstallationError, match=r"Error parsing pyproject\.toml"): - parse_dependency_groups(["foo"]) + parse_dependency_groups([("pyproject.toml", "foo")]) def test_parse_gets_unexpected_oserror( @@ -119,4 +124,4 @@ def epipe_toml_load(*args: Any, **kwargs: Any) -> None: ) with pytest.raises(InstallationError, match=r"Error reading pyproject\.toml"): - parse_dependency_groups(["foo"]) + parse_dependency_groups([("pyproject.toml", "foo")]) From 8aea313ea35c6045e9dc1ff1263f6c5b24630dcd Mon Sep 17 00:00:00 2001 From: Stephen Rosen Date: Sun, 15 Dec 2024 12:41:26 -0600 Subject: [PATCH 9/9] Update --group news fragment --- news/12963.feature.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/news/12963.feature.rst b/news/12963.feature.rst index 232dc3522e8..66320c7ba90 100644 --- a/news/12963.feature.rst +++ b/news/12963.feature.rst @@ -1,3 +1,4 @@ - Add a ``--group`` option which allows installation from PEP 735 Dependency - Groups. Only ``pyproject.toml`` files in the current working directory are - supported. + Groups. ``--group`` accepts arguments of the form ``group`` or + ``path:group``, where the default path is ``pyproject.toml``, and installs + the named Dependency Group from the provided ``pyproject.toml`` file.