diff --git a/README.md b/README.md index 5573d6bf3..c6e831b99 100644 --- a/README.md +++ b/README.md @@ -281,6 +281,26 @@ $ pip-compile requirements.in --pip-args "--retries 10 --timeout 30" ### Configuration +You can define project-level defaults for `pip-compile` and `pip-sync` by +writing them to a configuration file in the same directory as your requirements +input files (or the current working directory if piping input from stdin). +By default, both `pip-compile` and `pip-sync` will look first +for a `.pip-tools.toml` file and then in your `pyproject.toml`. You can +also specify an alternate TOML configuration file with the `--config` option. + +For example, to by default generate `pip` hashes in the resulting +requirements file output, you can specify in a configuration file + +```toml +[tool.pip-tools] +generate-hashes = true + +``` + +Options to `pip-compile` and `pip-sync` that may be used more than once +must be defined as lists in a configuration file, even if they only have one +value. + You might be wrapping the `pip-compile` command in another script. To avoid confusing consumers of your custom script you can override the update command generated at the top of requirements files by setting the diff --git a/piptools/locations.py b/piptools/locations.py index bf757603f..f31891cae 100644 --- a/piptools/locations.py +++ b/piptools/locations.py @@ -4,3 +4,6 @@ # The user_cache_dir helper comes straight from pip itself CACHE_DIR = user_cache_dir("pip-tools") + +# The project defaults specific to pip-tools should be written to this filename +CONFIG_FILE_NAME = ".pip-tools.toml" diff --git a/piptools/scripts/compile.py b/piptools/scripts/compile.py index b36ef8b30..837bb0f23 100755 --- a/piptools/scripts/compile.py +++ b/piptools/scripts/compile.py @@ -5,6 +5,7 @@ import shlex import sys import tempfile +from pathlib import Path from typing import IO, Any, BinaryIO, cast import click @@ -19,7 +20,7 @@ from .._compat import parse_requirements from ..cache import DependencyCache from ..exceptions import NoCandidateFound, PipToolsError -from ..locations import CACHE_DIR +from ..locations import CACHE_DIR, CONFIG_FILE_NAME from ..logging import log from ..repositories import LocalRequirementsRepository, PyPIRepository from ..repositories.base import BaseRepository @@ -30,6 +31,7 @@ drop_extras, is_pinned_requirement, key_from_ireq, + override_defaults_from_config_file, parse_requirements_from_wheel_metadata, ) from ..writer import OutputWriter @@ -302,6 +304,21 @@ def _determine_linesep( help="Specify a package to consider unsafe; may be used more than once. " f"Replaces default unsafe packages: {', '.join(sorted(UNSAFE_PACKAGES))}", ) +@click.option( + "--config", + type=click.Path( + exists=True, + file_okay=True, + dir_okay=False, + readable=True, + allow_dash=False, + path_type=str, + ), + help=f"Read configuration from TOML file. By default, looks for a {CONFIG_FILE_NAME} or " + "pyproject.toml.", + is_eager=True, + callback=override_defaults_from_config_file, +) def cli( ctx: click.Context, verbose: int, @@ -340,6 +357,7 @@ def cli( emit_index_url: bool, emit_options: bool, unsafe_package: tuple[str, ...], + config: Path | None, ) -> None: """ Compiles requirements.txt from requirements.in, pyproject.toml, setup.cfg, @@ -391,6 +409,9 @@ def cli( f"input and output filenames must not be matched: {output_file.name}" ) + if config: + log.info(f"Using pip-tools configuration defaults found in '{config !s}'.") + if resolver_name == "legacy": log.warning( "WARNING: the legacy dependency resolver is deprecated and will be removed" diff --git a/piptools/scripts/sync.py b/piptools/scripts/sync.py index 8b60adad5..d504d889a 100755 --- a/piptools/scripts/sync.py +++ b/piptools/scripts/sync.py @@ -5,6 +5,7 @@ import shlex import shutil import sys +from pathlib import Path from typing import cast import click @@ -17,6 +18,7 @@ from .._compat import parse_requirements from .._compat.pip_compat import Distribution from ..exceptions import PipToolsError +from ..locations import CONFIG_FILE_NAME from ..logging import log from ..repositories import PyPIRepository from ..utils import ( @@ -24,6 +26,7 @@ get_pip_version_for_python_executable, get_required_pip_specification, get_sys_path_for_python_executable, + override_defaults_from_config_file, ) DEFAULT_REQUIREMENTS_FILE = "requirements.txt" @@ -86,6 +89,21 @@ ) @click.argument("src_files", required=False, type=click.Path(exists=True), nargs=-1) @click.option("--pip-args", help="Arguments to pass directly to pip install.") +@click.option( + "--config", + type=click.Path( + exists=True, + file_okay=True, + dir_okay=False, + readable=True, + allow_dash=False, + path_type=str, + ), + help=f"Read configuration from TOML file. By default, looks for a {CONFIG_FILE_NAME} or " + "pyproject.toml.", + is_eager=True, + callback=override_defaults_from_config_file, +) def cli( ask: bool, dry_run: bool, @@ -103,6 +121,7 @@ def cli( client_cert: str | None, src_files: tuple[str, ...], pip_args: str | None, + config: Path | None, ) -> None: """Synchronize virtual environment with requirements.txt.""" log.verbosity = verbose - quiet @@ -127,6 +146,9 @@ def cli( log.error("ERROR: " + msg) sys.exit(2) + if config: + log.info(f"Using pip-tools configuration defaults found in '{config !s}'.") + if python_executable: _validate_python_executable(python_executable) diff --git a/piptools/utils.py b/piptools/utils.py index 64b5e96c6..4374b74a9 100644 --- a/piptools/utils.py +++ b/piptools/utils.py @@ -2,13 +2,21 @@ import collections import copy +import functools import itertools import json import os import re import shlex +import sys +from pathlib import Path from typing import TYPE_CHECKING, Any, Callable, Iterable, Iterator, TypeVar, cast +if sys.version_info >= (3, 11): + import tomllib +else: + import tomli as tomllib + import click from click.utils import LazyFile from pip._internal.req import InstallRequirement @@ -22,6 +30,7 @@ from pip._vendor.pkg_resources import Distribution, Requirement, get_distribution from piptools._compat import PIP_VERSION +from piptools.locations import CONFIG_FILE_NAME from piptools.subprocess_utils import run_python_snippet if TYPE_CHECKING: @@ -405,9 +414,9 @@ def get_required_pip_specification() -> SpecifierSet: Returns pip version specifier requested by current pip-tools installation. """ project_dist = get_distribution("pip-tools") - requirement = next( # pragma: no branch + requirement = next( (r for r in project_dist.requires() if r.name == "pip"), None - ) + ) # pragma: no branch assert ( requirement is not None ), "'pip' is expected to be in the list of pip-tools requirements" @@ -522,3 +531,128 @@ def parse_requirements_from_wheel_metadata( markers=parts.markers, extras=parts.extras, ) + + +def override_defaults_from_config_file( + ctx: click.Context, param: click.Parameter, value: str | None +) -> Path | None: + """ + Overrides ``click.Command`` defaults based on specified or discovered config + file, returning the ``pathlib.Path`` of that config file if specified or + discovered. + + ``None`` is returned if no such file is found. + + ``pip-tools`` will use the first config file found, searching in this order: + an explicitly given config file, a d``.pip-tools.toml``, a ``pyproject.toml`` + file. Those files are searched for in the same directory as the requirements + input file, or the current working directory if requirements come via stdin. + """ + if value is None: + config_file = select_config_file(ctx.params.get("src_files", ())) + if config_file is None: + return None + else: + config_file = Path(value) + + config = parse_config_file(config_file) + if config: + _assign_config_to_cli_context(ctx, config) + + return config_file + + +def _assign_config_to_cli_context( + click_context: click.Context, + cli_config_mapping: dict[str, Any], +) -> None: + if click_context.default_map is None: + click_context.default_map = {} + + click_context.default_map.update(cli_config_mapping) + + +def select_config_file(src_files: tuple[str, ...]) -> Path | None: + """ + Returns the config file to use for defaults given ``src_files`` provided. + """ + # NOTE: If no src_files were specified, consider the current directory the + # NOTE: only config file lookup candidate. This usually happens when a + # NOTE: pip-tools invocation gets its incoming requirements from standard + # NOTE: input. + working_directory = Path.cwd() + src_files_as_paths = ( + (working_directory / src_file).resolve() for src_file in src_files or (".",) + ) + candidate_dirs = (src if src.is_dir() else src.parent for src in src_files_as_paths) + config_file_path = next( + ( + candidate_dir / config_file + for candidate_dir in candidate_dirs + for config_file in (CONFIG_FILE_NAME, "pyproject.toml") + if (candidate_dir / config_file).is_file() + ), + None, + ) + return config_file_path + + +# Some of the defined click options have different `dest` values than the defaults +NON_STANDARD_OPTION_DEST_MAP: dict[str, str] = { + "extra": "extras", + "upgrade_package": "upgrade_packages", + "resolver": "resolver_name", + "user": "user_only", +} + + +def get_click_dest_for_option(option_name: str) -> str: + """ + Returns the click ``dest`` value for the given option name. + """ + # Format the keys properly + option_name = option_name.lstrip("-").replace("-", "_").lower() + # Some options have dest values that are overrides from the click generated default + option_name = NON_STANDARD_OPTION_DEST_MAP.get(option_name, option_name) + return option_name + + +# Ensure that any default overrides for these click options are lists, supporting multiple values +MULTIPLE_VALUE_OPTIONS = [ + "extras", + "upgrade_packages", + "unsafe_package", + "find_links", + "extra_index_url", + "trusted_host", +] + + +@functools.lru_cache() +def parse_config_file(config_file: Path) -> dict[str, Any]: + try: + config = tomllib.loads(config_file.read_text(encoding="utf-8")) + except OSError as os_err: + raise click.FileError( + filename=str(config_file), + hint=f"Could not read '{config_file !s}': {os_err !s}", + ) + except ValueError as value_err: + raise click.FileError( + filename=str(config_file), + hint=f"Could not parse '{config_file !s}': {value_err !s}", + ) + + # In a TOML file, we expect the config to be under `[tool.pip-tools]` + piptools_config: dict[str, Any] = config.get("tool", {}).get("pip-tools", {}) + piptools_config = { + get_click_dest_for_option(k): v for k, v in piptools_config.items() + } + # Any option with multiple values needs to be a list in the pyproject.toml + for mv_option in MULTIPLE_VALUE_OPTIONS: + if not isinstance(piptools_config.get(mv_option), (list, type(None))): + original_option = mv_option.replace("_", "-") + raise click.BadOptionUsage( + original_option, f"Config key '{original_option}' must be a list" + ) + return piptools_config diff --git a/pyproject.toml b/pyproject.toml index 8c061c9ad..47fa90566 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,6 +40,7 @@ dependencies = [ "build", "click >= 8", "pip >= 22.2", + "tomli; python_version < '3.11'", # indirect dependencies "setuptools", # typically needed when pip-tools invokes setup.py "wheel", # pip plugin needed by pip-tools @@ -57,6 +58,7 @@ testing = [ "pytest >= 7.2.0", "pytest-rerunfailures", "pytest-xdist", + "tomli-w", # build deps for tests "flit_core >=2,<4", "poetry_core>=1.0.0", diff --git a/tests/conftest.py b/tests/conftest.py index 01849de91..70f0e0e90 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -8,9 +8,12 @@ from contextlib import contextmanager from dataclasses import dataclass, field from functools import partial +from pathlib import Path from textwrap import dedent +from typing import Any import pytest +import tomli_w from click.testing import CliRunner from pip._internal.commands.install import InstallCommand from pip._internal.index.package_finder import PackageFinder @@ -26,6 +29,7 @@ from piptools._compat.pip_compat import PIP_VERSION, uses_pkg_resources from piptools.cache import DependencyCache from piptools.exceptions import NoCandidateFound +from piptools.locations import CONFIG_FILE_NAME from piptools.logging import log from piptools.repositories import PyPIRepository from piptools.repositories.base import BaseRepository @@ -450,3 +454,25 @@ def _reset_log(): with other tests that depend on it. """ log.reset() + + +@pytest.fixture +def make_config_file(tmpdir_cwd): + """ + Make a config file for pip-tools with a given parameter set to a specific + value, returning a ``pathlib.Path`` to the config file. + """ + + def _maker( + pyproject_param: str, new_default: Any, config_file_name: str = CONFIG_FILE_NAME + ) -> Path: + # Make a config file with this one config default override + config_path = Path(tmpdir_cwd) / pyproject_param + config_file = config_path / config_file_name + config_path.mkdir(exist_ok=True) + + config_to_dump = {"tool": {"pip-tools": {pyproject_param: new_default}}} + config_file.write_text(tomli_w.dumps(config_to_dump)) + return config_file + + return _maker diff --git a/tests/test_utils.py b/tests/test_utils.py index 9d25899f1..5f1e9ab5d 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -5,9 +5,11 @@ import os import shlex import sys +from pathlib import Path import pip import pytest +from click import BadOptionUsage, Context, FileError from pip._vendor.packaging.version import Version from piptools.scripts.compile import cli as compile_cli @@ -18,6 +20,7 @@ flat_map, format_requirement, format_specifier, + get_click_dest_for_option, get_compile_command, get_hashes_from_ireq, get_pip_version_for_python_executable, @@ -27,6 +30,7 @@ key_from_ireq, lookup_table, lookup_table_from_tuples, + override_defaults_from_config_file, ) @@ -540,3 +544,113 @@ def test_get_sys_path_for_python_executable(): # not testing for equality, because pytest adds extra paths into current sys.path for path in result: assert path in sys.path + + +@pytest.mark.parametrize( + ("pyproject_param", "new_default"), + ( + # From sync + ("ask", True), + ("dry-run", True), + ("find-links", ["changed"]), + ("extra-index-url", ["changed"]), + ("trusted-host", ["changed"]), + ("no-index", True), + ("python-executable", "changed"), + ("verbose", True), + ("quiet", True), + ("user", True), + ("cert", "changed"), + ("client-cert", "changed"), + ("pip-args", "changed"), + # From compile, unless also in sync + ("pre", True), + ("rebuild", True), + ("extras", ["changed"]), + ("all-extras", True), + ("index-url", "changed"), + ("header", False), + ("emit-trusted-host", False), + ("annotate", False), + ("annotation-style", "line"), + ("upgrade", True), + ("upgrade-package", ["changed"]), + ("output-file", "changed"), + ("newline", "native"), + ("allow-unsafe", True), + ("strip-extras", True), + ("generate-hashes", True), + ("reuse-hashes", False), + ("max-rounds", 100), + ("build-isolation", False), + ("emit-find-links", False), + ("cache-dir", "changed"), + ("resolver", "backtracking"), + ("emit-index-url", False), + ("emit-options", False), + ("unsafe-package", ["changed"]), + ), +) +def test_callback_config_file_defaults(pyproject_param, new_default, make_config_file): + config_file = make_config_file(pyproject_param, new_default) + # Create a "compile" run example pointing to the config file + ctx = Context(compile_cli) + ctx.params["src_files"] = (str(config_file),) + found_config_file = override_defaults_from_config_file(ctx, "config", None) + assert found_config_file == config_file + # Make sure the default has been updated + lookup_param = get_click_dest_for_option(pyproject_param) + assert ctx.default_map[lookup_param] == new_default + + +@pytest.mark.parametrize( + "mv_option", + ( + "extra", + "upgrade-package", + "unsafe-package", + "find-links", + "extra-index-url", + "trusted-host", + ), +) +def test_callback_config_file_defaults_multi_value_options(mv_option, make_config_file): + config_file = make_config_file(mv_option, "not-a-list") + ctx = Context(compile_cli) + ctx.params["src_files"] = (str(config_file),) + with pytest.raises(BadOptionUsage, match="must be a list"): + override_defaults_from_config_file(ctx, "config", None) + + +def test_callback_config_file_defaults_bad_toml(make_config_file): + config_file = make_config_file("verbose", True) + # Simple means of making invalid TOML: have duplicate keys + with Path(config_file).open("r+") as fs: + config_text_lines = fs.readlines() + fs.write(config_text_lines[-1]) + ctx = Context(compile_cli) + ctx.params["src_files"] = (str(config_file),) + with pytest.raises(FileError, match="Could not parse "): + override_defaults_from_config_file(ctx, "config", None) + + +def test_callback_config_file_defaults_precedence(make_config_file): + piptools_config_file = make_config_file("newline", "LF") + project_config_file = make_config_file("newline", "CRLF", "pyproject.toml") + ctx = Context(compile_cli) + ctx.params["src_files"] = (str(project_config_file),) + found_config_file = override_defaults_from_config_file(ctx, "config", None) + # The pip-tools specific config file should take precedence over pyproject.toml + assert found_config_file == piptools_config_file + lookup_param = get_click_dest_for_option("newline") + assert ctx.default_map[lookup_param] == "LF" + + +def test_callback_config_file_defaults_unreadable_toml(make_config_file): + ctx = Context(compile_cli) + with pytest.raises(FileError, match="Could not read "): + override_defaults_from_config_file( + ctx, + "config", + "/dev/null/path/does/not/exist/my-config.toml", + )