From 1204f721298132a5f1fdca6d19a655b2b9ae1d4d Mon Sep 17 00:00:00 2001 From: "Joshua \"jag\" Ginsberg" Date: Thu, 4 May 2023 23:00:35 -0400 Subject: [PATCH 01/27] Set config defaults using pyproject.toml --- .pre-commit-config.yaml | 1 + piptools/scripts/compile.py | 19 +++++++ piptools/scripts/sync.py | 19 +++++++ piptools/utils.py | 104 ++++++++++++++++++++++++++++++++++++ pyproject.toml | 1 + tests/conftest.py | 17 ++++++ tests/test_utils.py | 102 +++++++++++++++++++++++++++++++++++ 7 files changed, 263 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index fb5b54af8..e0a2cd3aa 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -33,6 +33,7 @@ repos: - toml==0.10.2 - pip==20.3.4 - build==0.9.0 + - types-toml==0.10.2 - repo: https://github.com/PyCQA/bandit rev: 1.7.5 hooks: diff --git a/piptools/scripts/compile.py b/piptools/scripts/compile.py index b36ef8b30..f8d8fd1c4 100755 --- a/piptools/scripts/compile.py +++ b/piptools/scripts/compile.py @@ -31,6 +31,7 @@ is_pinned_requirement, key_from_ireq, parse_requirements_from_wheel_metadata, + pyproject_toml_defaults_cb, ) from ..writer import OutputWriter @@ -302,6 +303,20 @@ def _determine_linesep( help="Specify a package to consider unsafe; may be used more than once. " f"Replaces default unsafe packages: {', '.join(sorted(UNSAFE_PACKAGES))}", ) +@click.option( + "--config", + type=click.Path( + exists=True, + file_okay=True, + dir_okay=False, + readable=True, + allow_dash=False, + path_type=str, + ), + help="Path to a pyproject.toml file with specialized defaults for pip-tools", + is_eager=True, + callback=pyproject_toml_defaults_cb, +) def cli( ctx: click.Context, verbose: int, @@ -340,6 +355,7 @@ def cli( emit_index_url: bool, emit_options: bool, unsafe_package: tuple[str, ...], + config: str | None, ) -> None: """ Compiles requirements.txt from requirements.in, pyproject.toml, setup.cfg, @@ -391,6 +407,9 @@ def cli( f"input and output filenames must not be matched: {output_file.name}" ) + if config: + log.info(f"Using pip-tools configuration defaults found in '{config}'.") + if resolver_name == "legacy": log.warning( "WARNING: the legacy dependency resolver is deprecated and will be removed" diff --git a/piptools/scripts/sync.py b/piptools/scripts/sync.py index 8b60adad5..2a7c60a6a 100755 --- a/piptools/scripts/sync.py +++ b/piptools/scripts/sync.py @@ -24,6 +24,7 @@ get_pip_version_for_python_executable, get_required_pip_specification, get_sys_path_for_python_executable, + pyproject_toml_defaults_cb, ) DEFAULT_REQUIREMENTS_FILE = "requirements.txt" @@ -86,6 +87,20 @@ ) @click.argument("src_files", required=False, type=click.Path(exists=True), nargs=-1) @click.option("--pip-args", help="Arguments to pass directly to pip install.") +@click.option( + "--config", + type=click.Path( + exists=True, + file_okay=True, + dir_okay=False, + readable=True, + allow_dash=False, + path_type=str, + ), + help="Path to a pyproject.toml file with specialized defaults for pip-tools", + is_eager=True, + callback=pyproject_toml_defaults_cb, +) def cli( ask: bool, dry_run: bool, @@ -103,6 +118,7 @@ def cli( client_cert: str | None, src_files: tuple[str, ...], pip_args: str | None, + config: str | None, ) -> None: """Synchronize virtual environment with requirements.txt.""" log.verbosity = verbose - quiet @@ -127,6 +143,9 @@ def cli( log.error("ERROR: " + msg) sys.exit(2) + if config: + log.info(f"Using pip-tools configuration defaults found in '{config}'.") + if python_executable: _validate_python_executable(python_executable) diff --git a/piptools/utils.py b/piptools/utils.py index 64b5e96c6..4509f8e37 100644 --- a/piptools/utils.py +++ b/piptools/utils.py @@ -2,14 +2,17 @@ import collections import copy +import functools import itertools import json import os import re import shlex +from pathlib import Path from typing import TYPE_CHECKING, Any, Callable, Iterable, Iterator, TypeVar, cast import click +import toml from click.utils import LazyFile from pip._internal.req import InstallRequirement from pip._internal.req.constructors import install_req_from_line, parse_req_from_line @@ -522,3 +525,104 @@ def parse_requirements_from_wheel_metadata( markers=parts.markers, extras=parts.extras, ) + + +def pyproject_toml_defaults_cb( + ctx: click.Context, param: click.Parameter, value: str | None +) -> str | None: + """ + Defaults for `click.Command` parameters should be override-able in pyproject.toml + + Returns the path to the configuration file found, or None if no such file is found. + """ + if value is None: + config_file = find_pyproject_toml(ctx.params.get("src_files", ())) + if config_file is None: + return None + else: + config_file = value + + try: + config = parse_pyproject_toml(config_file) + except OSError as e: + raise click.FileError( + filename=config_file, hint=f"Could not read '{config_file}': {e}" + ) + except ValueError as e: + raise click.FileError( + filename=config_file, hint=f"Could not parse '{config_file}': {e}" + ) + + if not config: + return None + + defaults: dict[str, Any] = ctx.default_map.copy() if ctx.default_map else {} + defaults.update(config) + + ctx.default_map = defaults + return config_file + + +def find_pyproject_toml(src_files: tuple[str, ...]) -> str | None: + if not src_files: + # If no src_files were specified, we consider the current directory the only candidate + candidates = [Path.cwd()] + else: + # Collect the candidate directories based on the src_file arguments provided + src_files_as_paths = [ + Path(Path.cwd(), src_file).resolve() for src_file in src_files + ] + candidates = [src if src.is_dir() else src.parent for src in src_files_as_paths] + pyproject_toml_path = next( + ( + str(candidate / "pyproject.toml") + for candidate in candidates + if (candidate / "pyproject.toml").is_file() + ), + None, + ) + return pyproject_toml_path + + +# Some of the defined click options have different `dest` values than the defaults +NON_STANDARD_OPTION_DEST_MAP: dict[str, str] = { + "extra": "extras", + "upgrade_package": "upgrade_packages", + "resolver": "resolver_name", + "user": "user_only", +} + + +def mutate_option_to_click_dest(option_name: str) -> str: + "Mutates an option from how click/pyproject.toml expect them to the click `dest` value" + # Format the keys properly + option_name = option_name.lstrip("-").replace("-", "_").lower() + # Some options have dest values that are overrides from the click generated default + option_name = NON_STANDARD_OPTION_DEST_MAP.get(option_name, option_name) + return option_name + + +# Ensure that any default overrides for these click options are lists, supporting multiple values +MULTIPLE_VALUE_OPTIONS = [ + "extras", + "upgrade_packages", + "unsafe_package", + "find_links", + "extra_index_url", + "trusted_host", +] + + +@functools.lru_cache() +def parse_pyproject_toml(config_file: str) -> dict[str, Any]: + pyproject_toml = toml.load(config_file) + config: dict[str, Any] = pyproject_toml.get("tool", {}).get("pip-tools", {}) + config = {mutate_option_to_click_dest(k): v for k, v in config.items()} + # Any option with multiple values needs to be a list in the pyproject.toml + for mv_option in MULTIPLE_VALUE_OPTIONS: + if not isinstance(config.get(mv_option), (list, type(None))): + original_option = mv_option.replace("_", "-") + raise click.BadOptionUsage( + original_option, f"Config key '{original_option}' must be a list" + ) + return config diff --git a/pyproject.toml b/pyproject.toml index 8c061c9ad..68e5ad4bb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,6 +42,7 @@ dependencies = [ "pip >= 22.2", # indirect dependencies "setuptools", # typically needed when pip-tools invokes setup.py + "toml >= 0.10.1", "wheel", # pip plugin needed by pip-tools ] diff --git a/tests/conftest.py b/tests/conftest.py index 01849de91..d6536b22d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -8,9 +8,11 @@ from contextlib import contextmanager from dataclasses import dataclass, field from functools import partial +from pathlib import Path from textwrap import dedent import pytest +import toml from click.testing import CliRunner from pip._internal.commands.install import InstallCommand from pip._internal.index.package_finder import PackageFinder @@ -450,3 +452,18 @@ def _reset_log(): with other tests that depend on it. """ log.reset() + + +@pytest.fixture +def make_pyproject_toml_conf(tmpdir_cwd): + def _maker(pyproject_param, new_default): + # Make a pyproject.toml with this one config default override + config_path = Path(tmpdir_cwd) / pyproject_param + config_file = config_path / "pyproject.toml" + config_path.mkdir() + + with open(config_file, "w") as ofs: + toml.dump({"tool": {"pip-tools": {pyproject_param: new_default}}}, ofs) + return config_file + + return _maker diff --git a/tests/test_utils.py b/tests/test_utils.py index 9d25899f1..9e91dbb44 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -8,6 +8,7 @@ import pip import pytest +from click import BadOptionUsage, Context, FileError from pip._vendor.packaging.version import Version from piptools.scripts.compile import cli as compile_cli @@ -27,6 +28,8 @@ key_from_ireq, lookup_table, lookup_table_from_tuples, + mutate_option_to_click_dest, + pyproject_toml_defaults_cb, ) @@ -540,3 +543,102 @@ def test_get_sys_path_for_python_executable(): # not testing for equality, because pytest adds extra paths into current sys.path for path in result: assert path in sys.path + + +@pytest.mark.parametrize( + ("pyproject_param", "new_default"), + ( + # From sync + ("ask", True), + ("dry-run", True), + ("find-links", ["changed"]), + ("extra-index-url", ["changed"]), + ("trusted-host", ["changed"]), + ("no-index", True), + ("python-executable", "changed"), + ("verbose", True), + ("quiet", True), + ("user", True), + ("cert", "changed"), + ("client-cert", "changed"), + ("pip-args", "changed"), + # From compile, unless also in sync + ("pre", True), + ("rebuild", True), + ("extras", ["changed"]), + ("all-extras", True), + ("index-url", "changed"), + ("header", False), + ("emit-trusted-host", False), + ("annotate", False), + ("annotation-style", "line"), + ("upgrade", True), + ("upgrade-package", ["changed"]), + ("output-file", "changed"), + ("newline", "native"), + ("allow-unsafe", True), + ("strip-extras", True), + ("generate-hashes", True), + ("reuse-hashes", False), + ("max-rounds", 100), + ("build-isolation", False), + ("emit-find-links", False), + ("cache-dir", "changed"), + ("resolver", "backtracking"), + ("emit-index-url", False), + ("emit-options", False), + ("unsafe-package", ["changed"]), + ), +) +def test_pyproject_toml_defaults_cb( + pyproject_param, new_default, make_pyproject_toml_conf +): + config_file = make_pyproject_toml_conf(pyproject_param, new_default) + # Create a "compile" run example pointing to the pyproject.toml + ctx = Context(compile_cli) + ctx.params["src_files"] = (str(config_file),) + found_config_file = pyproject_toml_defaults_cb(ctx, "config", None) + assert found_config_file == str(config_file) + # Make sure the default has been updated + lookup_param = mutate_option_to_click_dest(pyproject_param) + assert ctx.default_map[lookup_param] == new_default + + +@pytest.mark.parametrize( + "mv_option", + ( + "extra", + "upgrade-package", + "unsafe-package", + "find-links", + "extra-index-url", + "trusted-host", + ), +) +def test_pyproject_toml_defaults_cb_multi_value_options( + mv_option, make_pyproject_toml_conf +): + config_file = make_pyproject_toml_conf(mv_option, "not-a-list") + ctx = Context(compile_cli) + ctx.params["src_files"] = (str(config_file),) + pytest.raises(BadOptionUsage, pyproject_toml_defaults_cb, ctx, "config", None) + + +def test_pyproject_toml_defaults_cb_bad_toml(make_pyproject_toml_conf): + config_file = make_pyproject_toml_conf("verbose", True) + config_text = open(config_file).read() + open(config_file, "w").write(config_text[::-1]) + ctx = Context(compile_cli) + ctx.params["src_files"] = (str(config_file),) + pytest.raises(FileError, pyproject_toml_defaults_cb, ctx, "config", None) + + +def test_pyproject_toml_defaults_cb_unreadable_toml(make_pyproject_toml_conf): + ctx = Context(compile_cli) + pytest.raises( + FileError, + pyproject_toml_defaults_cb, + ctx, + "config", + "/path/does/not/exist/pyproject.toml", + ) From 04af2a5aaaa4ffe78b25a43781dbc6d47f0d2e24 Mon Sep 17 00:00:00 2001 From: "Joshua \"jag\" Ginsberg" Date: Sun, 7 May 2023 14:58:59 -0400 Subject: [PATCH 02/27] PR review feedback. Updated README. --- README.md | 23 ++++++++++ piptools/locations.py | 3 ++ piptools/scripts/compile.py | 9 ++-- piptools/scripts/sync.py | 8 ++-- piptools/utils.py | 84 ++++++++++++++++++++++++------------- pyproject.toml | 3 +- tests/conftest.py | 16 ++++--- tests/test_utils.py | 66 +++++++++++++++++------------ 8 files changed, 143 insertions(+), 69 deletions(-) diff --git a/README.md b/README.md index 5573d6bf3..997f4d222 100644 --- a/README.md +++ b/README.md @@ -281,6 +281,29 @@ $ pip-compile requirements.in --pip-args "--retries 10 --timeout 30" ### Configuration +You can define project-level defaults for `pip-compile` and `pip-sync` by +writing them to a configuration file in the same directory as your requirements +input file. By default, both `pip-compile` and `pip-sync` will look first +for a `.pip-tools.toml` file and then in your `pyproject.toml`. You can +also specify an alternate TOML configuration file with the `--config` option. + +For example, to by default generate `pip` hashes in the resulting +requirements file output, you can specify in a configuration file + +```toml +# In a .pip-tools.toml file +[pip-tools] +generate-hashes = true + +# In a pyproject.toml file +[tool.pip-tools] +generate-hashes = true +``` + +Options to `pip-compile` and `pip-sync` that may be used more than once +must be defined as lists in a configuration file, even if they only have one +value. + You might be wrapping the `pip-compile` command in another script. To avoid confusing consumers of your custom script you can override the update command generated at the top of requirements files by setting the diff --git a/piptools/locations.py b/piptools/locations.py index bf757603f..f31891cae 100644 --- a/piptools/locations.py +++ b/piptools/locations.py @@ -4,3 +4,6 @@ # The user_cache_dir helper comes straight from pip itself CACHE_DIR = user_cache_dir("pip-tools") + +# The project defaults specific to pip-tools should be written to this filename +CONFIG_FILE_NAME = ".pip-tools.toml" diff --git a/piptools/scripts/compile.py b/piptools/scripts/compile.py index f8d8fd1c4..1b3662ff1 100755 --- a/piptools/scripts/compile.py +++ b/piptools/scripts/compile.py @@ -19,19 +19,19 @@ from .._compat import parse_requirements from ..cache import DependencyCache from ..exceptions import NoCandidateFound, PipToolsError -from ..locations import CACHE_DIR +from ..locations import CACHE_DIR, CONFIG_FILE_NAME from ..logging import log from ..repositories import LocalRequirementsRepository, PyPIRepository from ..repositories.base import BaseRepository from ..resolver import BacktrackingResolver, LegacyResolver from ..utils import ( UNSAFE_PACKAGES, + callback_config_file_defaults, dedup, drop_extras, is_pinned_requirement, key_from_ireq, parse_requirements_from_wheel_metadata, - pyproject_toml_defaults_cb, ) from ..writer import OutputWriter @@ -313,9 +313,10 @@ def _determine_linesep( allow_dash=False, path_type=str, ), - help="Path to a pyproject.toml file with specialized defaults for pip-tools", + help=f"Read configuration from TOML file. By default, looks for a {CONFIG_FILE_NAME} or " + "pyproject.toml.", is_eager=True, - callback=pyproject_toml_defaults_cb, + callback=callback_config_file_defaults, ) def cli( ctx: click.Context, diff --git a/piptools/scripts/sync.py b/piptools/scripts/sync.py index 2a7c60a6a..c09fd082f 100755 --- a/piptools/scripts/sync.py +++ b/piptools/scripts/sync.py @@ -17,14 +17,15 @@ from .._compat import parse_requirements from .._compat.pip_compat import Distribution from ..exceptions import PipToolsError +from ..locations import CONFIG_FILE_NAME from ..logging import log from ..repositories import PyPIRepository from ..utils import ( + callback_config_file_defaults, flat_map, get_pip_version_for_python_executable, get_required_pip_specification, get_sys_path_for_python_executable, - pyproject_toml_defaults_cb, ) DEFAULT_REQUIREMENTS_FILE = "requirements.txt" @@ -97,9 +98,10 @@ allow_dash=False, path_type=str, ), - help="Path to a pyproject.toml file with specialized defaults for pip-tools", + help=f"Read configuration from TOML file. By default, looks for a {CONFIG_FILE_NAME} or " + "pyproject.toml.", is_eager=True, - callback=pyproject_toml_defaults_cb, + callback=callback_config_file_defaults, ) def cli( ask: bool, diff --git a/piptools/utils.py b/piptools/utils.py index 4509f8e37..e750903bd 100644 --- a/piptools/utils.py +++ b/piptools/utils.py @@ -8,11 +8,16 @@ import os import re import shlex +import sys from pathlib import Path from typing import TYPE_CHECKING, Any, Callable, Iterable, Iterator, TypeVar, cast import click -import toml + +if sys.version_info >= (3, 11): + import tomllib +else: + import toml from click.utils import LazyFile from pip._internal.req import InstallRequirement from pip._internal.req.constructors import install_req_from_line, parse_req_from_line @@ -25,6 +30,7 @@ from pip._vendor.pkg_resources import Distribution, Requirement, get_distribution from piptools._compat import PIP_VERSION +from piptools.locations import CONFIG_FILE_NAME from piptools.subprocess_utils import run_python_snippet if TYPE_CHECKING: @@ -408,9 +414,9 @@ def get_required_pip_specification() -> SpecifierSet: Returns pip version specifier requested by current pip-tools installation. """ project_dist = get_distribution("pip-tools") - requirement = next( # pragma: no branch + requirement = next( (r for r in project_dist.requires() if r.name == "pip"), None - ) + ) # pragma: no branch assert ( requirement is not None ), "'pip' is expected to be in the list of pip-tools requirements" @@ -527,30 +533,34 @@ def parse_requirements_from_wheel_metadata( ) -def pyproject_toml_defaults_cb( +def callback_config_file_defaults( ctx: click.Context, param: click.Parameter, value: str | None ) -> str | None: """ - Defaults for `click.Command` parameters should be override-able in pyproject.toml + Returns the path to the config file with defaults being used, or `None` if no such file is + found. - Returns the path to the configuration file found, or None if no such file is found. + Defaults for `click.Command` parameters should be override-able in a config file. `pip-tools` + will use the first file found, searching in this order: an explicitly given config file, a + `.pip-tools.toml`, a `pyproject.toml` file. Those files are searched for in the same directory + as the requirements input file. """ if value is None: - config_file = find_pyproject_toml(ctx.params.get("src_files", ())) + config_file = select_config_file(ctx.params.get("src_files", ())) if config_file is None: return None else: - config_file = value + config_file = Path(value) try: - config = parse_pyproject_toml(config_file) + config = parse_config_file(config_file) except OSError as e: raise click.FileError( - filename=config_file, hint=f"Could not read '{config_file}': {e}" + filename=str(config_file), hint=f"Could not read '{config_file}': {e}" ) except ValueError as e: raise click.FileError( - filename=config_file, hint=f"Could not parse '{config_file}': {e}" + filename=str(config_file), hint=f"Could not parse '{config_file}': {e}" ) if not config: @@ -560,28 +570,34 @@ def pyproject_toml_defaults_cb( defaults.update(config) ctx.default_map = defaults - return config_file + return str(config_file) -def find_pyproject_toml(src_files: tuple[str, ...]) -> str | None: +def select_config_file(src_files: tuple[str, ...]) -> Path | None: + """ + Returns the config file to use for defaults given `src_files` provided. + """ if not src_files: # If no src_files were specified, we consider the current directory the only candidate - candidates = [Path.cwd()] + candidate_dirs = [Path.cwd()] else: # Collect the candidate directories based on the src_file arguments provided src_files_as_paths = [ Path(Path.cwd(), src_file).resolve() for src_file in src_files ] - candidates = [src if src.is_dir() else src.parent for src in src_files_as_paths] - pyproject_toml_path = next( + candidate_dirs = [ + src if src.is_dir() else src.parent for src in src_files_as_paths + ] + config_file_path = next( ( - str(candidate / "pyproject.toml") - for candidate in candidates - if (candidate / "pyproject.toml").is_file() + candidate_dir / config_file + for candidate_dir in candidate_dirs + for config_file in (CONFIG_FILE_NAME, "pyproject.toml") + if (candidate_dir / config_file).is_file() ), None, ) - return pyproject_toml_path + return config_file_path # Some of the defined click options have different `dest` values than the defaults @@ -593,8 +609,8 @@ def find_pyproject_toml(src_files: tuple[str, ...]) -> str | None: } -def mutate_option_to_click_dest(option_name: str) -> str: - "Mutates an option from how click/pyproject.toml expect them to the click `dest` value" +def get_click_dest_for_option(option_name: str) -> str: + """Returns the click `dest` value for the given option name.""" # Format the keys properly option_name = option_name.lstrip("-").replace("-", "_").lower() # Some options have dest values that are overrides from the click generated default @@ -614,15 +630,27 @@ def mutate_option_to_click_dest(option_name: str) -> str: @functools.lru_cache() -def parse_pyproject_toml(config_file: str) -> dict[str, Any]: - pyproject_toml = toml.load(config_file) - config: dict[str, Any] = pyproject_toml.get("tool", {}).get("pip-tools", {}) - config = {mutate_option_to_click_dest(k): v for k, v in config.items()} +def parse_config_file(config_file: Path) -> dict[str, Any]: + if sys.version_info >= (3, 11): + # Python 3.11 stdlib tomllib load() requires a binary file object + with config_file.open("rb") as ifs: + config = tomllib.load(ifs) + else: + # Before 3.11, using the external toml library, load requires the filename + config = toml.load(str(config_file)) + # In a pyproject.toml file, we expect the config to be under `[tool.pip-tools]`, but in our + # native configuration, it would be just `[pip-tools]`. + if config_file.name == "pyproject.toml": + config = config.get("tool", {}) + piptools_config: dict[str, Any] = config.get("pip-tools", {}) + piptools_config = { + get_click_dest_for_option(k): v for k, v in piptools_config.items() + } # Any option with multiple values needs to be a list in the pyproject.toml for mv_option in MULTIPLE_VALUE_OPTIONS: - if not isinstance(config.get(mv_option), (list, type(None))): + if not isinstance(piptools_config.get(mv_option), (list, type(None))): original_option = mv_option.replace("_", "-") raise click.BadOptionUsage( original_option, f"Config key '{original_option}' must be a list" ) - return config + return piptools_config diff --git a/pyproject.toml b/pyproject.toml index 68e5ad4bb..716f9bf22 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,9 +40,9 @@ dependencies = [ "build", "click >= 8", "pip >= 22.2", + "toml >= 0.10.1; python_version<'3.11'", # indirect dependencies "setuptools", # typically needed when pip-tools invokes setup.py - "toml >= 0.10.1", "wheel", # pip plugin needed by pip-tools ] @@ -58,6 +58,7 @@ testing = [ "pytest >= 7.2.0", "pytest-rerunfailures", "pytest-xdist", + "toml >= 0.10.1", # build deps for tests "flit_core >=2,<4", "poetry_core>=1.0.0", diff --git a/tests/conftest.py b/tests/conftest.py index d6536b22d..166f86fa3 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -28,6 +28,7 @@ from piptools._compat.pip_compat import PIP_VERSION, uses_pkg_resources from piptools.cache import DependencyCache from piptools.exceptions import NoCandidateFound +from piptools.locations import CONFIG_FILE_NAME from piptools.logging import log from piptools.repositories import PyPIRepository from piptools.repositories.base import BaseRepository @@ -455,15 +456,18 @@ def _reset_log(): @pytest.fixture -def make_pyproject_toml_conf(tmpdir_cwd): - def _maker(pyproject_param, new_default): - # Make a pyproject.toml with this one config default override +def make_config_file(tmpdir_cwd): + def _maker(pyproject_param, new_default, config_file_name=CONFIG_FILE_NAME): + # Make a config file with this one config default override config_path = Path(tmpdir_cwd) / pyproject_param - config_file = config_path / "pyproject.toml" - config_path.mkdir() + config_file = config_path / config_file_name + config_path.mkdir(exist_ok=True) + config_to_dump = {"pip-tools": {pyproject_param: new_default}} + if config_file_name == "pyproject.toml": + config_to_dump = {"tool": config_to_dump} with open(config_file, "w") as ofs: - toml.dump({"tool": {"pip-tools": {pyproject_param: new_default}}}, ofs) + toml.dump(config_to_dump, ofs) return config_file return _maker diff --git a/tests/test_utils.py b/tests/test_utils.py index 9e91dbb44..e2bee3e33 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -5,6 +5,7 @@ import os import shlex import sys +from pathlib import Path import pip import pytest @@ -14,11 +15,13 @@ from piptools.scripts.compile import cli as compile_cli from piptools.utils import ( as_tuple, + callback_config_file_defaults, dedup, drop_extras, flat_map, format_requirement, format_specifier, + get_click_dest_for_option, get_compile_command, get_hashes_from_ireq, get_pip_version_for_python_executable, @@ -28,8 +31,6 @@ key_from_ireq, lookup_table, lookup_table_from_tuples, - mutate_option_to_click_dest, - pyproject_toml_defaults_cb, ) @@ -590,17 +591,15 @@ def test_get_sys_path_for_python_executable(): ("unsafe-package", ["changed"]), ), ) -def test_pyproject_toml_defaults_cb( - pyproject_param, new_default, make_pyproject_toml_conf -): - config_file = make_pyproject_toml_conf(pyproject_param, new_default) - # Create a "compile" run example pointing to the pyproject.toml +def test_callback_config_file_defaults(pyproject_param, new_default, make_config_file): + config_file = make_config_file(pyproject_param, new_default) + # Create a "compile" run example pointing to the config file ctx = Context(compile_cli) ctx.params["src_files"] = (str(config_file),) - found_config_file = pyproject_toml_defaults_cb(ctx, "config", None) + found_config_file = callback_config_file_defaults(ctx, "config", None) assert found_config_file == str(config_file) # Make sure the default has been updated - lookup_param = mutate_option_to_click_dest(pyproject_param) + lookup_param = get_click_dest_for_option(pyproject_param) assert ctx.default_map[lookup_param] == new_default @@ -615,30 +614,43 @@ def test_pyproject_toml_defaults_cb( "trusted-host", ), ) -def test_pyproject_toml_defaults_cb_multi_value_options( - mv_option, make_pyproject_toml_conf -): - config_file = make_pyproject_toml_conf(mv_option, "not-a-list") +def test_callback_config_file_defaults_multi_value_options(mv_option, make_config_file): + config_file = make_config_file(mv_option, "not-a-list") ctx = Context(compile_cli) ctx.params["src_files"] = (str(config_file),) - pytest.raises(BadOptionUsage, pyproject_toml_defaults_cb, ctx, "config", None) + with pytest.raises(BadOptionUsage, match="must be a list"): + callback_config_file_defaults(ctx, "config", None) -def test_pyproject_toml_defaults_cb_bad_toml(make_pyproject_toml_conf): - config_file = make_pyproject_toml_conf("verbose", True) - config_text = open(config_file).read() - open(config_file, "w").write(config_text[::-1]) +def test_callback_config_file_defaults_bad_toml(make_config_file): + config_file = make_config_file("verbose", True) + # Simple means of making invalid TOML: have duplicate keys + with Path(config_file).open("r+") as fs: + config_text_lines = fs.readlines() + fs.write(config_text_lines[-1]) ctx = Context(compile_cli) ctx.params["src_files"] = (str(config_file),) - pytest.raises(FileError, pyproject_toml_defaults_cb, ctx, "config", None) + with pytest.raises(FileError, match="Could not parse "): + callback_config_file_defaults(ctx, "config", None) -def test_pyproject_toml_defaults_cb_unreadable_toml(make_pyproject_toml_conf): +def test_callback_config_file_defaults_precedence(make_config_file): + piptools_config_file = make_config_file("newline", "LF") + project_config_file = make_config_file("newline", "CRLF", "pyproject.toml") ctx = Context(compile_cli) - pytest.raises( - FileError, - pyproject_toml_defaults_cb, - ctx, - "config", - "/path/does/not/exist/pyproject.toml", - ) + ctx.params["src_files"] = (str(project_config_file),) + found_config_file = callback_config_file_defaults(ctx, "config", None) + # The pip-tools specific config file should take precedence over pyproject.toml + assert found_config_file == str(piptools_config_file) + lookup_param = get_click_dest_for_option("newline") + assert ctx.default_map[lookup_param] == "LF" + + +def test_callback_config_file_defaults_unreadable_toml(make_config_file): + ctx = Context(compile_cli) + with pytest.raises(FileError, match="Could not read "): + callback_config_file_defaults( + ctx, + "config", + "/path/does/not/exist/my-config.toml", + ) From 06fb7934b8310c1302285397f8b3cc87fe8ac57e Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Tue, 9 May 2023 02:11:03 +0200 Subject: [PATCH 03/27] Switch the toml fallback library to `tomli` This implementation is the de-facto go-to choice nowadays. It is also API-compatible with the stdlib's `tomllib`. --- .pre-commit-config.yaml | 1 - piptools/utils.py | 14 ++++---------- pyproject.toml | 4 ++-- tests/conftest.py | 5 ++--- 4 files changed, 8 insertions(+), 16 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e0a2cd3aa..fb5b54af8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -33,7 +33,6 @@ repos: - toml==0.10.2 - pip==20.3.4 - build==0.9.0 - - types-toml==0.10.2 - repo: https://github.com/PyCQA/bandit rev: 1.7.5 hooks: diff --git a/piptools/utils.py b/piptools/utils.py index e750903bd..47aa092d4 100644 --- a/piptools/utils.py +++ b/piptools/utils.py @@ -12,12 +12,12 @@ from pathlib import Path from typing import TYPE_CHECKING, Any, Callable, Iterable, Iterator, TypeVar, cast -import click - if sys.version_info >= (3, 11): import tomllib else: - import toml + import tomli as tomllib + +import click from click.utils import LazyFile from pip._internal.req import InstallRequirement from pip._internal.req.constructors import install_req_from_line, parse_req_from_line @@ -631,13 +631,7 @@ def get_click_dest_for_option(option_name: str) -> str: @functools.lru_cache() def parse_config_file(config_file: Path) -> dict[str, Any]: - if sys.version_info >= (3, 11): - # Python 3.11 stdlib tomllib load() requires a binary file object - with config_file.open("rb") as ifs: - config = tomllib.load(ifs) - else: - # Before 3.11, using the external toml library, load requires the filename - config = toml.load(str(config_file)) + config = tomllib.loads(config_file.read_text(encoding="utf-8")) # In a pyproject.toml file, we expect the config to be under `[tool.pip-tools]`, but in our # native configuration, it would be just `[pip-tools]`. if config_file.name == "pyproject.toml": diff --git a/pyproject.toml b/pyproject.toml index 716f9bf22..47fa90566 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,7 +40,7 @@ dependencies = [ "build", "click >= 8", "pip >= 22.2", - "toml >= 0.10.1; python_version<'3.11'", + "tomli; python_version < '3.11'", # indirect dependencies "setuptools", # typically needed when pip-tools invokes setup.py "wheel", # pip plugin needed by pip-tools @@ -58,7 +58,7 @@ testing = [ "pytest >= 7.2.0", "pytest-rerunfailures", "pytest-xdist", - "toml >= 0.10.1", + "tomli-w", # build deps for tests "flit_core >=2,<4", "poetry_core>=1.0.0", diff --git a/tests/conftest.py b/tests/conftest.py index 166f86fa3..46709e670 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -12,7 +12,7 @@ from textwrap import dedent import pytest -import toml +import tomli_w from click.testing import CliRunner from pip._internal.commands.install import InstallCommand from pip._internal.index.package_finder import PackageFinder @@ -466,8 +466,7 @@ def _maker(pyproject_param, new_default, config_file_name=CONFIG_FILE_NAME): config_to_dump = {"pip-tools": {pyproject_param: new_default}} if config_file_name == "pyproject.toml": config_to_dump = {"tool": config_to_dump} - with open(config_file, "w") as ofs: - toml.dump(config_to_dump, ofs) + config_file.write_text(tomli_w.dumps(config_to_dump)) return config_file return _maker From 9ef5f8fe0419f712b7d19104dfc45180e7ff4d01 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Tue, 9 May 2023 02:41:47 +0200 Subject: [PATCH 04/27] Use `pathlib.Path` objects internally It is the best practice nowadays. Paths are only converted to strings for text representation, like when logging. --- piptools/scripts/compile.py | 5 +++-- piptools/scripts/sync.py | 5 +++-- piptools/utils.py | 4 ++-- tests/test_utils.py | 2 +- 4 files changed, 9 insertions(+), 7 deletions(-) diff --git a/piptools/scripts/compile.py b/piptools/scripts/compile.py index 1b3662ff1..1447cf7fd 100755 --- a/piptools/scripts/compile.py +++ b/piptools/scripts/compile.py @@ -16,6 +16,7 @@ from pip._internal.req.constructors import install_req_from_line from pip._internal.utils.misc import redact_auth_from_url +from pathlib import Path from .._compat import parse_requirements from ..cache import DependencyCache from ..exceptions import NoCandidateFound, PipToolsError @@ -356,7 +357,7 @@ def cli( emit_index_url: bool, emit_options: bool, unsafe_package: tuple[str, ...], - config: str | None, + config: Path | None, ) -> None: """ Compiles requirements.txt from requirements.in, pyproject.toml, setup.cfg, @@ -409,7 +410,7 @@ def cli( ) if config: - log.info(f"Using pip-tools configuration defaults found in '{config}'.") + log.info(f"Using pip-tools configuration defaults found in '{config !s}'.") if resolver_name == "legacy": log.warning( diff --git a/piptools/scripts/sync.py b/piptools/scripts/sync.py index c09fd082f..3f3360dff 100755 --- a/piptools/scripts/sync.py +++ b/piptools/scripts/sync.py @@ -14,6 +14,7 @@ from pip._internal.metadata import get_environment from .. import sync +from pathlib import Path from .._compat import parse_requirements from .._compat.pip_compat import Distribution from ..exceptions import PipToolsError @@ -120,7 +121,7 @@ def cli( client_cert: str | None, src_files: tuple[str, ...], pip_args: str | None, - config: str | None, + config: Path | None, ) -> None: """Synchronize virtual environment with requirements.txt.""" log.verbosity = verbose - quiet @@ -146,7 +147,7 @@ def cli( sys.exit(2) if config: - log.info(f"Using pip-tools configuration defaults found in '{config}'.") + log.info(f"Using pip-tools configuration defaults found in '{config !s}'.") if python_executable: _validate_python_executable(python_executable) diff --git a/piptools/utils.py b/piptools/utils.py index 47aa092d4..829920576 100644 --- a/piptools/utils.py +++ b/piptools/utils.py @@ -535,7 +535,7 @@ def parse_requirements_from_wheel_metadata( def callback_config_file_defaults( ctx: click.Context, param: click.Parameter, value: str | None -) -> str | None: +) -> Path | None: """ Returns the path to the config file with defaults being used, or `None` if no such file is found. @@ -570,7 +570,7 @@ def callback_config_file_defaults( defaults.update(config) ctx.default_map = defaults - return str(config_file) + return config_file def select_config_file(src_files: tuple[str, ...]) -> Path | None: diff --git a/tests/test_utils.py b/tests/test_utils.py index e2bee3e33..05371b474 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -597,7 +597,7 @@ def test_callback_config_file_defaults(pyproject_param, new_default, make_config ctx = Context(compile_cli) ctx.params["src_files"] = (str(config_file),) found_config_file = callback_config_file_defaults(ctx, "config", None) - assert found_config_file == str(config_file) + assert found_config_file == config_file # Make sure the default has been updated lookup_param = get_click_dest_for_option(pyproject_param) assert ctx.default_map[lookup_param] == new_default From 0098ee5355c19db14b40c0d43fb38550aeb01c50 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 9 May 2023 00:42:00 +0000 Subject: [PATCH 05/27] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- piptools/scripts/compile.py | 2 +- piptools/scripts/sync.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/piptools/scripts/compile.py b/piptools/scripts/compile.py index 1447cf7fd..cfced2d2c 100755 --- a/piptools/scripts/compile.py +++ b/piptools/scripts/compile.py @@ -5,6 +5,7 @@ import shlex import sys import tempfile +from pathlib import Path from typing import IO, Any, BinaryIO, cast import click @@ -16,7 +17,6 @@ from pip._internal.req.constructors import install_req_from_line from pip._internal.utils.misc import redact_auth_from_url -from pathlib import Path from .._compat import parse_requirements from ..cache import DependencyCache from ..exceptions import NoCandidateFound, PipToolsError diff --git a/piptools/scripts/sync.py b/piptools/scripts/sync.py index 3f3360dff..428d7d649 100755 --- a/piptools/scripts/sync.py +++ b/piptools/scripts/sync.py @@ -5,6 +5,7 @@ import shlex import shutil import sys +from pathlib import Path from typing import cast import click @@ -14,7 +15,6 @@ from pip._internal.metadata import get_environment from .. import sync -from pathlib import Path from .._compat import parse_requirements from .._compat.pip_compat import Distribution from ..exceptions import PipToolsError From fc4b8f50e20153ee9edc0552e42d2f2d3b8496e4 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Tue, 9 May 2023 02:48:33 +0200 Subject: [PATCH 06/27] Update the config defaults in CLI ctx in place This patch is supposed to improve the readability of said block of code and optimize it a bit. --- piptools/utils.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/piptools/utils.py b/piptools/utils.py index 829920576..5a69fc036 100644 --- a/piptools/utils.py +++ b/piptools/utils.py @@ -566,10 +566,9 @@ def callback_config_file_defaults( if not config: return None - defaults: dict[str, Any] = ctx.default_map.copy() if ctx.default_map else {} - defaults.update(config) - - ctx.default_map = defaults + if ctx.default_map is None: + ctx.default_map = {} + ctx.default_map.update(config) return config_file From cc28f4239fbfc352bd838d870076b7e5b4f21dba Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Tue, 9 May 2023 03:01:19 +0200 Subject: [PATCH 07/27] =?UTF-8?q?=F0=9F=8E=A8=20Separate=20updating=20CLI?= =?UTF-8?q?=20context=20with=20the=20config?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is a refactoring patch aimed at improving readability. --- piptools/utils.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/piptools/utils.py b/piptools/utils.py index 5a69fc036..1db201c1d 100644 --- a/piptools/utils.py +++ b/piptools/utils.py @@ -566,12 +566,20 @@ def callback_config_file_defaults( if not config: return None - if ctx.default_map is None: - ctx.default_map = {} - ctx.default_map.update(config) + _assign_config_to_cli_context(ctx, config) return config_file +def _assign_config_to_cli_context( + click_context: click.Context, + cli_config_mapping: dict[str, Any], +) -> None: + if click_context.default_map is None: + click_context.default_map = {} + + click_context.default_map.update(cli_config_mapping) + + def select_config_file(src_files: tuple[str, ...]) -> Path | None: """ Returns the config file to use for defaults given `src_files` provided. From 5dab95d7382691c11f40576ed4a7dc1583b0f218 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Tue, 9 May 2023 03:10:19 +0200 Subject: [PATCH 08/27] =?UTF-8?q?=F0=9F=8E=A8Mv=20parsing=20error=20proces?= =?UTF-8?q?sing->parse=5Fconfig=5Ffile?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This helper is a better location since it already contains similar processing code that converts problems into process-interrupting exceptions. It is also rather weird to process low-level exceptions on the higher level when the context is lost. --- piptools/utils.py | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/piptools/utils.py b/piptools/utils.py index 1db201c1d..89da43019 100644 --- a/piptools/utils.py +++ b/piptools/utils.py @@ -552,17 +552,7 @@ def callback_config_file_defaults( else: config_file = Path(value) - try: - config = parse_config_file(config_file) - except OSError as e: - raise click.FileError( - filename=str(config_file), hint=f"Could not read '{config_file}': {e}" - ) - except ValueError as e: - raise click.FileError( - filename=str(config_file), hint=f"Could not parse '{config_file}': {e}" - ) - + config = parse_config_file(config_file) if not config: return None @@ -638,7 +628,19 @@ def get_click_dest_for_option(option_name: str) -> str: @functools.lru_cache() def parse_config_file(config_file: Path) -> dict[str, Any]: - config = tomllib.loads(config_file.read_text(encoding="utf-8")) + try: + config = tomllib.loads(config_file.read_text(encoding="utf-8")) + except OSError as os_err: + raise click.FileError( + filename=str(config_file), + hint=f"Could not read '{config_file !s}': {os_err !s}" + ) + except ValueError as value_err: + raise click.FileError( + filename=str(config_file), + hint=f"Could not parse '{config_file !s}': {value_err !s}" + ) + # In a pyproject.toml file, we expect the config to be under `[tool.pip-tools]`, but in our # native configuration, it would be just `[pip-tools]`. if config_file.name == "pyproject.toml": From 92e455f886634c47993c3c83ec1c4841bfa1a4f4 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Tue, 9 May 2023 03:15:09 +0200 Subject: [PATCH 09/27] =?UTF-8?q?=F0=9F=8E=A8=20Couple=20config=20parsing-?= =?UTF-8?q?n-ctx-assigning=20w/=20walrus?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This change demonstrates a rather indivisible relation between getting the config contents into the CLI context making it a perfect target for the future refactoring. --- piptools/utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/piptools/utils.py b/piptools/utils.py index 89da43019..ed838b49e 100644 --- a/piptools/utils.py +++ b/piptools/utils.py @@ -552,11 +552,11 @@ def callback_config_file_defaults( else: config_file = Path(value) - config = parse_config_file(config_file) - if not config: + if config := parse_config_file(config_file): + _assign_config_to_cli_context(ctx, config) + else: return None - _assign_config_to_cli_context(ctx, config) return config_file From 2e86931dafd552b6f8847992bf49be8f65397d01 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 9 May 2023 01:17:45 +0000 Subject: [PATCH 10/27] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- piptools/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/piptools/utils.py b/piptools/utils.py index ed838b49e..aa9c21b60 100644 --- a/piptools/utils.py +++ b/piptools/utils.py @@ -633,12 +633,12 @@ def parse_config_file(config_file: Path) -> dict[str, Any]: except OSError as os_err: raise click.FileError( filename=str(config_file), - hint=f"Could not read '{config_file !s}': {os_err !s}" + hint=f"Could not read '{config_file !s}': {os_err !s}", ) except ValueError as value_err: raise click.FileError( filename=str(config_file), - hint=f"Could not parse '{config_file !s}': {value_err !s}" + hint=f"Could not parse '{config_file !s}': {value_err !s}", ) # In a pyproject.toml file, we expect the config to be under `[tool.pip-tools]`, but in our From f4692a7531d218b7070601ef1702f3df5c258356 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Tue, 9 May 2023 12:10:36 +0200 Subject: [PATCH 11/27] =?UTF-8?q?=F0=9F=90=9B=20Compare=20pathlib=20object?= =?UTF-8?q?s=20@=20precedence=20test?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This was missing from a previous commit, could be used to fix it up. --- tests/test_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_utils.py b/tests/test_utils.py index 05371b474..13c396adc 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -641,7 +641,7 @@ def test_callback_config_file_defaults_precedence(make_config_file): ctx.params["src_files"] = (str(project_config_file),) found_config_file = callback_config_file_defaults(ctx, "config", None) # The pip-tools specific config file should take precedence over pyproject.toml - assert found_config_file == str(piptools_config_file) + assert found_config_file == piptools_config_file lookup_param = get_click_dest_for_option("newline") assert ctx.default_map[lookup_param] == "LF" From c8298e0ef6031ab37e2d63c08e5fb62169c9c9da Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Tue, 9 May 2023 13:16:24 +0200 Subject: [PATCH 12/27] =?UTF-8?q?=F0=9F=8E=A8=20Rename=20config=20option?= =?UTF-8?q?=20callback=20to=20`determine=5Fconfig=5Ffile`?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This improves the readability and follows common function naming practices of using verbs to describe actions they perform. --- piptools/scripts/compile.py | 4 ++-- piptools/scripts/sync.py | 4 ++-- piptools/utils.py | 2 +- tests/test_utils.py | 12 ++++++------ 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/piptools/scripts/compile.py b/piptools/scripts/compile.py index cfced2d2c..3fc702550 100755 --- a/piptools/scripts/compile.py +++ b/piptools/scripts/compile.py @@ -27,7 +27,7 @@ from ..resolver import BacktrackingResolver, LegacyResolver from ..utils import ( UNSAFE_PACKAGES, - callback_config_file_defaults, + determine_config_file, dedup, drop_extras, is_pinned_requirement, @@ -317,7 +317,7 @@ def _determine_linesep( help=f"Read configuration from TOML file. By default, looks for a {CONFIG_FILE_NAME} or " "pyproject.toml.", is_eager=True, - callback=callback_config_file_defaults, + callback=determine_config_file, ) def cli( ctx: click.Context, diff --git a/piptools/scripts/sync.py b/piptools/scripts/sync.py index 428d7d649..e88910924 100755 --- a/piptools/scripts/sync.py +++ b/piptools/scripts/sync.py @@ -22,7 +22,7 @@ from ..logging import log from ..repositories import PyPIRepository from ..utils import ( - callback_config_file_defaults, + determine_config_file, flat_map, get_pip_version_for_python_executable, get_required_pip_specification, @@ -102,7 +102,7 @@ help=f"Read configuration from TOML file. By default, looks for a {CONFIG_FILE_NAME} or " "pyproject.toml.", is_eager=True, - callback=callback_config_file_defaults, + callback=determine_config_file, ) def cli( ask: bool, diff --git a/piptools/utils.py b/piptools/utils.py index aa9c21b60..9a48c585b 100644 --- a/piptools/utils.py +++ b/piptools/utils.py @@ -533,7 +533,7 @@ def parse_requirements_from_wheel_metadata( ) -def callback_config_file_defaults( +def determine_config_file( ctx: click.Context, param: click.Parameter, value: str | None ) -> Path | None: """ diff --git a/tests/test_utils.py b/tests/test_utils.py index 13c396adc..720fa1e15 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -15,7 +15,7 @@ from piptools.scripts.compile import cli as compile_cli from piptools.utils import ( as_tuple, - callback_config_file_defaults, + determine_config_file, dedup, drop_extras, flat_map, @@ -596,7 +596,7 @@ def test_callback_config_file_defaults(pyproject_param, new_default, make_config # Create a "compile" run example pointing to the config file ctx = Context(compile_cli) ctx.params["src_files"] = (str(config_file),) - found_config_file = callback_config_file_defaults(ctx, "config", None) + found_config_file = determine_config_file(ctx, "config", None) assert found_config_file == config_file # Make sure the default has been updated lookup_param = get_click_dest_for_option(pyproject_param) @@ -619,7 +619,7 @@ def test_callback_config_file_defaults_multi_value_options(mv_option, make_confi ctx = Context(compile_cli) ctx.params["src_files"] = (str(config_file),) with pytest.raises(BadOptionUsage, match="must be a list"): - callback_config_file_defaults(ctx, "config", None) + determine_config_file(ctx, "config", None) def test_callback_config_file_defaults_bad_toml(make_config_file): @@ -631,7 +631,7 @@ def test_callback_config_file_defaults_bad_toml(make_config_file): ctx = Context(compile_cli) ctx.params["src_files"] = (str(config_file),) with pytest.raises(FileError, match="Could not parse "): - callback_config_file_defaults(ctx, "config", None) + determine_config_file(ctx, "config", None) def test_callback_config_file_defaults_precedence(make_config_file): @@ -639,7 +639,7 @@ def test_callback_config_file_defaults_precedence(make_config_file): project_config_file = make_config_file("newline", "CRLF", "pyproject.toml") ctx = Context(compile_cli) ctx.params["src_files"] = (str(project_config_file),) - found_config_file = callback_config_file_defaults(ctx, "config", None) + found_config_file = determine_config_file(ctx, "config", None) # The pip-tools specific config file should take precedence over pyproject.toml assert found_config_file == piptools_config_file lookup_param = get_click_dest_for_option("newline") @@ -649,7 +649,7 @@ def test_callback_config_file_defaults_precedence(make_config_file): def test_callback_config_file_defaults_unreadable_toml(make_config_file): ctx = Context(compile_cli) with pytest.raises(FileError, match="Could not read "): - callback_config_file_defaults( + determine_config_file( ctx, "config", "/path/does/not/exist/my-config.toml", From fe16e2683b880b425e3e55b20b9015f0cd8aab52 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 9 May 2023 11:16:38 +0000 Subject: [PATCH 13/27] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- piptools/scripts/compile.py | 2 +- tests/test_utils.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/piptools/scripts/compile.py b/piptools/scripts/compile.py index 3fc702550..e26a689f9 100755 --- a/piptools/scripts/compile.py +++ b/piptools/scripts/compile.py @@ -27,8 +27,8 @@ from ..resolver import BacktrackingResolver, LegacyResolver from ..utils import ( UNSAFE_PACKAGES, - determine_config_file, dedup, + determine_config_file, drop_extras, is_pinned_requirement, key_from_ireq, diff --git a/tests/test_utils.py b/tests/test_utils.py index 720fa1e15..0f865bd26 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -15,8 +15,8 @@ from piptools.scripts.compile import cli as compile_cli from piptools.utils import ( as_tuple, - determine_config_file, dedup, + determine_config_file, drop_extras, flat_map, format_requirement, From cc4a91f4762397095e81e5fbd1f0c2ceb4e19923 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Tue, 9 May 2023 16:15:35 +0200 Subject: [PATCH 14/27] Unwalrus `piptools.utils` We'll be able to revert this commit in about 1.5 months, once Python 3.7 is no longer supported. --- piptools/utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/piptools/utils.py b/piptools/utils.py index 9a48c585b..b7a8d5a0b 100644 --- a/piptools/utils.py +++ b/piptools/utils.py @@ -552,7 +552,8 @@ def determine_config_file( else: config_file = Path(value) - if config := parse_config_file(config_file): + config = parse_config_file(config_file) + if config: _assign_config_to_cli_context(ctx, config) else: return None From c3c47958c5ce21b0bf51ece13f3faffa948a4ad7 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Tue, 9 May 2023 16:27:53 +0200 Subject: [PATCH 15/27] Simplify config file lookup @ `select_config_file` This makes it easier to follow that's happening in the helper function step-by-step. --- piptools/utils.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/piptools/utils.py b/piptools/utils.py index b7a8d5a0b..7f6d45e01 100644 --- a/piptools/utils.py +++ b/piptools/utils.py @@ -575,17 +575,17 @@ def select_config_file(src_files: tuple[str, ...]) -> Path | None: """ Returns the config file to use for defaults given `src_files` provided. """ - if not src_files: - # If no src_files were specified, we consider the current directory the only candidate - candidate_dirs = [Path.cwd()] - else: - # Collect the candidate directories based on the src_file arguments provided - src_files_as_paths = [ - Path(Path.cwd(), src_file).resolve() for src_file in src_files - ] - candidate_dirs = [ - src if src.is_dir() else src.parent for src in src_files_as_paths - ] + # NOTE: If no src_files were specified, consider the current directory the + # NOTE: only config file lookup candidate. This usually happens when a + # NOTE: pip-tools invocation gets its incoming requirements from standard + # NOTE: input. + src_files_as_paths = [ + Path(Path.cwd(), src_file).resolve() + for src_file in src_files or ('.', ) + ] + candidate_dirs = [ + src if src.is_dir() else src.parent for src in src_files_as_paths + ] config_file_path = next( ( candidate_dir / config_file From 64e3a2a055e42718b158ca540db1923083981f55 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Tue, 9 May 2023 16:30:46 +0200 Subject: [PATCH 16/27] Iterate over config lookup dirs lazily The change converts list comprehensions into generator expressions. This helps save memory and reduce unnecessary syscalls. --- piptools/utils.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/piptools/utils.py b/piptools/utils.py index 7f6d45e01..c506b84d3 100644 --- a/piptools/utils.py +++ b/piptools/utils.py @@ -579,13 +579,14 @@ def select_config_file(src_files: tuple[str, ...]) -> Path | None: # NOTE: only config file lookup candidate. This usually happens when a # NOTE: pip-tools invocation gets its incoming requirements from standard # NOTE: input. - src_files_as_paths = [ + src_files_as_paths = ( Path(Path.cwd(), src_file).resolve() for src_file in src_files or ('.', ) - ] - candidate_dirs = [ - src if src.is_dir() else src.parent for src in src_files_as_paths - ] + ) + candidate_dirs = ( + src if src.is_dir() else src.parent + for src in src_files_as_paths + ) config_file_path = next( ( candidate_dir / config_file From 050e278d59895e267f27419bb55c1171f97f6bfb Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 9 May 2023 14:31:54 +0000 Subject: [PATCH 17/27] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- piptools/utils.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/piptools/utils.py b/piptools/utils.py index c506b84d3..e6ef13ee1 100644 --- a/piptools/utils.py +++ b/piptools/utils.py @@ -580,13 +580,9 @@ def select_config_file(src_files: tuple[str, ...]) -> Path | None: # NOTE: pip-tools invocation gets its incoming requirements from standard # NOTE: input. src_files_as_paths = ( - Path(Path.cwd(), src_file).resolve() - for src_file in src_files or ('.', ) - ) - candidate_dirs = ( - src if src.is_dir() else src.parent - for src in src_files_as_paths + Path(Path.cwd(), src_file).resolve() for src_file in src_files or (".",) ) + candidate_dirs = (src if src.is_dir() else src.parent for src in src_files_as_paths) config_file_path = next( ( candidate_dir / config_file From a1116417e49828a7cd34f6248b7bcaf485768a25 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Tue, 9 May 2023 16:34:17 +0200 Subject: [PATCH 18/27] Drop unnecessary parens from `lru_cache` This is a tiny cosmetic change, making the code cleaner. --- piptools/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/piptools/utils.py b/piptools/utils.py index e6ef13ee1..0fda90c77 100644 --- a/piptools/utils.py +++ b/piptools/utils.py @@ -624,7 +624,7 @@ def get_click_dest_for_option(option_name: str) -> str: ] -@functools.lru_cache() +@functools.lru_cache def parse_config_file(config_file: Path) -> dict[str, Any]: try: config = tomllib.loads(config_file.read_text(encoding="utf-8")) From ca769007b151bf98dfa2e5be1e7ad9b21ebd16fd Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Tue, 9 May 2023 17:07:29 +0200 Subject: [PATCH 19/27] Reuse working dir in `select_config_file` This makes sure there's no unnecessary syscalls. --- piptools/utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/piptools/utils.py b/piptools/utils.py index 0fda90c77..388d7ba33 100644 --- a/piptools/utils.py +++ b/piptools/utils.py @@ -579,8 +579,9 @@ def select_config_file(src_files: tuple[str, ...]) -> Path | None: # NOTE: only config file lookup candidate. This usually happens when a # NOTE: pip-tools invocation gets its incoming requirements from standard # NOTE: input. + working_directory = Path.cwd() src_files_as_paths = ( - Path(Path.cwd(), src_file).resolve() for src_file in src_files or (".",) + (working_directory / src_file).resolve() for src_file in src_files or (".",) ) candidate_dirs = (src if src.is_dir() else src.parent for src in src_files_as_paths) config_file_path = next( From 8627df85b02f22ff93169c049cf60bcc942c06bb Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Tue, 9 May 2023 17:16:32 +0200 Subject: [PATCH 20/27] Revert "Drop unnecessary parens from `lru_cache` " This reverts commit 75f2729ddf2658873467e288eb1ae3b120359ae6. That patch was incompatible with Python 3.7. --- piptools/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/piptools/utils.py b/piptools/utils.py index 388d7ba33..df0ac2e8f 100644 --- a/piptools/utils.py +++ b/piptools/utils.py @@ -625,7 +625,7 @@ def get_click_dest_for_option(option_name: str) -> str: ] -@functools.lru_cache +@functools.lru_cache() def parse_config_file(config_file: Path) -> dict[str, Any]: try: config = tomllib.loads(config_file.read_text(encoding="utf-8")) From db2f30b7914f0c935cc700be050aedf9d80ad6a3 Mon Sep 17 00:00:00 2001 From: "Joshua \"jag\" Ginsberg" Date: Thu, 25 May 2023 12:24:55 -0400 Subject: [PATCH 21/27] PR review feedback. --- README.md | 8 ++------ piptools/utils.py | 17 ++++++----------- tests/conftest.py | 11 +++++++---- 3 files changed, 15 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 997f4d222..cfccd5714 100644 --- a/README.md +++ b/README.md @@ -283,7 +283,8 @@ $ pip-compile requirements.in --pip-args "--retries 10 --timeout 30" You can define project-level defaults for `pip-compile` and `pip-sync` by writing them to a configuration file in the same directory as your requirements -input file. By default, both `pip-compile` and `pip-sync` will look first +input files (or the current working directory if piping input from stdin). +By default, both `pip-compile` and `pip-sync` will look first for a `.pip-tools.toml` file and then in your `pyproject.toml`. You can also specify an alternate TOML configuration file with the `--config` option. @@ -291,11 +292,6 @@ For example, to by default generate `pip` hashes in the resulting requirements file output, you can specify in a configuration file ```toml -# In a .pip-tools.toml file -[pip-tools] -generate-hashes = true - -# In a pyproject.toml file [tool.pip-tools] generate-hashes = true ``` diff --git a/piptools/utils.py b/piptools/utils.py index df0ac2e8f..f0d7225f3 100644 --- a/piptools/utils.py +++ b/piptools/utils.py @@ -536,9 +536,9 @@ def parse_requirements_from_wheel_metadata( def determine_config_file( ctx: click.Context, param: click.Parameter, value: str | None ) -> Path | None: - """ - Returns the path to the config file with defaults being used, or `None` if no such file is - found. + """Return the config file path. + + ``None`` is returned if no such file is found. Defaults for `click.Command` parameters should be override-able in a config file. `pip-tools` will use the first file found, searching in this order: an explicitly given config file, a @@ -555,8 +555,6 @@ def determine_config_file( config = parse_config_file(config_file) if config: _assign_config_to_cli_context(ctx, config) - else: - return None return config_file @@ -625,7 +623,7 @@ def get_click_dest_for_option(option_name: str) -> str: ] -@functools.lru_cache() +@functools.lru_cache def parse_config_file(config_file: Path) -> dict[str, Any]: try: config = tomllib.loads(config_file.read_text(encoding="utf-8")) @@ -640,11 +638,8 @@ def parse_config_file(config_file: Path) -> dict[str, Any]: hint=f"Could not parse '{config_file !s}': {value_err !s}", ) - # In a pyproject.toml file, we expect the config to be under `[tool.pip-tools]`, but in our - # native configuration, it would be just `[pip-tools]`. - if config_file.name == "pyproject.toml": - config = config.get("tool", {}) - piptools_config: dict[str, Any] = config.get("pip-tools", {}) + # In a TOML file, we expect the config to be under `[tool.pip-tools]` + piptools_config: dict[str, Any] = config.get("tool", {}).get("pip-tools", {}) piptools_config = { get_click_dest_for_option(k): v for k, v in piptools_config.items() } diff --git a/tests/conftest.py b/tests/conftest.py index 46709e670..cd560ae24 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -10,6 +10,7 @@ from functools import partial from pathlib import Path from textwrap import dedent +from typing import Any import pytest import tomli_w @@ -457,15 +458,17 @@ def _reset_log(): @pytest.fixture def make_config_file(tmpdir_cwd): - def _maker(pyproject_param, new_default, config_file_name=CONFIG_FILE_NAME): + """ + Make a config file for pip-tools with a given parameter set to a specific + value, returning a `pathlib.Path` to the config file. + """ + def _maker(pyproject_param: str, new_default: Any, config_file_name: str = CONFIG_FILE_NAME) -> Path: # Make a config file with this one config default override config_path = Path(tmpdir_cwd) / pyproject_param config_file = config_path / config_file_name config_path.mkdir(exist_ok=True) - config_to_dump = {"pip-tools": {pyproject_param: new_default}} - if config_file_name == "pyproject.toml": - config_to_dump = {"tool": config_to_dump} + config_to_dump = {"tool": {"pip-tools": {pyproject_param: new_default}}} config_file.write_text(tomli_w.dumps(config_to_dump)) return config_file From bb015ad913e3f9748a6f31fa33a41542ce2019f2 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 25 May 2023 16:25:57 +0000 Subject: [PATCH 22/27] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/conftest.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index cd560ae24..9e33b2672 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -459,10 +459,13 @@ def _reset_log(): @pytest.fixture def make_config_file(tmpdir_cwd): """ - Make a config file for pip-tools with a given parameter set to a specific + Make a config file for pip-tools with a given parameter set to a specific value, returning a `pathlib.Path` to the config file. """ - def _maker(pyproject_param: str, new_default: Any, config_file_name: str = CONFIG_FILE_NAME) -> Path: + + def _maker( + pyproject_param: str, new_default: Any, config_file_name: str = CONFIG_FILE_NAME + ) -> Path: # Make a config file with this one config default override config_path = Path(tmpdir_cwd) / pyproject_param config_file = config_path / config_file_name From d01af78f116104b74355bbea78216d2e97db1376 Mon Sep 17 00:00:00 2001 From: "Joshua \"jag\" Ginsberg" Date: Thu, 25 May 2023 16:27:50 -0400 Subject: [PATCH 23/27] Improve test value for non-existant TOML file. Co-authored-by: Sviatoslav Sydorenko --- tests/test_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_utils.py b/tests/test_utils.py index 0f865bd26..f38a76c18 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -652,5 +652,5 @@ def test_callback_config_file_defaults_unreadable_toml(make_config_file): determine_config_file( ctx, "config", - "/path/does/not/exist/my-config.toml", + "/dev/null/path/does/not/exist/my-config.toml", ) From 97d30bccbf6a5f536c816b9c5af174d5319d068f Mon Sep 17 00:00:00 2001 From: "Joshua \"jag\" Ginsberg" Date: Wed, 31 May 2023 10:39:18 -0400 Subject: [PATCH 24/27] Docstring RST fixes; Py3.7 compat fix. --- piptools/utils.py | 17 ++++++++++------- tests/conftest.py | 2 +- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/piptools/utils.py b/piptools/utils.py index f0d7225f3..836c4335c 100644 --- a/piptools/utils.py +++ b/piptools/utils.py @@ -540,10 +540,11 @@ def determine_config_file( ``None`` is returned if no such file is found. - Defaults for `click.Command` parameters should be override-able in a config file. `pip-tools` - will use the first file found, searching in this order: an explicitly given config file, a - `.pip-tools.toml`, a `pyproject.toml` file. Those files are searched for in the same directory - as the requirements input file. + Defaults for ``click.Command`` parameters should be override-able in a config + file. ``pip-tools`` will use the first file found, searching in this order: + an explicitly given config file, a ``.pip-tools.toml``, a ``pyproject.toml`` + file. Those files are searched for in the same directory as the requirements + input file. """ if value is None: config_file = select_config_file(ctx.params.get("src_files", ())) @@ -571,7 +572,7 @@ def _assign_config_to_cli_context( def select_config_file(src_files: tuple[str, ...]) -> Path | None: """ - Returns the config file to use for defaults given `src_files` provided. + Returns the config file to use for defaults given ``src_files`` provided. """ # NOTE: If no src_files were specified, consider the current directory the # NOTE: only config file lookup candidate. This usually happens when a @@ -604,7 +605,9 @@ def select_config_file(src_files: tuple[str, ...]) -> Path | None: def get_click_dest_for_option(option_name: str) -> str: - """Returns the click `dest` value for the given option name.""" + """ + Returns the click ``dest`` value for the given option name. + """ # Format the keys properly option_name = option_name.lstrip("-").replace("-", "_").lower() # Some options have dest values that are overrides from the click generated default @@ -623,7 +626,7 @@ def get_click_dest_for_option(option_name: str) -> str: ] -@functools.lru_cache +@functools.lru_cache() def parse_config_file(config_file: Path) -> dict[str, Any]: try: config = tomllib.loads(config_file.read_text(encoding="utf-8")) diff --git a/tests/conftest.py b/tests/conftest.py index 9e33b2672..70f0e0e90 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -460,7 +460,7 @@ def _reset_log(): def make_config_file(tmpdir_cwd): """ Make a config file for pip-tools with a given parameter set to a specific - value, returning a `pathlib.Path` to the config file. + value, returning a ``pathlib.Path`` to the config file. """ def _maker( From 228f2a75554cc3198b5f80641b14fbb48aa58fa4 Mon Sep 17 00:00:00 2001 From: "Joshua \"jag\" Ginsberg" Date: Fri, 9 Jun 2023 15:22:27 -0400 Subject: [PATCH 25/27] Rename click callback fn name, clarify docstring. --- piptools/scripts/compile.py | 4 ++-- piptools/scripts/sync.py | 4 ++-- piptools/utils.py | 14 ++++++++------ tests/test_utils.py | 12 ++++++------ 4 files changed, 18 insertions(+), 16 deletions(-) diff --git a/piptools/scripts/compile.py b/piptools/scripts/compile.py index e26a689f9..df9fc49ab 100755 --- a/piptools/scripts/compile.py +++ b/piptools/scripts/compile.py @@ -28,7 +28,7 @@ from ..utils import ( UNSAFE_PACKAGES, dedup, - determine_config_file, + override_defaults_from_config_file, drop_extras, is_pinned_requirement, key_from_ireq, @@ -317,7 +317,7 @@ def _determine_linesep( help=f"Read configuration from TOML file. By default, looks for a {CONFIG_FILE_NAME} or " "pyproject.toml.", is_eager=True, - callback=determine_config_file, + callback=override_defaults_from_config_file, ) def cli( ctx: click.Context, diff --git a/piptools/scripts/sync.py b/piptools/scripts/sync.py index e88910924..51a886a62 100755 --- a/piptools/scripts/sync.py +++ b/piptools/scripts/sync.py @@ -22,7 +22,7 @@ from ..logging import log from ..repositories import PyPIRepository from ..utils import ( - determine_config_file, + override_defaults_from_config_file, flat_map, get_pip_version_for_python_executable, get_required_pip_specification, @@ -102,7 +102,7 @@ help=f"Read configuration from TOML file. By default, looks for a {CONFIG_FILE_NAME} or " "pyproject.toml.", is_eager=True, - callback=determine_config_file, + callback=override_defaults_from_config_file, ) def cli( ask: bool, diff --git a/piptools/utils.py b/piptools/utils.py index 836c4335c..4374b74a9 100644 --- a/piptools/utils.py +++ b/piptools/utils.py @@ -533,18 +533,20 @@ def parse_requirements_from_wheel_metadata( ) -def determine_config_file( +def override_defaults_from_config_file( ctx: click.Context, param: click.Parameter, value: str | None ) -> Path | None: - """Return the config file path. + """ + Overrides ``click.Command`` defaults based on specified or discovered config + file, returning the ``pathlib.Path`` of that config file if specified or + discovered. ``None`` is returned if no such file is found. - Defaults for ``click.Command`` parameters should be override-able in a config - file. ``pip-tools`` will use the first file found, searching in this order: - an explicitly given config file, a ``.pip-tools.toml``, a ``pyproject.toml`` + ``pip-tools`` will use the first config file found, searching in this order: + an explicitly given config file, a d``.pip-tools.toml``, a ``pyproject.toml`` file. Those files are searched for in the same directory as the requirements - input file. + input file, or the current working directory if requirements come via stdin. """ if value is None: config_file = select_config_file(ctx.params.get("src_files", ())) diff --git a/tests/test_utils.py b/tests/test_utils.py index f38a76c18..9d2937b17 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -16,7 +16,7 @@ from piptools.utils import ( as_tuple, dedup, - determine_config_file, + override_defaults_from_config_file, drop_extras, flat_map, format_requirement, @@ -596,7 +596,7 @@ def test_callback_config_file_defaults(pyproject_param, new_default, make_config # Create a "compile" run example pointing to the config file ctx = Context(compile_cli) ctx.params["src_files"] = (str(config_file),) - found_config_file = determine_config_file(ctx, "config", None) + found_config_file = override_defaults_from_config_file(ctx, "config", None) assert found_config_file == config_file # Make sure the default has been updated lookup_param = get_click_dest_for_option(pyproject_param) @@ -619,7 +619,7 @@ def test_callback_config_file_defaults_multi_value_options(mv_option, make_confi ctx = Context(compile_cli) ctx.params["src_files"] = (str(config_file),) with pytest.raises(BadOptionUsage, match="must be a list"): - determine_config_file(ctx, "config", None) + override_defaults_from_config_file(ctx, "config", None) def test_callback_config_file_defaults_bad_toml(make_config_file): @@ -631,7 +631,7 @@ def test_callback_config_file_defaults_bad_toml(make_config_file): ctx = Context(compile_cli) ctx.params["src_files"] = (str(config_file),) with pytest.raises(FileError, match="Could not parse "): - determine_config_file(ctx, "config", None) + override_defaults_from_config_file(ctx, "config", None) def test_callback_config_file_defaults_precedence(make_config_file): @@ -639,7 +639,7 @@ def test_callback_config_file_defaults_precedence(make_config_file): project_config_file = make_config_file("newline", "CRLF", "pyproject.toml") ctx = Context(compile_cli) ctx.params["src_files"] = (str(project_config_file),) - found_config_file = determine_config_file(ctx, "config", None) + found_config_file = override_defaults_from_config_file(ctx, "config", None) # The pip-tools specific config file should take precedence over pyproject.toml assert found_config_file == piptools_config_file lookup_param = get_click_dest_for_option("newline") @@ -649,7 +649,7 @@ def test_callback_config_file_defaults_precedence(make_config_file): def test_callback_config_file_defaults_unreadable_toml(make_config_file): ctx = Context(compile_cli) with pytest.raises(FileError, match="Could not read "): - determine_config_file( + override_defaults_from_config_file( ctx, "config", "/dev/null/path/does/not/exist/my-config.toml", From 8d9dd8a969ab9af3ecf1cdbf7103ee04ba748dbe Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 9 Jun 2023 19:22:57 +0000 Subject: [PATCH 26/27] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- piptools/scripts/compile.py | 2 +- piptools/scripts/sync.py | 2 +- tests/test_utils.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/piptools/scripts/compile.py b/piptools/scripts/compile.py index df9fc49ab..837bb0f23 100755 --- a/piptools/scripts/compile.py +++ b/piptools/scripts/compile.py @@ -28,10 +28,10 @@ from ..utils import ( UNSAFE_PACKAGES, dedup, - override_defaults_from_config_file, drop_extras, is_pinned_requirement, key_from_ireq, + override_defaults_from_config_file, parse_requirements_from_wheel_metadata, ) from ..writer import OutputWriter diff --git a/piptools/scripts/sync.py b/piptools/scripts/sync.py index 51a886a62..d504d889a 100755 --- a/piptools/scripts/sync.py +++ b/piptools/scripts/sync.py @@ -22,11 +22,11 @@ from ..logging import log from ..repositories import PyPIRepository from ..utils import ( - override_defaults_from_config_file, flat_map, get_pip_version_for_python_executable, get_required_pip_specification, get_sys_path_for_python_executable, + override_defaults_from_config_file, ) DEFAULT_REQUIREMENTS_FILE = "requirements.txt" diff --git a/tests/test_utils.py b/tests/test_utils.py index 9d2937b17..5f1e9ab5d 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -16,7 +16,6 @@ from piptools.utils import ( as_tuple, dedup, - override_defaults_from_config_file, drop_extras, flat_map, format_requirement, @@ -31,6 +30,7 @@ key_from_ireq, lookup_table, lookup_table_from_tuples, + override_defaults_from_config_file, ) From 8de18db39d3ec2b35957a14058790df60aa7b0e0 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 13 Jun 2023 02:36:02 +0000 Subject: [PATCH 27/27] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index cfccd5714..c6e831b99 100644 --- a/README.md +++ b/README.md @@ -283,7 +283,7 @@ $ pip-compile requirements.in --pip-args "--retries 10 --timeout 30" You can define project-level defaults for `pip-compile` and `pip-sync` by writing them to a configuration file in the same directory as your requirements -input files (or the current working directory if piping input from stdin). +input files (or the current working directory if piping input from stdin). By default, both `pip-compile` and `pip-sync` will look first for a `.pip-tools.toml` file and then in your `pyproject.toml`. You can also specify an alternate TOML configuration file with the `--config` option. @@ -294,6 +294,7 @@ requirements file output, you can specify in a configuration file ```toml [tool.pip-tools] generate-hashes = true + ``` Options to `pip-compile` and `pip-sync` that may be used more than once