From 8e8aa44ce79ea27cf886afe045615db6cd7ca50d Mon Sep 17 00:00:00 2001 From: David Hotham Date: Sun, 25 Sep 2022 13:55:57 +0100 Subject: [PATCH 01/25] normalized name when registering package at upload --- src/poetry/publishing/uploader.py | 12 +++++------- tests/helpers.py | 3 +-- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/src/poetry/publishing/uploader.py b/src/poetry/publishing/uploader.py index 4f57b370b8b..7b1e8e19048 100644 --- a/src/poetry/publishing/uploader.py +++ b/src/poetry/publishing/uploader.py @@ -77,13 +77,10 @@ def adapter(self) -> adapters.HTTPAdapter: def files(self) -> list[Path]: dist = self._poetry.file.parent / "dist" version = self._package.version.to_string() + escaped_name = distribution_name(self._package.name) - wheels = list( - dist.glob(f"{distribution_name(self._package.name)}-{version}-*.whl") - ) - tars = list( - dist.glob(f"{distribution_name(self._package.name)}-{version}.tar.gz") - ) + wheels = list(dist.glob(f"{escaped_name}-{version}-*.whl")) + tars = list(dist.glob(f"{escaped_name}-{version}.tar.gz")) return sorted(wheels + tars) @@ -303,7 +300,8 @@ def _register(self, session: requests.Session, url: str) -> requests.Response: Register a package to a repository. """ dist = self._poetry.file.parent / "dist" - file = dist / f"{self._package.name}-{self._package.version.to_string()}.tar.gz" + escaped_name = distribution_name(self._package.name) + file = dist / f"{escaped_name}-{self._package.version.to_string()}.tar.gz" if not file.exists(): raise RuntimeError(f'"{file.name}" does not exist.') diff --git a/tests/helpers.py b/tests/helpers.py index ca3a93b9ce6..2e57f1b2356 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -11,7 +11,6 @@ from typing import TYPE_CHECKING from typing import Any -from poetry.core.masonry.utils.helpers import distribution_name from poetry.core.packages.package import Package from poetry.core.packages.utils.link import Link from poetry.core.toml.file import TOMLFile @@ -235,7 +234,7 @@ def find_packages(self, dependency: Dependency) -> list[Package]: def find_links_for_package(self, package: Package) -> list[Link]: return [ Link( - f"https://foo.bar/files/{distribution_name(package.name)}" + f"https://foo.bar/files/{package.name.replace('-', '_')}" f"-{package.version.to_string()}-py2.py3-none-any.whl" ) ] From 96cdcdb192ea07b5e1b5f60167ac7f880a6a0400 Mon Sep 17 00:00:00 2001 From: David Hotham Date: Thu, 6 Oct 2022 19:38:11 +0100 Subject: [PATCH 02/25] pass --no-input to pip --- src/poetry/utils/pip.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/poetry/utils/pip.py b/src/poetry/utils/pip.py index 65414ac36b2..58b3504fba7 100644 --- a/src/poetry/utils/pip.py +++ b/src/poetry/utils/pip.py @@ -29,6 +29,7 @@ def pip_install( "install", "--disable-pip-version-check", "--isolated", + "--no-input", "--prefix", str(environment.path), ] From 4578c6f4be6b8f2eb92d43d6815fd9243d50a087 Mon Sep 17 00:00:00 2001 From: finswimmer Date: Fri, 7 Oct 2022 11:06:08 +0200 Subject: [PATCH 03/25] fix: turn debug message about invalid constraint into warning --- src/poetry/inspection/info.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/poetry/inspection/info.py b/src/poetry/inspection/info.py index 5347629c1ba..665fd641c84 100644 --- a/src/poetry/inspection/info.py +++ b/src/poetry/inspection/info.py @@ -205,7 +205,7 @@ def to_package( dependency = Dependency.create_from_pep_508(req, relative_to=root_dir) except ValueError: # Likely unable to parse constraint so we skip it - logger.debug( + logger.warning( "Invalid constraint (%s) found in %s-%s dependencies, skipping", req, package.name, From 0418424761190e80619714cf52c3e79f770a952f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Randy=20D=C3=B6ring?= <30527984+radoering@users.noreply.github.com> Date: Fri, 7 Oct 2022 19:11:08 +0200 Subject: [PATCH 04/25] locker: less verbose output for `package.files` in lockfile 2.0 --- src/poetry/packages/locker.py | 13 +++---------- tests/packages/test_locker.py | 19 +++++++------------ 2 files changed, 10 insertions(+), 22 deletions(-) diff --git a/src/poetry/packages/locker.py b/src/poetry/packages/locker.py index 7fa5fd92958..8a6c62e75f7 100644 --- a/src/poetry/packages/locker.py +++ b/src/poetry/packages/locker.py @@ -23,10 +23,8 @@ from tomlkit import comment from tomlkit import document from tomlkit import inline_table -from tomlkit import item from tomlkit import table from tomlkit.exceptions import TOMLKitError -from tomlkit.items import Array if TYPE_CHECKING: @@ -228,24 +226,19 @@ def locked_repository(self) -> LockfileRepository: return repository def set_lock_data(self, root: Package, packages: list[Package]) -> bool: - files: dict[str, Any] = table() package_specs = self._lock_packages(packages) # Retrieving hashes for package in package_specs: - if package["name"] not in files: - files[package["name"]] = [] + files = array() for f in package["files"]: file_metadata = inline_table() for k, v in sorted(f.items()): file_metadata[k] = v - files[package["name"]].append(file_metadata) + files.append(file_metadata) - if files[package["name"]]: - package_files = item(files[package["name"]]) - assert isinstance(package_files, Array) - files[package["name"]] = package_files.multiline(True) + package["files"] = files.multiline(True) lock = document() lock.add(comment(GENERATED_COMMENT)) diff --git a/tests/packages/test_locker.py b/tests/packages/test_locker.py index 7fb2772e6ba..9da6c422f7c 100644 --- a/tests/packages/test_locker.py +++ b/tests/packages/test_locker.py @@ -104,14 +104,10 @@ def test_lock_file_data_is_ordered(locker: Locker, root: ProjectPackage): category = "main" optional = false python-versions = "*" - -[[package.files]] -file = "bar" -hash = "123" - -[[package.files]] -file = "foo" -hash = "456" +files = [ + {{file = "bar", hash = "123"}}, + {{file = "foo", hash = "456"}}, +] [package.dependencies] B = "^1.0" @@ -123,10 +119,9 @@ def test_lock_file_data_is_ordered(locker: Locker, root: ProjectPackage): category = "main" optional = false python-versions = "*" - -[[package.files]] -file = "baz" -hash = "345" +files = [ + {{file = "baz", hash = "345"}}, +] [[package]] name = "B" From 770a8148c9ceb583053578af0480e815af01614f Mon Sep 17 00:00:00 2001 From: Brian Marroquin Date: Mon, 19 Sep 2022 00:21:23 -0700 Subject: [PATCH 05/25] adds fallback behavior for non-ms shells --- src/poetry/utils/shell.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/src/poetry/utils/shell.py b/src/poetry/utils/shell.py index 45d002fec85..7c9b23e58ee 100644 --- a/src/poetry/utils/shell.py +++ b/src/poetry/utils/shell.py @@ -78,12 +78,22 @@ def activate(self, env: VirtualEnv) -> int | None: if sys.platform == "win32": if self._name in ("powershell", "pwsh"): args = ["-NoExit", "-File", str(activate_path)] - else: + elif self._name == "cmd": # /K will execute the bat file and # keep the cmd process from terminating args = ["/K", str(activate_path)] - completed_proc = subprocess.run([self.path, *args]) - return completed_proc.returncode + else: + args = None + + if args: + completed_proc = subprocess.run([self.path, *args]) + return completed_proc.returncode + else: + # If no args are set, execute the shell within the venv + # This activates it, but there could be some features missing: + # deactivate command might not work + # shell prompt will not be modified. + return env.execute(self._path) import shlex From 3d422d72159418ca23105fb6ea38f86af62af23f Mon Sep 17 00:00:00 2001 From: Brian Marroquin Date: Mon, 19 Sep 2022 02:15:45 -0700 Subject: [PATCH 06/25] hoists args out of else block --- src/poetry/utils/shell.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/poetry/utils/shell.py b/src/poetry/utils/shell.py index 7c9b23e58ee..d5b23288540 100644 --- a/src/poetry/utils/shell.py +++ b/src/poetry/utils/shell.py @@ -76,14 +76,13 @@ def activate(self, env: VirtualEnv) -> int | None: # mypy requires using sys.platform instead of WINDOWS constant # in if statements to properly type check on Windows if sys.platform == "win32": + args = None if self._name in ("powershell", "pwsh"): args = ["-NoExit", "-File", str(activate_path)] elif self._name == "cmd": # /K will execute the bat file and # keep the cmd process from terminating args = ["/K", str(activate_path)] - else: - args = None if args: completed_proc = subprocess.run([self.path, *args]) From 8a500f053b756616be99454939bcbf7ad42c7e3b Mon Sep 17 00:00:00 2001 From: arl Date: Fri, 7 Oct 2022 19:31:43 -0400 Subject: [PATCH 07/25] docs(plugins): change the example's minimum poetry version to 1.2 this is the first poetry version with plugin support, so plugins cannot be compatible with any version before 1.2. --- docs/plugins.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/plugins.md b/docs/plugins.md index 7fc8bfe081b..1db8c065f0f 100644 --- a/docs/plugins.md +++ b/docs/plugins.md @@ -39,7 +39,7 @@ version = "1.0.0" # ... [tool.poetry.dependencies] python = "^3.7" -poetry = "^1.0" +poetry = "^1.2" [tool.poetry.plugins."poetry.plugin"] demo = "poetry_demo_plugin.plugin:MyPlugin" From 6ae4855d7ca85ae57dc635cdc0e2e2368e348cf8 Mon Sep 17 00:00:00 2001 From: Bart Kamphorst Date: Sun, 9 Oct 2022 20:37:56 -0600 Subject: [PATCH 08/25] tests: 100% coverage for test_repository --- tests/repositories/test_pool.py | 157 +++++++++++++++++++++++--- tests/repositories/test_repository.py | 21 +++- 2 files changed, 156 insertions(+), 22 deletions(-) diff --git a/tests/repositories/test_pool.py b/tests/repositories/test_pool.py index 84f18c4ea56..9d660ad7b46 100644 --- a/tests/repositories/test_pool.py +++ b/tests/repositories/test_pool.py @@ -8,39 +8,46 @@ from poetry.repositories import Repository from poetry.repositories.exceptions import PackageNotFound from poetry.repositories.legacy_repository import LegacyRepository +from tests.helpers import get_dependency +from tests.helpers import get_package -def test_pool_raises_package_not_found_when_no_package_is_found() -> None: - pool = Pool() - pool.add_repository(Repository("repo")) - - with pytest.raises(PackageNotFound): - pool.package("foo", Version.parse("1.0.0")) - - -def test_pool(): +def test_pool() -> None: pool = Pool() assert len(pool.repositories) == 0 assert not pool.has_default() + assert not pool.has_primary_repositories() -def test_pool_with_initial_repositories(): +def test_pool_with_initial_repositories() -> None: repo = Repository("repo") pool = Pool([repo]) assert len(pool.repositories) == 1 assert not pool.has_default() + assert pool.has_primary_repositories() -def test_repository_no_repository(): +def test_repository_no_repository() -> None: pool = Pool() with pytest.raises(ValueError): pool.repository("foo") -def test_repository_from_normal_pool(): +def test_adding_repositories_with_same_name_twice_raises_value_error() -> None: + repo1 = Repository("repo") + repo2 = Repository("repo") + + with pytest.raises(ValueError): + Pool([repo1, repo2]) + + with pytest.raises(ValueError): + Pool([repo1]).add_repository(repo2) + + +def test_repository_from_normal_pool() -> None: repo = LegacyRepository("foo", "https://foo.bar") pool = Pool() pool.add_repository(repo) @@ -48,7 +55,7 @@ def test_repository_from_normal_pool(): assert pool.repository("foo") is repo -def test_repository_from_secondary_pool(): +def test_repository_from_secondary_pool() -> None: repo = LegacyRepository("foo", "https://foo.bar") pool = Pool() pool.add_repository(repo, secondary=True) @@ -56,7 +63,7 @@ def test_repository_from_secondary_pool(): assert pool.repository("foo") is repo -def test_repository_with_normal_default_and_secondary_repositories(): +def test_repository_with_normal_default_and_secondary_repositories() -> None: secondary = LegacyRepository("secondary", "https://secondary.com") default = LegacyRepository("default", "https://default.com") repo1 = LegacyRepository("foo", "https://foo.bar") @@ -73,9 +80,17 @@ def test_repository_with_normal_default_and_secondary_repositories(): assert pool.repository("foo") is repo1 assert pool.repository("bar") is repo2 assert pool.has_default() + assert pool.has_primary_repositories() -def test_remove_repository(): +def test_remove_non_existing_repository_raises_indexerror() -> None: + pool = Pool() + + with pytest.raises(IndexError): + pool.remove_repository("foo") + + +def test_remove_existing_repository_successful() -> None: repo1 = LegacyRepository("foo", "https://foo.bar") repo2 = LegacyRepository("bar", "https://bar.baz") repo3 = LegacyRepository("baz", "https://baz.quux") @@ -91,7 +106,7 @@ def test_remove_repository(): assert pool.repository("baz") is repo3 -def test_remove_default_repository(): +def test_remove_default_repository() -> None: default = LegacyRepository("default", "https://default.com") repo1 = LegacyRepository("foo", "https://foo.bar") repo2 = LegacyRepository("bar", "https://bar.baz") @@ -115,7 +130,7 @@ def test_remove_default_repository(): assert not pool.has_repository("default") -def test_repository_ordering(): +def test_repository_ordering() -> None: default1 = LegacyRepository("default1", "https://default1.com") default2 = LegacyRepository("default2", "https://default2.com") primary1 = LegacyRepository("primary1", "https://primary1.com") @@ -141,3 +156,111 @@ def test_repository_ordering(): assert pool.repositories == [default1, primary1, primary3, secondary1, secondary3] with pytest.raises(ValueError): pool.add_repository(default2, default=True) + + +def test_pool_get_package_in_any_repository() -> None: + package1 = get_package("foo", "1.0.0") + repo1 = Repository("repo1", [package1]) + package2 = get_package("bar", "1.0.0") + repo2 = Repository("repo2", [package1, package2]) + pool = Pool([repo1, repo2]) + + returned_package1 = pool.package("foo", Version.parse("1.0.0")) + returned_package2 = pool.package("bar", Version.parse("1.0.0")) + + assert returned_package1 == package1 + assert returned_package2 == package2 + + +def test_pool_get_package_in_specified_repository() -> None: + package = get_package("foo", "1.0.0") + repo1 = Repository("repo1") + repo2 = Repository("repo2", [package]) + pool = Pool([repo1, repo2]) + + returned_package = pool.package("foo", Version.parse("1.0.0"), repository="repo2") + + assert returned_package == package + + +def test_pool_no_package_from_any_repository_raises_package_not_found() -> None: + pool = Pool() + pool.add_repository(Repository("repo")) + + with pytest.raises(PackageNotFound): + pool.package("foo", Version.parse("1.0.0")) + + +def test_pool_no_package_from_specified_repository_raises_package_not_found() -> None: + package = get_package("foo", "1.0.0") + repo1 = Repository("repo1") + repo2 = Repository("repo2", [package]) + pool = Pool([repo1, repo2]) + + with pytest.raises(PackageNotFound): + pool.package("foo", Version.parse("1.0.0"), repository="repo1") + + +def test_pool_find_packages_in_any_repository() -> None: + package1 = get_package("foo", "1.1.1") + package2 = get_package("foo", "1.2.3") + package3 = get_package("foo", "2.0.0") + package4 = get_package("bar", "1.2.3") + repo1 = Repository("repo1", [package1, package3]) + repo2 = Repository("repo2", [package1, package2, package4]) + pool = Pool([repo1, repo2]) + + available_dependency = get_dependency("foo", "^1.0.0") + returned_packages_available = pool.find_packages(available_dependency) + unavailable_dependency = get_dependency("foo", "999.9.9") + returned_packages_unavailable = pool.find_packages(unavailable_dependency) + + assert returned_packages_available == [package1, package1, package2] + assert returned_packages_unavailable == [] + + +def test_pool_find_packages_in_specified_repository() -> None: + package_foo1 = get_package("foo", "1.1.1") + package_foo2 = get_package("foo", "1.2.3") + package_foo3 = get_package("foo", "2.0.0") + package_bar = get_package("bar", "1.2.3") + repo1 = Repository("repo1", [package_foo1, package_foo3]) + repo2 = Repository("repo2", [package_foo1, package_foo2, package_bar]) + pool = Pool([repo1, repo2]) + + available_dependency = get_dependency("foo", "^1.0.0") + available_dependency.source_name = "repo2" + returned_packages_available = pool.find_packages(available_dependency) + unavailable_dependency = get_dependency("foo", "999.9.9") + unavailable_dependency.source_name = "repo2" + returned_packages_unavailable = pool.find_packages(unavailable_dependency) + + assert returned_packages_available == [package_foo1, package_foo2] + assert returned_packages_unavailable == [] + + +def test_search_no_legacy_repositories() -> None: + package_foo1 = get_package("foo", "1.0.0") + package_foo2 = get_package("foo", "2.0.0") + package_foobar = get_package("foobar", "1.0.0") + repo1 = Repository("repo1", [package_foo1, package_foo2]) + repo2 = Repository("repo2", [package_foo1, package_foobar]) + pool = Pool([repo1, repo2]) + + assert pool.search("foo") == [ + package_foo1, + package_foo2, + package_foo1, + package_foobar, + ] + assert pool.search("bar") == [package_foobar] + assert pool.search("nothing") == [] + + +def test_search_legacy_repositories_are_skipped() -> None: + package = get_package("foo", "1.0.0") + repo1 = Repository("repo1", [package]) + repo2 = LegacyRepository("repo2", "https://fake.repo/") + pool = Pool([repo1, repo2]) + + assert pool.search("foo") == [package] diff --git a/tests/repositories/test_repository.py b/tests/repositories/test_repository.py index d2ff270b10a..83a7287cce4 100644 --- a/tests/repositories/test_repository.py +++ b/tests/repositories/test_repository.py @@ -2,18 +2,18 @@ import pytest -from poetry.core.constraints.version import Version -from poetry.core.packages.package import Package +from poetry.core.semver.version import Version from poetry.factory import Factory from poetry.repositories import Repository +from tests.helpers import get_package @pytest.fixture(scope="module") def black_repository() -> Repository: repo = Repository("repo") - repo.add_package(Package("black", "19.10b0")) - repo.add_package(Package("black", "21.11b0", yanked="reason")) + repo.add_package(get_package("black", "19.10b0")) + repo.add_package(get_package("black", "21.11b0", yanked="reason")) return repo @@ -63,7 +63,18 @@ def test_package_yanked( def test_package_pretty_name_is_kept() -> None: pretty_name = "Not_canoni-calized.name" repo = Repository("repo") - repo.add_package(Package(pretty_name, "1.0")) + repo.add_package(get_package(pretty_name, "1.0")) package = repo.package(pretty_name, Version.parse("1.0")) assert package.pretty_name == pretty_name + + +def test_search() -> None: + package_foo1 = get_package("foo", "1.0.0") + package_foo2 = get_package("foo", "2.0.0") + package_foobar = get_package("foobar", "1.0.0") + repo = Repository("repo", [package_foo1, package_foo2, package_foobar]) + + assert repo.search("foo") == [package_foo1, package_foo2, package_foobar] + assert repo.search("bar") == [package_foobar] + assert repo.search("nothing") == [] From 2d2ee7ef905baff476f755d3e8a8511e72586779 Mon Sep 17 00:00:00 2001 From: Bart Kamphorst Date: Fri, 30 Sep 2022 23:26:39 +0200 Subject: [PATCH 09/25] refactor: Pool should not inherit from Repository --- src/poetry/repositories/pool.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/poetry/repositories/pool.py b/src/poetry/repositories/pool.py index 43f1e9d2a06..3dfacd66b9e 100644 --- a/src/poetry/repositories/pool.py +++ b/src/poetry/repositories/pool.py @@ -3,7 +3,6 @@ from typing import TYPE_CHECKING from poetry.repositories.exceptions import PackageNotFound -from poetry.repositories.repository import Repository if TYPE_CHECKING: @@ -11,14 +10,16 @@ from poetry.core.packages.dependency import Dependency from poetry.core.packages.package import Package + from poetry.repositories.repository import Repository -class Pool(Repository): + +class Pool: def __init__( self, repositories: list[Repository] | None = None, ignore_repository_names: bool = False, ) -> None: - super().__init__("poetry-pool") + self._name = "poetry-pool" if repositories is None: repositories = [] @@ -34,6 +35,10 @@ def __init__( self._ignore_repository_names = ignore_repository_names + @property + def name(self) -> str: + return self._name + @property def repositories(self) -> list[Repository]: return self._repositories @@ -128,9 +133,6 @@ def remove_repository(self, repository_name: str) -> Pool: return self - def has_package(self, package: Package) -> bool: - raise NotImplementedError() - def package( self, name: str, From 46ae4f542ef2be7ef9755aea22d2a055469c7015 Mon Sep 17 00:00:00 2001 From: Bart Kamphorst Date: Fri, 30 Sep 2022 23:39:20 +0200 Subject: [PATCH 10/25] refactor: simplify Pool logic Introduces Priority enum to help in bookkeeping of repository types. Additionally specifies parameter 'repository' to 'repository_name' where relevant. --- src/poetry/puzzle/provider.py | 2 +- src/poetry/repositories/legacy_repository.py | 12 ++ src/poetry/repositories/pool.py | 180 +++++++------------ tests/console/commands/test_add.py | 4 +- tests/repositories/test_legacy_repository.py | 7 + tests/repositories/test_pool.py | 8 +- tests/test_factory.py | 2 +- 7 files changed, 88 insertions(+), 127 deletions(-) diff --git a/src/poetry/puzzle/provider.py b/src/poetry/puzzle/provider.py index baf04dc60ef..1e634995dd7 100644 --- a/src/poetry/puzzle/provider.py +++ b/src/poetry/puzzle/provider.py @@ -555,7 +555,7 @@ def complete_package( package.pretty_name, package.version, extras=list(dependency.extras), - repository=dependency.source_name, + repository_name=dependency.source_name, ), ) except PackageNotFound as e: diff --git a/src/poetry/repositories/legacy_repository.py b/src/poetry/repositories/legacy_repository.py index bc20b98d9a6..4823a0b71fd 100644 --- a/src/poetry/repositories/legacy_repository.py +++ b/src/poetry/repositories/legacy_repository.py @@ -33,6 +33,18 @@ def __init__( super().__init__(name, url.rstrip("/"), config, disable_cache) + @property + def packages(self) -> list[Package]: + # LegacyRepository._packages is not populated and other implementations + # implicitly rely on this (e.g. Pool.search via + # LegacyRepository.search). To avoid special-casing Pool or changing + # behavior, we stub and return an empty list. + # + # TODO: Rethinking search behaviour and design. + # Ref: https://github.com/python-poetry/poetry/issues/2446 and + # https://github.com/python-poetry/poetry/pull/6669#discussion_r990874908. + return [] + def package( self, name: str, version: Version, extras: list[str] | None = None ) -> Package: diff --git a/src/poetry/repositories/pool.py b/src/poetry/repositories/pool.py index 3dfacd66b9e..419d8990c2d 100644 --- a/src/poetry/repositories/pool.py +++ b/src/poetry/repositories/pool.py @@ -1,5 +1,9 @@ from __future__ import annotations +import enum + +from collections import OrderedDict +from enum import IntEnum from typing import TYPE_CHECKING from poetry.repositories.exceptions import PackageNotFound @@ -13,6 +17,14 @@ from poetry.repositories.repository import Repository +class Priority(IntEnum): + # The order of the members below dictates the actual priority. The first member has + # top priority. + DEFAULT = enum.auto() + PRIMARY = enum.auto() + SECONDARY = enum.auto() + + class Pool: def __init__( self, @@ -20,46 +32,45 @@ def __init__( ignore_repository_names: bool = False, ) -> None: self._name = "poetry-pool" + self._repositories: OrderedDict[ + str, tuple[Repository, RepositoryPriority] + ] = OrderedDict() + self._ignore_repository_names = ignore_repository_names if repositories is None: repositories = [] - - self._lookup: dict[str, int] = {} - self._repositories: list[Repository] = [] - self._default = False - self._has_primary_repositories = False - self._secondary_start_idx: int | None = None - for repository in repositories: self.add_repository(repository) - self._ignore_repository_names = ignore_repository_names - @property def name(self) -> str: return self._name @property def repositories(self) -> list[Repository]: - return self._repositories + unsorted_repositories = self._repositories.values() + sorted_repositories = sorted(unsorted_repositories, key=lambda p: p[1].value) + return [repo for (repo, _) in sorted_repositories] def has_default(self) -> bool: - return self._default + return self._contains_priority(Priority.DEFAULT) def has_primary_repositories(self) -> bool: - return self._has_primary_repositories + return self._contains_priority(Priority.PRIMARY) + + def _contains_priority(self, priority: Priority) -> bool: + return any( + repo_prio is priority for _, repo_prio in self._repositories.values() + ) def has_repository(self, name: str) -> bool: - return name.lower() in self._lookup + return name.lower() in self._repositories def repository(self, name: str) -> Repository: name = name.lower() - - lookup = self._lookup.get(name) - if lookup is not None: - return self._repositories[lookup] - - raise ValueError(f'Repository "{name}" does not exist.') + if self.has_repository(name): + return self._repositories[name][0] + raise IndexError(f'Repository "{name}" does not exist.') def add_repository( self, repository: Repository, default: bool = False, secondary: bool = False @@ -68,69 +79,26 @@ def add_repository( Adds a repository to the pool. """ repository_name = repository.name.lower() - if repository_name in self._lookup: - raise ValueError(f"{repository_name} already added") - - if default: - if self.has_default(): - raise ValueError("Only one repository can be the default") - - self._default = True - self._repositories.insert(0, repository) - for name in self._lookup: - self._lookup[name] += 1 + if self.has_repository(repository_name): + raise ValueError( + f"A repository with name {repository_name} was already added." + ) - if self._secondary_start_idx is not None: - self._secondary_start_idx += 1 + if default and self.has_default(): + raise ValueError("Only one repository can be the default.") - self._lookup[repository_name] = 0 + priority = Priority.PRIMARY + if default: + priority = Priority.DEFAULT elif secondary: - if self._secondary_start_idx is None: - self._secondary_start_idx = len(self._repositories) - - self._repositories.append(repository) - self._lookup[repository_name] = len(self._repositories) - 1 - else: - self._has_primary_repositories = True - if self._secondary_start_idx is None: - self._repositories.append(repository) - self._lookup[repository_name] = len(self._repositories) - 1 - else: - self._repositories.insert(self._secondary_start_idx, repository) - - for name, idx in self._lookup.items(): - if idx < self._secondary_start_idx: - continue - - self._lookup[name] += 1 - - self._lookup[repository_name] = self._secondary_start_idx - self._secondary_start_idx += 1 - + priority = Priority.SECONDARY + self._repositories[repository_name] = (repository, priority) return self - def remove_repository(self, repository_name: str) -> Pool: - if repository_name is not None: - repository_name = repository_name.lower() - - idx = self._lookup.get(repository_name) - if idx is not None: - del self._repositories[idx] - del self._lookup[repository_name] - - if idx == 0: - self._default = False - - for name in self._lookup: - if self._lookup[name] > idx: - self._lookup[name] -= 1 - - if ( - self._secondary_start_idx is not None - and self._secondary_start_idx > idx - ): - self._secondary_start_idx -= 1 - + def remove_repository(self, name: str) -> Pool: + if not self.has_repository(name): + raise IndexError(f"Pool can not remove unknown repository '{name}'.") + del self._repositories[name.lower()] return self def package( @@ -138,60 +106,32 @@ def package( name: str, version: Version, extras: list[str] | None = None, - repository: str | None = None, + repository_name: str | None = None, ) -> Package: - if repository is not None: - repository = repository.lower() - - if ( - repository is not None - and repository not in self._lookup - and not self._ignore_repository_names - ): - raise ValueError(f'Repository "{repository}" does not exist.') - - if repository is not None and not self._ignore_repository_names: - return self.repository(repository).package(name, version, extras=extras) + if repository_name and not self._ignore_repository_names: + return self.repository(repository_name).package( + name, version, extras=extras + ) - for repo in self._repositories: + for repo in self.repositories: try: - package = repo.package(name, version, extras=extras) + return repo.package(name, version, extras=extras) except PackageNotFound: continue - - return package - raise PackageNotFound(f"Package {name} ({version}) not found.") def find_packages(self, dependency: Dependency) -> list[Package]: - repository = dependency.source_name - if repository is not None: - repository = repository.lower() - - if ( - repository is not None - and repository not in self._lookup - and not self._ignore_repository_names - ): - raise ValueError(f'Repository "{repository}" does not exist.') - - if repository is not None and not self._ignore_repository_names: - return self.repository(repository).find_packages(dependency) - - packages = [] - for repo in self._repositories: - packages += repo.find_packages(dependency) + repository_name = dependency.source_name + if repository_name and not self._ignore_repository_names: + return self.repository(repository_name).find_packages(dependency) + packages: list[Package] = [] + for repo in self.repositories: + packages += repo.find_packages(dependency) return packages def search(self, query: str) -> list[Package]: - from poetry.repositories.legacy_repository import LegacyRepository - - results = [] - for repository in self._repositories: - if isinstance(repository, LegacyRepository): - continue - + results: list[Package] = [] + for repository in self.repositories: results += repository.search(query) - return results diff --git a/tests/console/commands/test_add.py b/tests/console/commands/test_add.py index fedaacc020e..7947d209d8c 100644 --- a/tests/console/commands/test_add.py +++ b/tests/console/commands/test_add.py @@ -858,7 +858,7 @@ def test_add_constraint_with_source( def test_add_constraint_with_source_that_does_not_exist( app: PoetryTestApplication, tester: CommandTester ): - with pytest.raises(ValueError) as e: + with pytest.raises(IndexError) as e: tester.execute("foo --source i-dont-exist") assert str(e.value) == 'Repository "i-dont-exist" does not exist.' @@ -1848,7 +1848,7 @@ def test_add_constraint_with_source_old_installer( def test_add_constraint_with_source_that_does_not_exist_old_installer( app: PoetryTestApplication, old_tester: CommandTester ): - with pytest.raises(ValueError) as e: + with pytest.raises(IndexError) as e: old_tester.execute("foo --source i-dont-exist") assert str(e.value) == 'Repository "i-dont-exist" does not exist.' diff --git a/tests/repositories/test_legacy_repository.py b/tests/repositories/test_legacy_repository.py index fb96c65b1cd..ab8f7022654 100644 --- a/tests/repositories/test_legacy_repository.py +++ b/tests/repositories/test_legacy_repository.py @@ -63,6 +63,13 @@ def _download(self, url: str, dest: Path) -> None: shutil.copyfile(str(filepath), dest) +def test_packages_property_returns_empty_list() -> None: + repo = MockRepository() + repo._packages = [repo.package("jupyter", Version.parse("1.0.0"))] + + assert repo.packages == [] + + def test_page_relative_links_path_are_correct() -> None: repo = MockRepository() diff --git a/tests/repositories/test_pool.py b/tests/repositories/test_pool.py index 9d660ad7b46..dc68bf0015b 100644 --- a/tests/repositories/test_pool.py +++ b/tests/repositories/test_pool.py @@ -32,7 +32,7 @@ def test_pool_with_initial_repositories() -> None: def test_repository_no_repository() -> None: pool = Pool() - with pytest.raises(ValueError): + with pytest.raises(IndexError): pool.repository("foo") @@ -178,7 +178,9 @@ def test_pool_get_package_in_specified_repository() -> None: repo2 = Repository("repo2", [package]) pool = Pool([repo1, repo2]) - returned_package = pool.package("foo", Version.parse("1.0.0"), repository="repo2") + returned_package = pool.package( + "foo", Version.parse("1.0.0"), repository_name="repo2" + ) assert returned_package == package @@ -198,7 +200,7 @@ def test_pool_no_package_from_specified_repository_raises_package_not_found() -> pool = Pool([repo1, repo2]) with pytest.raises(PackageNotFound): - pool.package("foo", Version.parse("1.0.0"), repository="repo1") + pool.package("foo", Version.parse("1.0.0"), repository_name="repo1") def test_pool_find_packages_in_any_repository() -> None: diff --git a/tests/test_factory.py b/tests/test_factory.py index 5671fdc5edf..7eafc85d210 100644 --- a/tests/test_factory.py +++ b/tests/test_factory.py @@ -299,7 +299,7 @@ def test_poetry_with_two_default_sources(with_simple_keyring: None): with pytest.raises(ValueError) as e: Factory().create_poetry(fixtures_dir / "with_two_default_sources") - assert str(e.value) == "Only one repository can be the default" + assert str(e.value) == "Only one repository can be the default." def test_validate(): From df90acb62dc5f98233b7002448f97dedd294b7bb Mon Sep 17 00:00:00 2001 From: Bart Kamphorst Date: Sat, 1 Oct 2022 12:23:06 +0200 Subject: [PATCH 11/25] refactor: introduce AbstractRepository --- .../repositories/abstract_repository.py | 37 +++++++++++++++++++ src/poetry/repositories/cached.py | 2 +- src/poetry/repositories/http.py | 3 +- src/poetry/repositories/pool.py | 9 ++--- src/poetry/repositories/repository.py | 9 ++--- 5 files changed, 45 insertions(+), 15 deletions(-) create mode 100644 src/poetry/repositories/abstract_repository.py diff --git a/src/poetry/repositories/abstract_repository.py b/src/poetry/repositories/abstract_repository.py new file mode 100644 index 00000000000..6725aca279c --- /dev/null +++ b/src/poetry/repositories/abstract_repository.py @@ -0,0 +1,37 @@ +from __future__ import annotations + +from abc import ABC +from abc import abstractmethod +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from poetry.core.packages.dependency import Dependency + from poetry.core.packages.package import Package + from poetry.core.semver.version import Version + + +class AbstractRepository(ABC): + def __init__(self, name: str) -> None: + self._name = name + + @property + def name(self) -> str: + return self._name + + @abstractmethod + def find_packages(self, dependency: Dependency) -> list[Package]: + ... + + @abstractmethod + def search(self, query: str) -> list[Package]: + ... + + @abstractmethod + def package( + self, + name: str, + version: Version, + extras: list[str] | None = None, + ) -> Package: + ... diff --git a/src/poetry/repositories/cached.py b/src/poetry/repositories/cached.py index d3c6064d00e..59d6ba5fb71 100644 --- a/src/poetry/repositories/cached.py +++ b/src/poetry/repositories/cached.py @@ -46,7 +46,7 @@ def __init__( def _get_release_info( self, name: NormalizedName, version: Version ) -> dict[str, Any]: - raise NotImplementedError() + ... def get_release_info(self, name: NormalizedName, version: Version) -> PackageInfo: """ diff --git a/src/poetry/repositories/http.py b/src/poetry/repositories/http.py index 16bac1e4c1b..07315736d5a 100644 --- a/src/poetry/repositories/http.py +++ b/src/poetry/repositories/http.py @@ -5,7 +5,6 @@ import urllib import urllib.parse -from abc import ABC from collections import defaultdict from pathlib import Path from typing import TYPE_CHECKING @@ -35,7 +34,7 @@ from poetry.utils.authenticator import RepositoryCertificateConfig -class HTTPRepository(CachedRepository, ABC): +class HTTPRepository(CachedRepository): def __init__( self, name: str, diff --git a/src/poetry/repositories/pool.py b/src/poetry/repositories/pool.py index 419d8990c2d..32340329bdc 100644 --- a/src/poetry/repositories/pool.py +++ b/src/poetry/repositories/pool.py @@ -6,6 +6,7 @@ from enum import IntEnum from typing import TYPE_CHECKING +from poetry.repositories.abstract_repository import AbstractRepository from poetry.repositories.exceptions import PackageNotFound @@ -25,13 +26,13 @@ class Priority(IntEnum): SECONDARY = enum.auto() -class Pool: +class Pool(AbstractRepository): def __init__( self, repositories: list[Repository] | None = None, ignore_repository_names: bool = False, ) -> None: - self._name = "poetry-pool" + super().__init__("poetry-pool") self._repositories: OrderedDict[ str, tuple[Repository, RepositoryPriority] ] = OrderedDict() @@ -42,10 +43,6 @@ def __init__( for repository in repositories: self.add_repository(repository) - @property - def name(self) -> str: - return self._name - @property def repositories(self) -> list[Repository]: unsorted_repositories = self._repositories.values() diff --git a/src/poetry/repositories/repository.py b/src/poetry/repositories/repository.py index ee54362af55..46ecaac7475 100644 --- a/src/poetry/repositories/repository.py +++ b/src/poetry/repositories/repository.py @@ -8,6 +8,7 @@ from poetry.core.constraints.version import Version from poetry.core.constraints.version import VersionRange +from poetry.repositories.abstract_repository import AbstractRepository from poetry.repositories.exceptions import PackageNotFound @@ -19,18 +20,14 @@ from poetry.core.packages.utils.link import Link -class Repository: +class Repository(AbstractRepository): def __init__(self, name: str, packages: list[Package] | None = None) -> None: - self._name = name + super().__init__(name) self._packages: list[Package] = [] for package in packages or []: self.add_package(package) - @property - def name(self) -> str: - return self._name - @property def packages(self) -> list[Package]: return self._packages From b144f02d6c41a5faa7c9e4228e73ce0c9f7863a8 Mon Sep 17 00:00:00 2001 From: Bart Kamphorst Date: Mon, 3 Oct 2022 22:51:45 +0200 Subject: [PATCH 12/25] refactor: introduce PrioritizedRepository --- src/poetry/repositories/pool.py | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/src/poetry/repositories/pool.py b/src/poetry/repositories/pool.py index 32340329bdc..8a345b16cfc 100644 --- a/src/poetry/repositories/pool.py +++ b/src/poetry/repositories/pool.py @@ -3,6 +3,7 @@ import enum from collections import OrderedDict +from dataclasses import dataclass from enum import IntEnum from typing import TYPE_CHECKING @@ -26,6 +27,12 @@ class Priority(IntEnum): SECONDARY = enum.auto() +@dataclass(frozen=True) +class PrioritizedRepository: + repository: Repository + priority: Priority + + class Pool(AbstractRepository): def __init__( self, @@ -33,9 +40,7 @@ def __init__( ignore_repository_names: bool = False, ) -> None: super().__init__("poetry-pool") - self._repositories: OrderedDict[ - str, tuple[Repository, RepositoryPriority] - ] = OrderedDict() + self._repositories: OrderedDict[str, PrioritizedRepository] = OrderedDict() self._ignore_repository_names = ignore_repository_names if repositories is None: @@ -46,8 +51,10 @@ def __init__( @property def repositories(self) -> list[Repository]: unsorted_repositories = self._repositories.values() - sorted_repositories = sorted(unsorted_repositories, key=lambda p: p[1].value) - return [repo for (repo, _) in sorted_repositories] + sorted_repositories = sorted( + unsorted_repositories, key=lambda prio_repo: prio_repo.priority + ) + return [prio_repo.repository for prio_repo in sorted_repositories] def has_default(self) -> bool: return self._contains_priority(Priority.DEFAULT) @@ -57,7 +64,7 @@ def has_primary_repositories(self) -> bool: def _contains_priority(self, priority: Priority) -> bool: return any( - repo_prio is priority for _, repo_prio in self._repositories.values() + prio_repo.priority is priority for prio_repo in self._repositories.values() ) def has_repository(self, name: str) -> bool: @@ -66,7 +73,7 @@ def has_repository(self, name: str) -> bool: def repository(self, name: str) -> Repository: name = name.lower() if self.has_repository(name): - return self._repositories[name][0] + return self._repositories[name].repository raise IndexError(f'Repository "{name}" does not exist.') def add_repository( @@ -89,7 +96,9 @@ def add_repository( priority = Priority.DEFAULT elif secondary: priority = Priority.SECONDARY - self._repositories[repository_name] = (repository, priority) + self._repositories[repository_name] = PrioritizedRepository( + repository, priority + ) return self def remove_repository(self, name: str) -> Pool: From 94a5ce4070df4b81ee7fc4a091a00b96bd918909 Mon Sep 17 00:00:00 2001 From: Bjorn Neergaard Date: Sun, 9 Oct 2022 20:07:57 -0600 Subject: [PATCH 13/25] chore: adjust import paths for poetry-core deprecations --- src/poetry/repositories/abstract_repository.py | 2 +- tests/repositories/test_repository.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/poetry/repositories/abstract_repository.py b/src/poetry/repositories/abstract_repository.py index 6725aca279c..dbbc6a95026 100644 --- a/src/poetry/repositories/abstract_repository.py +++ b/src/poetry/repositories/abstract_repository.py @@ -6,9 +6,9 @@ if TYPE_CHECKING: + from poetry.core.constraints.version import Version from poetry.core.packages.dependency import Dependency from poetry.core.packages.package import Package - from poetry.core.semver.version import Version class AbstractRepository(ABC): diff --git a/tests/repositories/test_repository.py b/tests/repositories/test_repository.py index 83a7287cce4..551691d9ff7 100644 --- a/tests/repositories/test_repository.py +++ b/tests/repositories/test_repository.py @@ -2,7 +2,7 @@ import pytest -from poetry.core.semver.version import Version +from poetry.core.constraints.version import Version from poetry.factory import Factory from poetry.repositories import Repository From 7d414af680c967d0396ba46d3535d15ab8eca884 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Randy=20D=C3=B6ring?= <30527984+radoering@users.noreply.github.com> Date: Mon, 10 Oct 2022 20:58:02 +0200 Subject: [PATCH 14/25] feat(locker): `poetry lock` works if an invalid/incompatible lock file exists (#6753) After having created a lock file 2.0, running `poetry lock` with poetry 1.2.1 results in the following output: ``` The lock file is not compatible with the current version of Poetry. Upgrade Poetry to be able to read the lock file or, alternatively, regenerate the lock file with the `poetry lock` command. ``` Ironically, the error message proposes to run `poetry lock` which results in this error message. Further, it doesn't make sense that `poetry lock` fails because it creates a new lock file from scratch (in contrast to `poetry lock --no-update`). Running `poetry lock` is now also possible if there is a broken lock file. Resolves: #1196 --- .pre-commit-config.yaml | 1 + src/poetry/installation/installer.py | 2 +- src/poetry/packages/locker.py | 21 +++--- tests/console/commands/test_lock.py | 71 +++++++++++++++++++ tests/fixtures/incompatible_lock/poetry.lock | 2 + .../fixtures/incompatible_lock/pyproject.toml | 15 ++++ tests/fixtures/invalid_lock/poetry.lock | 1 + tests/fixtures/invalid_lock/pyproject.toml | 15 ++++ 8 files changed, 118 insertions(+), 10 deletions(-) create mode 100644 tests/fixtures/incompatible_lock/poetry.lock create mode 100644 tests/fixtures/incompatible_lock/pyproject.toml create mode 100644 tests/fixtures/invalid_lock/poetry.lock create mode 100644 tests/fixtures/invalid_lock/pyproject.toml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index cf3064c6a08..753817ab8e6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -12,6 +12,7 @@ repos: - id: check-case-conflict - id: check-json - id: check-toml + exclude: tests/fixtures/invalid_lock/poetry\.lock - id: check-yaml - id: pretty-format-json args: [--autofix, --no-ensure-ascii, --no-sort-keys] diff --git a/src/poetry/installation/installer.py b/src/poetry/installation/installer.py index 8567d04dd4a..83082020d21 100644 --- a/src/poetry/installation/installer.py +++ b/src/poetry/installation/installer.py @@ -218,7 +218,7 @@ def _do_install(self) -> int: locked_repository = Repository("poetry-locked") if self._update: - if self._locker.is_locked() and not self._lock: + if not self._lock and self._locker.is_locked(): locked_repository = self._locker.locked_repository() # If no packages have been whitelisted (The ones we want to update), diff --git a/src/poetry/packages/locker.py b/src/poetry/packages/locker.py index 8a6c62e75f7..b9df3e15456 100644 --- a/src/poetry/packages/locker.py +++ b/src/poetry/packages/locker.py @@ -73,10 +73,7 @@ def is_locked(self) -> bool: """ Checks whether the locker has been locked (lockfile found). """ - if not self._lock.exists(): - return False - - return "package" in self.lock_data + return self._lock.exists() def is_fresh(self) -> bool: """ @@ -256,12 +253,18 @@ def set_lock_data(self, root: Package, packages: list[Package]) -> bool: "content-hash": self._content_hash, } - if not self.is_locked() or lock != self.lock_data: + do_write = True + if self.is_locked(): + try: + lock_data = self.lock_data + except RuntimeError: + # incompatible, invalid or no lock file + pass + else: + do_write = lock != lock_data + if do_write: self._write_lock_data(lock) - - return True - - return False + return do_write def _write_lock_data(self, data: TOMLDocument) -> None: self.lock.write(data) diff --git a/tests/console/commands/test_lock.py b/tests/console/commands/test_lock.py index 5d6eb53887e..69de642e117 100644 --- a/tests/console/commands/test_lock.py +++ b/tests/console/commands/test_lock.py @@ -67,6 +67,20 @@ def poetry_with_old_lockfile( return _project_factory("old_lock", project_factory, fixture_dir) +@pytest.fixture +def poetry_with_incompatible_lockfile( + project_factory: ProjectFactory, fixture_dir: FixtureDirGetter +) -> Poetry: + return _project_factory("incompatible_lock", project_factory, fixture_dir) + + +@pytest.fixture +def poetry_with_invalid_lockfile( + project_factory: ProjectFactory, fixture_dir: FixtureDirGetter +) -> Poetry: + return _project_factory("invalid_lock", project_factory, fixture_dir) + + def test_lock_check_outdated( command_tester_factory: CommandTesterFactory, poetry_with_outdated_lockfile: Poetry, @@ -150,3 +164,60 @@ def test_lock_no_update( for package in packages: assert locked_repository.find_packages(package.to_dependency()) + + +@pytest.mark.parametrize("is_no_update", [False, True]) +def test_lock_with_incompatible_lockfile( + command_tester_factory: CommandTesterFactory, + poetry_with_incompatible_lockfile: Poetry, + repo: TestRepository, + is_no_update: bool, +) -> None: + repo.add_package(get_package("sampleproject", "1.3.1")) + + locker = Locker( + lock=poetry_with_incompatible_lockfile.pyproject.file.path.parent + / "poetry.lock", + local_config=poetry_with_incompatible_lockfile.locker._local_config, + ) + poetry_with_incompatible_lockfile.set_locker(locker) + + tester = command_tester_factory("lock", poetry=poetry_with_incompatible_lockfile) + if is_no_update: + # not possible because of incompatible lock file + expected = ( + "(?s)lock file is not compatible .*" + " regenerate the lock file with the `poetry lock` command" + ) + with pytest.raises(RuntimeError, match=expected): + tester.execute("--no-update") + else: + # still possible because lock file is not required + status_code = tester.execute() + assert status_code == 0 + + +@pytest.mark.parametrize("is_no_update", [False, True]) +def test_lock_with_invalid_lockfile( + command_tester_factory: CommandTesterFactory, + poetry_with_invalid_lockfile: Poetry, + repo: TestRepository, + is_no_update: bool, +) -> None: + repo.add_package(get_package("sampleproject", "1.3.1")) + + locker = Locker( + lock=poetry_with_invalid_lockfile.pyproject.file.path.parent / "poetry.lock", + local_config=poetry_with_invalid_lockfile.locker._local_config, + ) + poetry_with_invalid_lockfile.set_locker(locker) + + tester = command_tester_factory("lock", poetry=poetry_with_invalid_lockfile) + if is_no_update: + # not possible because of broken lock file + with pytest.raises(RuntimeError, match="Unable to read the lock file"): + tester.execute("--no-update") + else: + # still possible because lock file is not required + status_code = tester.execute() + assert status_code == 0 diff --git a/tests/fixtures/incompatible_lock/poetry.lock b/tests/fixtures/incompatible_lock/poetry.lock new file mode 100644 index 00000000000..37615c8ce57 --- /dev/null +++ b/tests/fixtures/incompatible_lock/poetry.lock @@ -0,0 +1,2 @@ +[metadata] +lock-version = "999.0" diff --git a/tests/fixtures/incompatible_lock/pyproject.toml b/tests/fixtures/incompatible_lock/pyproject.toml new file mode 100644 index 00000000000..377aa676be9 --- /dev/null +++ b/tests/fixtures/incompatible_lock/pyproject.toml @@ -0,0 +1,15 @@ +[tool.poetry] +name = "foobar" +version = "0.1.0" +description = "" +authors = ["Poetry Developer "] + +[tool.poetry.dependencies] +python = "^3.8" +sampleproject = ">=1.3.1" + +[tool.poetry.dev-dependencies] + +[build-system] +requires = ["poetry-core>=1.0.0"] +build-backend = "poetry.core.masonry.api" diff --git a/tests/fixtures/invalid_lock/poetry.lock b/tests/fixtures/invalid_lock/poetry.lock new file mode 100644 index 00000000000..59f4e044a6c --- /dev/null +++ b/tests/fixtures/invalid_lock/poetry.lock @@ -0,0 +1 @@ +This lock file is broken! diff --git a/tests/fixtures/invalid_lock/pyproject.toml b/tests/fixtures/invalid_lock/pyproject.toml new file mode 100644 index 00000000000..377aa676be9 --- /dev/null +++ b/tests/fixtures/invalid_lock/pyproject.toml @@ -0,0 +1,15 @@ +[tool.poetry] +name = "foobar" +version = "0.1.0" +description = "" +authors = ["Poetry Developer "] + +[tool.poetry.dependencies] +python = "^3.8" +sampleproject = ">=1.3.1" + +[tool.poetry.dev-dependencies] + +[build-system] +requires = ["poetry-core>=1.0.0"] +build-backend = "poetry.core.masonry.api" From 29728b5ad9289ad832f20e7315a4c91ba4a6743d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Randy=20D=C3=B6ring?= <30527984+radoering@users.noreply.github.com> Date: Mon, 10 Oct 2022 21:41:20 +0200 Subject: [PATCH 15/25] chore: post 1.2.2 cleanup --- CHANGELOG.md | 46 +++++++++++++++++++++++++++++++++++++++++++++- poetry.lock | 14 +++++++------- pyproject.toml | 4 ++-- 3 files changed, 54 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e3961ac434d..e4fe5ea3b92 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,51 @@ # Change Log +## [1.2.2] - 2022-10-10 + +### Added + +- Add forward compatibility for lock file format 2.0, which will be used by Poetry 1.3 ([#6608](https://github.com/python-poetry/poetry/pull/6608)). + +### Changed + +- Allow `poetry lock` to re-generate the lock file when invalid or incompatible ([#6753](https://github.com/python-poetry/poetry/pull/6753)). + +### Fixed + +- Fix an issue where the deprecated JSON API was used to query PyPI for available versions of a package ([#6081](https://github.com/python-poetry/poetry/pull/6081)). +- Fix an issue where versions were escaped wrongly when building the wheel name ([#6476](https://github.com/python-poetry/poetry/pull/6476)). +- Fix an issue where the installation of dependencies failed if pip is a dependency and is updated in parallel to other dependencies ([#6582](https://github.com/python-poetry/poetry/pull/6582)). +- Fix an issue where the names of extras were not normalized according to PEP 685 ([#6541](https://github.com/python-poetry/poetry/pull/6541)). +- Fix an issue where sdist names were not normalized ([#6621](https://github.com/python-poetry/poetry/pull/6621)). +- Fix an issue where invalid constraints, which are ignored, were only reported in a debug message instead of a warning ([#6730](https://github.com/python-poetry/poetry/pull/6730)). +- Fix an issue where `poetry shell` was broken in git bash on Windows ([#6560](https://github.com/python-poetry/poetry/pull/6560)). + +### Docs + +- Rework the README and contribution docs ([#6552](https://github.com/python-poetry/poetry/pull/6552)). +- Fix for inconsistent docs for multiple-constraint dependencies ([#6604](https://github.com/python-poetry/poetry/pull/6604)). +- Rephrase plugin configuration ([#6557](https://github.com/python-poetry/poetry/pull/6557)). +- Add a note about publishable repositories to `publish` ([#6641](https://github.com/python-poetry/poetry/pull/6641)). +- Fix the path for lazy-loaded bash completion ([#6656](https://github.com/python-poetry/poetry/pull/6656)). +- Fix a reference to the invalid option `--require` ([#6672](https://github.com/python-poetry/poetry/pull/6672)). +- Add a PowerShell one-liner to the basic usage section ([#6683](https://github.com/python-poetry/poetry/pull/6683)). +- Fix the minimum poetry version in the example for plugins ([#6739](https://github.com/python-poetry/poetry/pull/6739)). + +### poetry-core ([`1.3.2`](https://github.com/python-poetry/poetry-core/releases/tag/1.3.2)) + +- Add `3.11` to the list of available Python versions ([#477](https://github.com/python-poetry/poetry-core/pull/477)). +- Fix an issue where caret constraints of pre-releases with a major version of 0 resulted in an empty version range ([#475](https://github.com/python-poetry/poetry-core/pull/475)). + +### poetry-plugin-export ([`^1.1.2`](https://github.com/python-poetry/poetry-plugin-export/releases/tag/1.1.2)) + +- Add support for exporting `constraints.txt` files ([#128](https://github.com/python-poetry/poetry-plugin-export/pull/128)). +- Fix an issue where a relative path passed via `-o` was not interpreted relative to the current working directory ([#130](https://github.com/python-poetry/poetry-plugin-export/pull/130)). + + ## [1.2.1] - 2022-09-16 ### Changed + - Bump `poetry-core` to [`1.2.0`](https://github.com/python-poetry/poetry-core/releases/tag/1.2.0). - Bump `poetry-plugin-export` to [`^1.0.7`](https://github.com/python-poetry/poetry-plugin-export/releases/tag/1.0.7). @@ -1544,7 +1587,8 @@ Initial release -[Unreleased]: https://github.com/python-poetry/poetry/compare/1.2.1...master +[Unreleased]: https://github.com/python-poetry/poetry/compare/1.2.2...master +[1.2.2]: https://github.com/python-poetry/poetry/releases/tag/1.2.2 [1.2.1]: https://github.com/python-poetry/poetry/releases/tag/1.2.1 [1.2.0]: https://github.com/python-poetry/poetry/releases/tag/1.2.0 [1.2.0rc2]: https://github.com/python-poetry/poetry/releases/tag/1.2.0rc2 diff --git a/poetry.lock b/poetry.lock index 69ea29c1b13..6a538a93c9d 100644 --- a/poetry.lock +++ b/poetry.lock @@ -513,7 +513,7 @@ testing = ["pytest", "pytest-benchmark"] [[package]] name = "poetry-core" -version = "1.3.1" +version = "1.3.2" description = "Poetry PEP 517 Build Backend" category = "main" optional = false @@ -524,7 +524,7 @@ importlib-metadata = {version = ">=1.7.0", markers = "python_version < \"3.8\""} [[package]] name = "poetry-plugin-export" -version = "1.1.1" +version = "1.1.2" description = "Poetry plugin to export the dependencies to various formats" category = "main" optional = false @@ -951,7 +951,7 @@ testing = ["func-timeout", "jaraco.itertools", "pytest (>=6)", "pytest-black (>= [metadata] lock-version = "1.1" python-versions = "^3.7" -content-hash = "1ebadc410b20420b0b570d94dc78220ae54ad814498b1c2fa3fb3cbd183a1413" +content-hash = "6e19a6f10b73d0236013c755105f3ee0ef1b4a619aaab90134cff5bd88dcdb96" [metadata.files] attrs = [ @@ -1360,12 +1360,12 @@ pluggy = [ {file = "pluggy-1.0.0.tar.gz", hash = "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159"}, ] poetry-core = [ - {file = "poetry-core-1.3.1.tar.gz", hash = "sha256:f4e68f596ba7560d41f67c705e160a0ed1a061c564cc1f989a5829aa64ea6279"}, - {file = "poetry_core-1.3.1-py3-none-any.whl", hash = "sha256:da1018d30dcbed101865a24c2a79c60be3f9800e7c91721c6ec9d4bc9e210538"}, + {file = "poetry-core-1.3.2.tar.gz", hash = "sha256:0ab006a40cb38d6a38b97264f6835da2f08a96912f2728ce668e9ac6a34f686f"}, + {file = "poetry_core-1.3.2-py3-none-any.whl", hash = "sha256:ea0f5a90b339cde132b4e43cff78a1b440cd928db864bb67cfc97fdfcefe7218"}, ] poetry-plugin-export = [ - {file = "poetry-plugin-export-1.1.1.tar.gz", hash = "sha256:23e3e512a609b54ef5ac441339fc9e68fd41e61d15bd924eb0094b4fda1e30d0"}, - {file = "poetry_plugin_export-1.1.1-py3-none-any.whl", hash = "sha256:170fa367794d2385975d75298fe5509f772d35216ee36b8fa50c0350a064b761"}, + {file = "poetry-plugin-export-1.1.2.tar.gz", hash = "sha256:5e92525dd63f38ce74a51ed68ea91d753523f21ce5f9ef8d3b793e2a4b2222ef"}, + {file = "poetry_plugin_export-1.1.2-py3-none-any.whl", hash = "sha256:946e3313b3d00c18fb9a50522e9d5e6a7e111beaba8d6ae33297662fc2070ac1"}, ] pre-commit = [ {file = "pre_commit-2.20.0-py2.py3-none-any.whl", hash = "sha256:51a5ba7c480ae8072ecdb6933df22d2f812dc897d5fe848778116129a681aac7"}, diff --git a/pyproject.toml b/pyproject.toml index c66038743a9..1041a96a8e8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,8 +44,8 @@ generate-setup-file = false [tool.poetry.dependencies] python = "^3.7" -poetry-core = "^1.3.1" -poetry-plugin-export = "^1.1.1" +poetry-core = "^1.3.2" +poetry-plugin-export = "^1.1.2" "backports.cached-property" = { version = "^1.0.2", python = "<3.8" } cachecontrol = { version = "^0.12.9", extras = ["filecache"] } cachy = "^0.3.0" From 46ca87d2a2dae368773e80a18df724f481a764f4 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 11 Oct 2022 00:35:30 +0000 Subject: [PATCH 16/25] [pre-commit.ci] pre-commit autoupdate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/asottile/pyupgrade: v2.38.2 → v3.1.0](https://github.com/asottile/pyupgrade/compare/v2.38.2...v3.1.0) - [github.com/psf/black: 22.8.0 → 22.10.0](https://github.com/psf/black/compare/22.8.0...22.10.0) --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 753817ab8e6..27563811d61 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -47,7 +47,7 @@ repos: - flake8-pie==0.16.0 - repo: https://github.com/asottile/pyupgrade - rev: v2.38.2 + rev: v3.1.0 hooks: - id: pyupgrade args: [--py37-plus] @@ -77,7 +77,7 @@ repos: args: [--lines-after-imports, "-1"] - repo: https://github.com/psf/black - rev: 22.8.0 + rev: 22.10.0 hooks: - id: black From 7e46798a3e3fac09c7ea37dfbbbd3d9c1124a359 Mon Sep 17 00:00:00 2001 From: Bjorn Neergaard Date: Mon, 10 Oct 2022 22:49:59 -0600 Subject: [PATCH 17/25] chore: add missing CHANGELOG entry for 1.1.15 (#6760) also fix formatting for some older entries --- CHANGELOG.md | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e4fe5ea3b92..3635a702289 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -140,6 +140,14 @@ - Document suggested `tox` config for different use cases ([#6026](https://github.com/python-poetry/poetry/pull/6026)) +## [1.1.15] - 2022-08-22 + +### Changed + +- Poetry now fallback to gather metadata for dependencies via pep517 if parsing pyproject.toml fail ([#6206](https://github.com/python-poetry/poetry/pull/6206)) +- Extras and extras dependencies are now sorted in lock file ([#6207](https://github.com/python-poetry/poetry/pull/6207)) + + ## [1.2.0b3] - 2022-07-13 **Important**: This release fixes a critical issue that prevented hashes from being retrieved when locking dependencies, @@ -205,12 +213,14 @@ $ poetry cache clear pypi --all - Improved autocompletion documentation ([#5879](https://github.com/python-poetry/poetry/pull/5879)) - Improved `scripts` definition documentation ([#5884](https://github.com/python-poetry/poetry/pull/5884)) + ## [1.1.14] - 2022-07-08 -## Fixed +### Fixed - Fixed an issue where dependencies hashes could not be retrieved when locking due to a breaking change on PyPI JSON API ([#5973](https://github.com/python-poetry/poetry/pull/5973)) + ## [1.2.0b2] - 2022-06-07 ### Added @@ -319,6 +329,7 @@ $ poetry cache clear pypi --all - Improved contributing documentation ([#5708](https://github.com/python-poetry/poetry/pull/5708)) - Remove all references to `--dev-only` option ([#5771](https://github.com/python-poetry/poetry/pull/5771)) + ## [1.2.0b1] - 2022-03-17 ### Fixed @@ -377,6 +388,7 @@ $ poetry cache clear pypi --all - Fixed an issue where conda envs in windows are always reported as broken([#5008](https://github.com/python-poetry/poetry/pull/5008)) - Fixed an issue where Poetry doesn't find its own venv on `poetry self update` ([#5048](https://github.com/python-poetry/poetry/pull/5048)) + ## [1.1.12] - 2021-11-27 ### Fixed @@ -385,6 +397,7 @@ $ poetry cache clear pypi --all - Fixed `JSONDecodeError` when installing packages by updating `cachecontrol` version ([#4831](https://github.com/python-poetry/poetry/pull/4831)) - Fixed dropped markers in dependency walk ([#4686](https://github.com/python-poetry/poetry/pull/4686)) + ## [1.1.11] - 2021-10-04 ### Fixed @@ -393,12 +406,14 @@ $ poetry cache clear pypi --all - Fixed an issue where the wrong `git` executable could be used on Windows. ([python-poetry/poetry-core#213](https://github.com/python-poetry/poetry-core/pull/213)) - Fixed an issue where the Python 3.10 classifier was not automatically added. ([python-poetry/poetry-core#215](https://github.com/python-poetry/poetry-core/pull/215)) + ## [1.1.10] - 2021-09-21 ### Fixed - Fixed an issue where non-sha256 hashes were not checked. ([#4529](https://github.com/python-poetry/poetry/pull/4529)) + ## [1.1.9] - 2021-09-18 ### Fixed @@ -408,6 +423,7 @@ $ poetry cache clear pypi --all - Fixed an issue where unsafe parameters could be passed to `git` commands. ([python-poetry/poetry-core#203](https://github.com/python-poetry/poetry-core/pull/203)) - Fixed an issue where the wrong `git` executable could be used on Windows. ([python-poetry/poetry-core#205](https://github.com/python-poetry/poetry-core/pull/205)) + ## [1.1.8] - 2021-08-19 ### Fixed @@ -418,6 +434,7 @@ $ poetry cache clear pypi --all - Fixed environment detection for Python 3.10 environments. ([#4387](https://github.com/python-poetry/poetry/pull/4387)) - Fixed an error in the evaluation of `in/not in` markers ([python-poetry/poetry-core#189](https://github.com/python-poetry/poetry-core/pull/189)) + ## [1.2.0a2] - 2021-08-01 ### Added @@ -454,6 +471,7 @@ You can use `poetry lock` to do so without the `--no-update` option. - Fixed an issue where transitive dependencies of directory or VCS dependencies were not installed or otherwise removed. ([#4203](https://github.com/python-poetry/poetry/pull/4203)) - Fixed an issue where the combination of the `--tree` and `--no-dev` options for the show command was still displaying development dependencies. ([#3992](https://github.com/python-poetry/poetry/pull/3992)) + ## [1.2.0a1] - 2021-05-21 This release is the first testing release of the upcoming 1.2.0 version. @@ -483,6 +501,7 @@ It **drops** support for Python 2.7 and 3.5. - Fixed an error where command line options were not taken into account when using the `run` command. ([#3618](https://github.com/python-poetry/poetry/pull/3618)) - Fixed an error in the way custom repositories were resolved. ([#3406](https://github.com/python-poetry/poetry/pull/3406)) + ## [1.1.6] - 2021-04-14 ### Fixed @@ -494,6 +513,7 @@ It **drops** support for Python 2.7 and 3.5. - Fixed an error where VCS dependencies were always updated. ([#3947](https://github.com/python-poetry/poetry/pull/3947)) - Fixed an error where the incorrect version of a package was locked when using environment markers. ([#3945](https://github.com/python-poetry/poetry/pull/3945)) + ## [1.1.5] - 2021-03-04 ### Fixed @@ -504,6 +524,7 @@ It **drops** support for Python 2.7 and 3.5. - Fixed errors when trying to handle newer wheels by using the latest version of poetry-core and packaging. (#3677) - Fixed an error when using some versions of poetry-core due to an incorrect import. (#3696) + ## [1.1.4] - 2020-10-23 ### Added @@ -524,6 +545,7 @@ It **drops** support for Python 2.7 and 3.5. - Fixed recursion error when locked dependencies contain cyclic dependencies. ([#3237](https://github.com/python-poetry/poetry/pull/3237)) - Fixed propagation of editable flag for VCS dependencies. ([#3264](https://github.com/python-poetry/poetry/pull/3264)) + ## [1.1.3] - 2020-10-14 ### Changed @@ -538,6 +560,7 @@ It **drops** support for Python 2.7 and 3.5. - Fixed `show` command to use already resolved package metadata. ([#3117](https://github.com/python-poetry/poetry/pull/3117)) - Fixed multiple issues with `export` command output when using `requirements.txt` format. ([#3119](https://github.com/python-poetry/poetry/pull/3119)) + ## [1.1.2] - 2020-10-06 ### Changed @@ -548,6 +571,7 @@ It **drops** support for Python 2.7 and 3.5. - Fixed export of `requirements.txt` when project dependency contains git dependencies. ([#3100](https://github.com/python-poetry/poetry/pull/3100)) + ## [1.1.1] - 2020-10-05 ### Added @@ -564,6 +588,7 @@ It **drops** support for Python 2.7 and 3.5. - Fixed incorrect selection of configured source url when a publish repository url configuration with the same name already exists. ([#3047](https://github.com/python-poetry/poetry/pull/3047)) - Fixed dependency resolution issues when the same package is specified in multiple dependency extras. ([#3046](https://github.com/python-poetry/poetry/pull/3046)) + ## [1.1.0] - 2020-10-01 ### Changed From a27074e51ce24702258a92249e71c5bf9d9659bc Mon Sep 17 00:00:00 2001 From: Otto Fowler Date: Tue, 11 Oct 2022 11:07:36 -0400 Subject: [PATCH 18/25] improve cache clear description (#6776) --- src/poetry/console/commands/cache/clear.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/poetry/console/commands/cache/clear.py b/src/poetry/console/commands/cache/clear.py index 6134bf202f0..8faf7a4c702 100644 --- a/src/poetry/console/commands/cache/clear.py +++ b/src/poetry/console/commands/cache/clear.py @@ -12,7 +12,7 @@ class CacheClearCommand(Command): name = "cache clear" - description = "Clears Poetry's cache." + description = "Clears a Poetry cache by name." arguments = [argument("cache", description="The name of the cache to clear.")] options = [option("all", description="Clear all entries in the cache.")] From 633f8f6b44015fe410c3e6d2646f0fd8e4440dca Mon Sep 17 00:00:00 2001 From: Marti Raudsepp Date: Tue, 11 Oct 2022 22:50:39 +0300 Subject: [PATCH 19/25] Add changelog URL to package metadata (#6770) When Renovate bot creates an automated pull request to update Poetry, it can automatically include the link to changelog so it's easier to find out what changed. The URL name has to be one of: 'changelog', 'change log', 'changes', 'release notes', 'news', "what's new" (case insensitive) https://github.com/renovatebot/renovate/blob/c3a87b687ed44a1a100e0e4f91b81e22b7c32482/lib/modules/datasource/pypi/index.ts#L130-L140 Co-authored-by: Bjorn Neergaard --- pyproject.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 1041a96a8e8..b2280b59eac 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,6 +37,9 @@ classifiers = [ "Topic :: Software Development :: Libraries :: Python Modules" ] +[tool.poetry.urls] +Changelog = "https://python-poetry.org/history/" + [tool.poetry.build] generate-setup-file = false From e512cbee1bffa21a7e7fb901fbfb0b58ffa2d830 Mon Sep 17 00:00:00 2001 From: Chad Crawford Date: Tue, 11 Oct 2022 11:52:38 -0700 Subject: [PATCH 20/25] Add `poetry.utils.cache` to replace cachy dependency. --- src/poetry/utils/cache.py | 192 ++++++++++++++++++++++++++++++++++++++ tests/utils/test_cache.py | 188 +++++++++++++++++++++++++++++++++++++ 2 files changed, 380 insertions(+) create mode 100644 src/poetry/utils/cache.py create mode 100644 tests/utils/test_cache.py diff --git a/src/poetry/utils/cache.py b/src/poetry/utils/cache.py new file mode 100644 index 00000000000..9eb7872b09f --- /dev/null +++ b/src/poetry/utils/cache.py @@ -0,0 +1,192 @@ +from __future__ import annotations + +import contextlib +import dataclasses +import hashlib +import json +import shutil +import time + +from pathlib import Path +from typing import Any +from typing import Callable +from typing import Generic +from typing import TypeVar + + +# Used by Cachy for items that do not expire. +MAX_DATE = 9999999999 +T = TypeVar("T") + + +def decode(string: bytes, encodings: list[str] | None = None) -> str: + """ + Compatiblity decode function pulled from cachy. + + :param string: The byte string to decode. + :param encodings: List of encodings to apply + :return: Decoded string + """ + if encodings is None: + encodings = ["utf-8", "latin1", "ascii"] + + for encoding in encodings: + with contextlib.suppress(UnicodeDecodeError): + return string.decode(encoding) + + return string.decode(encodings[0], errors="ignore") + + +def encode(string: str, encodings: list[str] | None = None) -> bytes: + """ + Compatibility encode function from cachy. + + :param string: The string to encode. + :param encodings: List of encodings to apply + :return: Encoded byte string + """ + if encodings is None: + encodings = ["utf-8", "latin1", "ascii"] + + for encoding in encodings: + with contextlib.suppress(UnicodeDecodeError): + return string.encode(encoding) + + return string.encode(encodings[0], errors="ignore") + + +def _expiration(minutes: int) -> int: + """ + Calculates the time in seconds since epoch that occurs 'minutes' from now. + + :param minutes: The number of minutes to count forward + """ + return round(time.time()) + minutes * 60 + + +_HASHES = { + "md5": (hashlib.md5, 2), + "sha1": (hashlib.sha1, 4), + "sha256": (hashlib.sha256, 8), +} + + +@dataclasses.dataclass(frozen=True) +class CacheItem(Generic[T]): + """ + Stores data and metadata for cache items. + """ + + data: T + expires: int | None = None + + @property + def expired(self) -> bool: + """ + Return true if the cache item has exceeded its expiration period. + """ + return self.expires is not None and time.time() >= self.expires + + +@dataclasses.dataclass(frozen=True) +class FileCache(Generic[T]): + """ + Cachy-compatible minimal file cache. Stores subsequent data in a JSON format. + + :param path: The path that the cache starts at. + :param hash_type: The hash to use for encoding keys/building directories. + """ + + path: Path + hash_type: str = "sha256" + + def get(self, key: str) -> T | None: + return self._get_payload(key) + + def has(self, key: str) -> bool: + """ + Determine if a file exists and has not expired in the cache. + :param key: The cache key + :returns: True if the key exists in the cache + """ + return self.get(key) is not None + + def put(self, key: str, value: Any, minutes: int | None = None) -> None: + """ + Store an item in the cache. + + :param key: The cache key + :param value: The cache value + :param minutes: The lifetime in minutes of the cached value + """ + payload: CacheItem[Any] = CacheItem( + value, expires=_expiration(minutes) if minutes is not None else None + ) + path = self._path(key) + path.parent.mkdir(parents=True, exist_ok=True) + with open(path, "wb") as f: + f.write(self._serialize(payload)) + + def forget(self, key: str) -> None: + """ + Remove an item from the cache. + + :param key: The cache key + """ + path = self._path(key) + if path.exists(): + path.unlink() + + def flush(self) -> None: + """ + Clear the cache. + """ + shutil.rmtree(self.path) + + def remember( + self, key: str, callback: T | Callable[[], T], minutes: int | None = None + ) -> T: + """ + Get an item from the cache, or use a default from callback. + + :param key: The cache key + :param callback: Callback function providing default value + :param minutes: The lifetime in minutes of the cached value + """ + value = self.get(key) + if value is None: + value = callback() if callable(callback) else callback + self.put(key, value, minutes) + return value + + def _get_payload(self, key: str) -> T | None: + path = self._path(key) + + if not path.exists(): + return None + + with open(path, "rb") as f: + payload = self._deserialize(f.read()) + + if payload.expired: + self.forget(key) + return None + else: + return payload.data + + def _path(self, key: str) -> Path: + hash_type, parts_count = _HASHES[self.hash_type] + h = hash_type(encode(key)).hexdigest() + parts = [h[i : i + 2] for i in range(0, len(h), 2)][:parts_count] + return Path(self.path, *parts, h) + + def _serialize(self, payload: CacheItem[T]) -> bytes: + expires = payload.expires or MAX_DATE + data = json.dumps(payload.data) + return encode(f"{expires:010d}{data}") + + def _deserialize(self, data_raw: bytes) -> CacheItem[T]: + data_str = decode(data_raw) + data = json.loads(data_str[10:]) + expires = int(data_str[:10]) + return CacheItem(data, expires) diff --git a/tests/utils/test_cache.py b/tests/utils/test_cache.py new file mode 100644 index 00000000000..21079ee42e0 --- /dev/null +++ b/tests/utils/test_cache.py @@ -0,0 +1,188 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING +from typing import Any +from typing import TypeVar +from typing import Union +from unittest.mock import Mock + +import pytest + +from cachy import CacheManager + +from poetry.utils.cache import FileCache + + +if TYPE_CHECKING: + from pathlib import Path + + from _pytest.monkeypatch import MonkeyPatch + from pytest import FixtureRequest + from pytest_mock import MockerFixture + + from tests.conftest import Config + + +FILE_CACHE = Union[FileCache, CacheManager] +T = TypeVar("T") + + +@pytest.fixture +def repository_cache_dir(monkeypatch: MonkeyPatch, config: Config) -> Path: + return config.repository_cache_directory + + +def patch_cachy(cache: CacheManager) -> CacheManager: + old_put = cache.put + old_remember = cache.remember + + def new_put(key: str, value: Any, minutes: int | None = None) -> Any: + if minutes is not None: + return old_put(key, value, minutes=minutes) + else: + return cache.forever(key, value) + + cache.put = new_put + + def new_remember(key: str, value: Any, minutes: int | None = None) -> Any: + if minutes is not None: + return old_remember(key, value, minutes=minutes) + else: + return cache.remember_forever(key, value) + + cache.remember = new_remember + return cache + + +@pytest.fixture +def cachy_file_cache(repository_cache_dir: Path) -> CacheManager: + cache = CacheManager( + { + "default": "cache", + "serializer": "json", + "stores": { + "cache": {"driver": "file", "path": str(repository_cache_dir / "cache")} + }, + } + ) + return patch_cachy(cache) + + +@pytest.fixture +def poetry_file_cache(repository_cache_dir: Path) -> FileCache[T]: + return FileCache(repository_cache_dir / "cache") + + +@pytest.fixture +def cachy_dict_cache() -> CacheManager: + cache = CacheManager( + { + "default": "cache", + "serializer": "json", + "stores": {"cache": {"driver": "dict"}}, + } + ) + return patch_cachy(cache) + + +@pytest.mark.parametrize("cache_name", ["cachy_file_cache", "poetry_file_cache"]) +def test_cache_get_put_has(cache_name: str, request: FixtureRequest) -> None: + cache = request.getfixturevalue(cache_name) + cache.put("key1", "value") + cache.put("key2", {"a": ["json-encoded", "value"]}) + + assert cache.get("key1") == "value" + assert cache.get("key2") == {"a": ["json-encoded", "value"]} + assert cache.has("key1") + assert cache.has("key2") + assert not cache.has("key3") + + +@pytest.mark.parametrize("cache_name", ["cachy_file_cache", "poetry_file_cache"]) +def test_cache_forget(cache_name: str, request: FixtureRequest) -> None: + cache = request.getfixturevalue(cache_name) + cache.put("key1", "value") + cache.put("key2", "value") + + assert cache.has("key1") + assert cache.has("key2") + + cache.forget("key1") + + assert not cache.has("key1") + assert cache.has("key2") + + +@pytest.mark.parametrize("cache_name", ["cachy_file_cache", "poetry_file_cache"]) +def test_cache_flush(cache_name: str, request: FixtureRequest) -> None: + cache = request.getfixturevalue(cache_name) + cache.put("key1", "value") + cache.put("key2", "value") + + assert cache.has("key1") + assert cache.has("key2") + + cache.flush() + + assert not cache.has("key1") + assert not cache.has("key2") + + +@pytest.mark.parametrize("cache_name", ["cachy_file_cache", "poetry_file_cache"]) +def test_cache_remember( + cache_name: str, request: FixtureRequest, mocker: MockerFixture +) -> None: + cache = request.getfixturevalue(cache_name) + + method = Mock(return_value="value2") + cache.put("key1", "value1") + assert cache.remember("key1", method) == "value1" + method.assert_not_called() + + assert cache.remember("key2", method) == "value2" + method.assert_called() + + +@pytest.mark.parametrize("cache_name", ["cachy_file_cache", "poetry_file_cache"]) +def test_cache_get_limited_minutes( + mocker: MockerFixture, + cache_name: str, + request: FixtureRequest, +) -> None: + cache = request.getfixturevalue(cache_name) + + # needs to be 10 digits because cachy assumes it's a 10-digit int. + start_time = 1111111111 + + mocker.patch("time.time", return_value=start_time) + cache.put("key1", "value", minutes=5) + cache.put("key2", "value", minutes=5) + + assert cache.get("key1") is not None + assert cache.get("key2") is not None + + mocker.patch("time.time", return_value=start_time + 5 * 60 + 1) + # check to make sure that the cache deletes for has() and get() + assert not cache.has("key1") + assert cache.get("key2") is None + + +def test_cachy_compatibility( + cachy_file_cache: CacheManager, poetry_file_cache: FileCache[T] +) -> None: + """ + The new file cache should be able to support reading legacy caches. + """ + test_str = "value" + test_obj = {"a": ["json", "object"]} + cachy_file_cache.put("key1", test_str) + cachy_file_cache.put("key2", test_obj) + + assert poetry_file_cache.get("key1") == test_str + assert poetry_file_cache.get("key2") == test_obj + + poetry_file_cache.put("key3", test_str) + poetry_file_cache.put("key4", test_obj) + + assert cachy_file_cache.get("key3") == test_str + assert cachy_file_cache.get("key4") == test_obj From c2b1fb3d17f9da1bfc8dc483fbb8d916917c03f0 Mon Sep 17 00:00:00 2001 From: Chad Crawford Date: Tue, 11 Oct 2022 11:55:18 -0700 Subject: [PATCH 21/25] Migrate codebase from `cachy` to `poetry.utils.cache`. --- poetry.lock | 98 ++++++++++---------- pyproject.toml | 3 +- src/poetry/console/commands/cache/clear.py | 11 +-- src/poetry/repositories/cached.py | 18 +--- src/poetry/repositories/legacy_repository.py | 30 +++--- src/poetry/repositories/pypi_repository.py | 22 ++--- tests/console/commands/cache/test_clear.py | 5 +- tests/console/commands/test_add.py | 37 +++++++- 8 files changed, 111 insertions(+), 113 deletions(-) diff --git a/poetry.lock b/poetry.lock index 6a538a93c9d..3a4cb997d61 100644 --- a/poetry.lock +++ b/poetry.lock @@ -41,7 +41,7 @@ redis = ["redis (>=2.10.5)"] name = "cachy" version = "0.3.0" description = "Cachy provides a simple yet effective caching library." -category = "main" +category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" @@ -247,7 +247,7 @@ python-versions = ">=3" [[package]] name = "identify" -version = "2.5.5" +version = "2.5.6" description = "File identification library for Python" category = "dev" optional = false @@ -283,7 +283,7 @@ testing = ["flake8 (<5)", "flufl.flake8", "importlib-resources (>=1.3)", "packag [[package]] name = "importlib-resources" -version = "5.9.0" +version = "5.10.0" description = "Read resources from Python packages" category = "main" optional = false @@ -293,8 +293,8 @@ python-versions = ">=3.7" zipp = {version = ">=3.1.0", markers = "python_version < \"3.10\""} [package.extras] -docs = ["jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx"] -testing = ["pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)"] +docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)"] +testing = ["flake8 (<5)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)"] [[package]] name = "iniconfig" @@ -396,7 +396,7 @@ python-versions = "*" [[package]] name = "mypy" -version = "0.981" +version = "0.982" description = "Optional static typing for Python" category = "dev" optional = false @@ -674,7 +674,7 @@ pytest = ">=4.0.0" [[package]] name = "pytest-mock" -version = "3.9.0" +version = "3.10.0" description = "Thin-wrapper around the mock package for easier use with pytest" category = "dev" optional = false @@ -861,7 +861,7 @@ python-versions = "*" [[package]] name = "types-requests" -version = "2.28.11" +version = "2.28.11.2" description = "Typing stubs for requests" category = "dev" optional = false @@ -880,7 +880,7 @@ python-versions = "*" [[package]] name = "typing-extensions" -version = "4.3.0" +version = "4.4.0" description = "Backported and Experimental Type Hints for Python 3.7+" category = "main" optional = false @@ -938,20 +938,20 @@ cffi = ">=1.0" [[package]] name = "zipp" -version = "3.8.1" +version = "3.9.0" description = "Backport of pathlib-compatible object wrapper for zip files" category = "main" optional = false python-versions = ">=3.7" [package.extras] -docs = ["jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx"] -testing = ["func-timeout", "jaraco.itertools", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)"] +docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)"] +testing = ["flake8 (<5)", "func-timeout", "jaraco.functools", "jaraco.itertools", "more-itertools", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)"] [metadata] lock-version = "1.1" python-versions = "^3.7" -content-hash = "6e19a6f10b73d0236013c755105f3ee0ef1b4a619aaab90134cff5bd88dcdb96" +content-hash = "7db62de4ee4f2831829fc08b6a3d190f177c62d5af7676e20fb228bb58274850" [metadata.files] attrs = [ @@ -1200,8 +1200,8 @@ httpretty = [ {file = "httpretty-1.1.4.tar.gz", hash = "sha256:20de0e5dd5a18292d36d928cc3d6e52f8b2ac73daec40d41eb62dee154933b68"}, ] identify = [ - {file = "identify-2.5.5-py2.py3-none-any.whl", hash = "sha256:ef78c0d96098a3b5fe7720be4a97e73f439af7cf088ebf47b620aeaa10fadf97"}, - {file = "identify-2.5.5.tar.gz", hash = "sha256:322a5699daecf7c6fd60e68852f36f2ecbb6a36ff6e6e973e0d2bb6fca203ee6"}, + {file = "identify-2.5.6-py2.py3-none-any.whl", hash = "sha256:b276db7ec52d7e89f5bc4653380e33054ddc803d25875952ad90b0f012cbcdaa"}, + {file = "identify-2.5.6.tar.gz", hash = "sha256:6c32dbd747aa4ceee1df33f25fed0b0f6e0d65721b15bd151307ff7056d50245"}, ] idna = [ {file = "idna-3.4-py3-none-any.whl", hash = "sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"}, @@ -1212,8 +1212,8 @@ importlib-metadata = [ {file = "importlib_metadata-4.13.0.tar.gz", hash = "sha256:dd0173e8f150d6815e098fd354f6414b0f079af4644ddfe90c71e2fc6174346d"}, ] importlib-resources = [ - {file = "importlib_resources-5.9.0-py3-none-any.whl", hash = "sha256:f78a8df21a79bcc30cfd400bdc38f314333de7c0fb619763f6b9dabab8268bb7"}, - {file = "importlib_resources-5.9.0.tar.gz", hash = "sha256:5481e97fb45af8dcf2f798952625591c58fe599d0735d86b10f54de086a61681"}, + {file = "importlib_resources-5.10.0-py3-none-any.whl", hash = "sha256:ee17ec648f85480d523596ce49eae8ead87d5631ae1551f913c0100b5edd3437"}, + {file = "importlib_resources-5.10.0.tar.gz", hash = "sha256:c01b1b94210d9849f286b86bb51bcea7cd56dde0600d8db721d7b81330711668"}, ] iniconfig = [ {file = "iniconfig-1.1.1-py2.py3-none-any.whl", hash = "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3"}, @@ -1298,30 +1298,30 @@ msgpack = [ {file = "msgpack-1.0.4.tar.gz", hash = "sha256:f5d869c18f030202eb412f08b28d2afeea553d6613aee89e200d7aca7ef01f5f"}, ] mypy = [ - {file = "mypy-0.981-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:4bc460e43b7785f78862dab78674e62ec3cd523485baecfdf81a555ed29ecfa0"}, - {file = "mypy-0.981-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:756fad8b263b3ba39e4e204ee53042671b660c36c9017412b43af210ddee7b08"}, - {file = "mypy-0.981-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a16a0145d6d7d00fbede2da3a3096dcc9ecea091adfa8da48fa6a7b75d35562d"}, - {file = "mypy-0.981-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ce65f70b14a21fdac84c294cde75e6dbdabbcff22975335e20827b3b94bdbf49"}, - {file = "mypy-0.981-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:6e35d764784b42c3e256848fb8ed1d4292c9fc0098413adb28d84974c095b279"}, - {file = "mypy-0.981-cp310-cp310-win_amd64.whl", hash = "sha256:e53773073c864d5f5cec7f3fc72fbbcef65410cde8cc18d4f7242dea60dac52e"}, - {file = "mypy-0.981-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:6ee196b1d10b8b215e835f438e06965d7a480f6fe016eddbc285f13955cca659"}, - {file = "mypy-0.981-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8ad21d4c9d3673726cf986ea1d0c9fb66905258709550ddf7944c8f885f208be"}, - {file = "mypy-0.981-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:d1debb09043e1f5ee845fa1e96d180e89115b30e47c5d3ce53bc967bab53f62d"}, - {file = "mypy-0.981-cp37-cp37m-win_amd64.whl", hash = "sha256:9f362470a3480165c4c6151786b5379351b790d56952005be18bdbdd4c7ce0ae"}, - {file = "mypy-0.981-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:c9e0efb95ed6ca1654951bd5ec2f3fa91b295d78bf6527e026529d4aaa1e0c30"}, - {file = "mypy-0.981-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:e178eaffc3c5cd211a87965c8c0df6da91ed7d258b5fc72b8e047c3771317ddb"}, - {file = "mypy-0.981-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:06e1eac8d99bd404ed8dd34ca29673c4346e76dd8e612ea507763dccd7e13c7a"}, - {file = "mypy-0.981-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fa38f82f53e1e7beb45557ff167c177802ba7b387ad017eab1663d567017c8ee"}, - {file = "mypy-0.981-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:64e1f6af81c003f85f0dfed52db632817dabb51b65c0318ffbf5ff51995bbb08"}, - {file = "mypy-0.981-cp38-cp38-win_amd64.whl", hash = "sha256:e1acf62a8c4f7c092462c738aa2c2489e275ed386320c10b2e9bff31f6f7e8d6"}, - {file = "mypy-0.981-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:b6ede64e52257931315826fdbfc6ea878d89a965580d1a65638ef77cb551f56d"}, - {file = "mypy-0.981-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:eb3978b191b9fa0488524bb4ffedf2c573340e8c2b4206fc191d44c7093abfb7"}, - {file = "mypy-0.981-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:77f8fcf7b4b3cc0c74fb33ae54a4cd00bb854d65645c48beccf65fa10b17882c"}, - {file = "mypy-0.981-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f64d2ce043a209a297df322eb4054dfbaa9de9e8738291706eaafda81ab2b362"}, - {file = "mypy-0.981-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:2ee3dbc53d4df7e6e3b1c68ac6a971d3a4fb2852bf10a05fda228721dd44fae1"}, - {file = "mypy-0.981-cp39-cp39-win_amd64.whl", hash = "sha256:8e8e49aa9cc23aa4c926dc200ce32959d3501c4905147a66ce032f05cb5ecb92"}, - {file = "mypy-0.981-py3-none-any.whl", hash = "sha256:794f385653e2b749387a42afb1e14c2135e18daeb027e0d97162e4b7031210f8"}, - {file = "mypy-0.981.tar.gz", hash = "sha256:ad77c13037d3402fbeffda07d51e3f228ba078d1c7096a73759c9419ea031bf4"}, + {file = "mypy-0.982-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:5085e6f442003fa915aeb0a46d4da58128da69325d8213b4b35cc7054090aed5"}, + {file = "mypy-0.982-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:41fd1cf9bc0e1c19b9af13a6580ccb66c381a5ee2cf63ee5ebab747a4badeba3"}, + {file = "mypy-0.982-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f793e3dd95e166b66d50e7b63e69e58e88643d80a3dcc3bcd81368e0478b089c"}, + {file = "mypy-0.982-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:86ebe67adf4d021b28c3f547da6aa2cce660b57f0432617af2cca932d4d378a6"}, + {file = "mypy-0.982-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:175f292f649a3af7082fe36620369ffc4661a71005aa9f8297ea473df5772046"}, + {file = "mypy-0.982-cp310-cp310-win_amd64.whl", hash = "sha256:8ee8c2472e96beb1045e9081de8e92f295b89ac10c4109afdf3a23ad6e644f3e"}, + {file = "mypy-0.982-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:58f27ebafe726a8e5ccb58d896451dd9a662a511a3188ff6a8a6a919142ecc20"}, + {file = "mypy-0.982-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d6af646bd46f10d53834a8e8983e130e47d8ab2d4b7a97363e35b24e1d588947"}, + {file = "mypy-0.982-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:e7aeaa763c7ab86d5b66ff27f68493d672e44c8099af636d433a7f3fa5596d40"}, + {file = "mypy-0.982-cp37-cp37m-win_amd64.whl", hash = "sha256:724d36be56444f569c20a629d1d4ee0cb0ad666078d59bb84f8f887952511ca1"}, + {file = "mypy-0.982-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:14d53cdd4cf93765aa747a7399f0961a365bcddf7855d9cef6306fa41de01c24"}, + {file = "mypy-0.982-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:26ae64555d480ad4b32a267d10cab7aec92ff44de35a7cd95b2b7cb8e64ebe3e"}, + {file = "mypy-0.982-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:6389af3e204975d6658de4fb8ac16f58c14e1bacc6142fee86d1b5b26aa52bda"}, + {file = "mypy-0.982-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7b35ce03a289480d6544aac85fa3674f493f323d80ea7226410ed065cd46f206"}, + {file = "mypy-0.982-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:c6e564f035d25c99fd2b863e13049744d96bd1947e3d3d2f16f5828864506763"}, + {file = "mypy-0.982-cp38-cp38-win_amd64.whl", hash = "sha256:cebca7fd333f90b61b3ef7f217ff75ce2e287482206ef4a8b18f32b49927b1a2"}, + {file = "mypy-0.982-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:a705a93670c8b74769496280d2fe6cd59961506c64f329bb179970ff1d24f9f8"}, + {file = "mypy-0.982-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:75838c649290d83a2b83a88288c1eb60fe7a05b36d46cbea9d22efc790002146"}, + {file = "mypy-0.982-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:91781eff1f3f2607519c8b0e8518aad8498af1419e8442d5d0afb108059881fc"}, + {file = "mypy-0.982-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eaa97b9ddd1dd9901a22a879491dbb951b5dec75c3b90032e2baa7336777363b"}, + {file = "mypy-0.982-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:a692a8e7d07abe5f4b2dd32d731812a0175626a90a223d4b58f10f458747dd8a"}, + {file = "mypy-0.982-cp39-cp39-win_amd64.whl", hash = "sha256:eb7a068e503be3543c4bd329c994103874fa543c1727ba5288393c21d912d795"}, + {file = "mypy-0.982-py3-none-any.whl", hash = "sha256:1021c241e8b6e1ca5a47e4d52601274ac078a89845cfde66c6d5f769819ffa1d"}, + {file = "mypy-0.982.tar.gz", hash = "sha256:85f7a343542dc8b1ed0a888cdd34dca56462654ef23aa673907305b260b3d746"}, ] mypy-extensions = [ {file = "mypy_extensions-0.4.3-py2.py3-none-any.whl", hash = "sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d"}, @@ -1465,8 +1465,8 @@ pytest-github-actions-annotate-failures = [ {file = "pytest_github_actions_annotate_failures-0.1.7-py2.py3-none-any.whl", hash = "sha256:c4a7346d1d95f731a6b53e9a45f10ca56593978149266dd7526876cce403ea38"}, ] pytest-mock = [ - {file = "pytest-mock-3.9.0.tar.gz", hash = "sha256:c899a0dcc8a5f22930acd020b500abd5f956911f326864a3b979e4866e14da82"}, - {file = "pytest_mock-3.9.0-py3-none-any.whl", hash = "sha256:1a1b9264224d026932d6685a0f9cef3b61d91563c3e74af9fe5afb2767e13812"}, + {file = "pytest-mock-3.10.0.tar.gz", hash = "sha256:fbbdb085ef7c252a326fd8cdcac0aa3b1333d8811f131bdcc701002e1be7ed4f"}, + {file = "pytest_mock-3.10.0-py3-none-any.whl", hash = "sha256:f4c973eeae0282963eb293eb173ce91b091a79c1334455acfac9ddee8a1c784b"}, ] pytest-randomly = [ {file = "pytest-randomly-3.12.0.tar.gz", hash = "sha256:d60c2db71ac319aee0fc6c4110a7597d611a8b94a5590918bfa8583f00caccb2"}, @@ -1597,16 +1597,16 @@ types-jsonschema = [ {file = "types_jsonschema-4.16.1-py3-none-any.whl", hash = "sha256:21ca9a227185b83655c71755b5834c36d66ca43f9de77c018d61c4f917f851ab"}, ] types-requests = [ - {file = "types-requests-2.28.11.tar.gz", hash = "sha256:7ee827eb8ce611b02b5117cfec5da6455365b6a575f5e3ff19f655ba603e6b4e"}, - {file = "types_requests-2.28.11-py3-none-any.whl", hash = "sha256:af5f55e803cabcfb836dad752bd6d8a0fc8ef1cd84243061c0e27dee04ccf4fd"}, + {file = "types-requests-2.28.11.2.tar.gz", hash = "sha256:fdcd7bd148139fb8eef72cf4a41ac7273872cad9e6ada14b11ff5dfdeee60ed3"}, + {file = "types_requests-2.28.11.2-py3-none-any.whl", hash = "sha256:14941f8023a80b16441b3b46caffcbfce5265fd14555844d6029697824b5a2ef"}, ] types-urllib3 = [ {file = "types-urllib3-1.26.25.tar.gz", hash = "sha256:5aef0e663724eef924afa8b320b62ffef2c1736c1fa6caecfc9bc6c8ae2c3def"}, {file = "types_urllib3-1.26.25-py3-none-any.whl", hash = "sha256:c1d78cef7bd581e162e46c20a57b2e1aa6ebecdcf01fd0713bb90978ff3e3427"}, ] typing-extensions = [ - {file = "typing_extensions-4.3.0-py3-none-any.whl", hash = "sha256:25642c956049920a5aa49edcdd6ab1e06d7e5d467fc00e0506c44ac86fbfca02"}, - {file = "typing_extensions-4.3.0.tar.gz", hash = "sha256:e6d2677a32f47fc7eb2795db1dd15c1f34eff616bcaf2cfb5e997f854fa1c4a6"}, + {file = "typing_extensions-4.4.0-py3-none-any.whl", hash = "sha256:16fa4864408f655d35ec496218b85f79b3437c829e93320c7c9215ccfd92489e"}, + {file = "typing_extensions-4.4.0.tar.gz", hash = "sha256:1511434bb92bf8dd198c12b1cc812e800d4181cfcb867674e0f8279cc93087aa"}, ] urllib3 = [ {file = "urllib3-1.26.12-py2.py3-none-any.whl", hash = "sha256:b930dd878d5a8afb066a637fbb35144fe7901e3b209d1cd4f524bd0e9deee997"}, @@ -1678,6 +1678,6 @@ xattr = [ {file = "xattr-0.9.9.tar.gz", hash = "sha256:09cb7e1efb3aa1b4991d6be4eb25b73dc518b4fe894f0915f5b0dcede972f346"}, ] zipp = [ - {file = "zipp-3.8.1-py3-none-any.whl", hash = "sha256:47c40d7fe183a6f21403a199b3e4192cca5774656965b0a4988ad2f8feb5f009"}, - {file = "zipp-3.8.1.tar.gz", hash = "sha256:05b45f1ee8f807d0cc928485ca40a07cb491cf092ff587c0df9cb1fd154848d2"}, + {file = "zipp-3.9.0-py3-none-any.whl", hash = "sha256:972cfa31bc2fedd3fa838a51e9bc7e64b7fb725a8c00e7431554311f180e9980"}, + {file = "zipp-3.9.0.tar.gz", hash = "sha256:3a7af91c3db40ec72dd9d154ae18e008c69efe8ca88dde4f9a731bb82fe2f9eb"}, ] diff --git a/pyproject.toml b/pyproject.toml index b2280b59eac..ecf47ae886d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,7 +51,6 @@ poetry-core = "^1.3.2" poetry-plugin-export = "^1.1.2" "backports.cached-property" = { version = "^1.0.2", python = "<3.8" } cachecontrol = { version = "^0.12.9", extras = ["filecache"] } -cachy = "^0.3.0" cleo = "^1.0.0a5" crashtest = "^0.3.0" dulwich = "^0.20.46" @@ -80,6 +79,8 @@ urllib3 = "^1.26.0" pre-commit = "^2.6" [tool.poetry.group.test.dependencies] +# Cachy frozen to test backwards compatibility for `poetry.utils.cache`. +cachy = "0.3.0" deepdiff = "^5.0" flatdict = "^4.0.1" httpretty = "^1.0" diff --git a/src/poetry/console/commands/cache/clear.py b/src/poetry/console/commands/cache/clear.py index 8faf7a4c702..9896efd43d1 100644 --- a/src/poetry/console/commands/cache/clear.py +++ b/src/poetry/console/commands/cache/clear.py @@ -8,6 +8,7 @@ from poetry.config.config import Config from poetry.console.commands.command import Command +from poetry.utils.cache import FileCache class CacheClearCommand(Command): @@ -18,8 +19,6 @@ class CacheClearCommand(Command): options = [option("all", description="Clear all entries in the cache.")] def handle(self) -> int: - from cachy import CacheManager - cache = self.argument("cache") parts = cache.split(":") @@ -33,13 +32,7 @@ def handle(self) -> int: except ValueError: raise ValueError(f"{root} is not a valid repository cache") - cache = CacheManager( - { - "default": parts[0], - "serializer": "json", - "stores": {parts[0]: {"driver": "file", "path": str(cache_dir)}}, - } - ) + cache = FileCache(cache_dir) if len(parts) == 1: if not self.option("all"): diff --git a/src/poetry/repositories/cached.py b/src/poetry/repositories/cached.py index 59d6ba5fb71..b593feb44a2 100644 --- a/src/poetry/repositories/cached.py +++ b/src/poetry/repositories/cached.py @@ -5,12 +5,12 @@ from typing import TYPE_CHECKING from typing import Any -from cachy import CacheManager from packaging.utils import canonicalize_name from poetry.core.constraints.version import parse_constraint from poetry.config.config import Config from poetry.repositories.repository import Repository +from poetry.utils.cache import FileCache if TYPE_CHECKING: @@ -30,17 +30,7 @@ def __init__( super().__init__(name) self._disable_cache = disable_cache self._cache_dir = (config or Config.create()).repository_cache_directory / name - self._cache = CacheManager( - { - "default": "releases", - "serializer": "json", - "stores": { - "releases": {"driver": "file", "path": str(self._cache_dir)}, - "packages": {"driver": "dict"}, - "matches": {"driver": "dict"}, - }, - } - ) + self._release_cache: FileCache[dict[str, Any]] = FileCache(path=self._cache_dir) @abstractmethod def _get_release_info( @@ -60,7 +50,7 @@ def get_release_info(self, name: NormalizedName, version: Version) -> PackageInf if self._disable_cache: return PackageInfo.load(self._get_release_info(name, version)) - cached = self._cache.remember_forever( + cached = self._release_cache.remember( f"{name}:{version}", lambda: self._get_release_info(name, version) ) @@ -73,7 +63,7 @@ def get_release_info(self, name: NormalizedName, version: Version) -> PackageInf ) cached = self._get_release_info(name, version) - self._cache.forever(f"{name}:{version}", cached) + self._release_cache.put(f"{name}:{version}", cached) return PackageInfo.load(cached) diff --git a/src/poetry/repositories/legacy_repository.py b/src/poetry/repositories/legacy_repository.py index 4823a0b71fd..9eaf286928b 100644 --- a/src/poetry/repositories/legacy_repository.py +++ b/src/poetry/repositories/legacy_repository.py @@ -90,23 +90,19 @@ def _find_packages( if not constraint.is_any(): key = f"{key}:{constraint!s}" - if self._cache.store("matches").has(key): - versions = self._cache.store("matches").get(key) - else: - page = self._get_page(f"/{name}/") - if page is None: - self._log( - f"No packages found for {name}", - level="debug", - ) - return [] - - versions = [ - (version, page.yanked(name, version)) - for version in page.versions(name) - if constraint.allows(version) - ] - self._cache.store("matches").put(key, versions, 5) + page = self._get_page(f"/{name}/") + if page is None: + self._log( + f"No packages found for {name}", + level="debug", + ) + return [] + + versions = [ + (version, page.yanked(name, version)) + for version in page.versions(name) + if constraint.allows(version) + ] return [ Package( diff --git a/src/poetry/repositories/pypi_repository.py b/src/poetry/repositories/pypi_repository.py index 491f6e03179..8846b417739 100644 --- a/src/poetry/repositories/pypi_repository.py +++ b/src/poetry/repositories/pypi_repository.py @@ -100,13 +100,7 @@ def get_package_info(self, name: NormalizedName) -> dict[str, Any]: The information is returned from the cache if it exists or retrieved from the remote server. """ - if self._disable_cache: - return self._get_package_info(name) - - package_info: dict[str, Any] = self._cache.store("packages").remember_forever( - name, lambda: self._get_package_info(name) - ) - return package_info + return self._get_package_info(name) def _find_packages( self, name: NormalizedName, constraint: VersionConstraint @@ -129,15 +123,11 @@ def _find_packages( if not constraint.is_any(): key = f"{key}:{constraint!s}" - if self._cache.store("matches").has(key): - versions = self._cache.store("matches").get(key) - else: - versions = [ - (version, json_page.yanked(name, version)) - for version in json_page.versions(name) - if constraint.allows(version) - ] - self._cache.store("matches").put(key, versions, 5) + versions = [ + (version, json_page.yanked(name, version)) + for version in json_page.versions(name) + if constraint.allows(version) + ] pretty_name = json_page.content["name"] packages = [ diff --git a/tests/console/commands/cache/test_clear.py b/tests/console/commands/cache/test_clear.py index a8fd85c876c..fb8900d8ebb 100644 --- a/tests/console/commands/cache/test_clear.py +++ b/tests/console/commands/cache/test_clear.py @@ -30,11 +30,12 @@ def test_cache_clear_all( cache: CacheManager, ): exit_code = tester.execute(f"cache clear {repository_one} --all", inputs="yes") + repository_one_dir = repository_cache_dir / repository_one assert exit_code == 0 assert tester.io.fetch_output() == "" - # ensure directory is empty - assert not any((repository_cache_dir / repository_one).iterdir()) + # ensure directory is empty or doesn't exist + assert not repository_one_dir.exists() or not any(repository_one_dir.iterdir()) assert not cache.has("cachy:0.1") assert not cache.has("cleo:0.2") diff --git a/tests/console/commands/test_add.py b/tests/console/commands/test_add.py index 7947d209d8c..ff34c4f8b20 100644 --- a/tests/console/commands/test_add.py +++ b/tests/console/commands/test_add.py @@ -8,6 +8,7 @@ import pytest from poetry.core.constraints.version import Version +from poetry.core.packages.package import Package from poetry.repositories.legacy_repository import LegacyRepository from tests.helpers import get_dependency @@ -819,12 +820,26 @@ def test_add_constraint_with_platform( def test_add_constraint_with_source( - app: PoetryTestApplication, poetry: Poetry, tester: CommandTester + app: PoetryTestApplication, + poetry: Poetry, + tester: CommandTester, + mocker: MockerFixture, ): repo = LegacyRepository(name="my-index", url="https://my-index.fake") repo.add_package(get_package("cachy", "0.2.0")) - repo._cache.store("matches").put( - "cachy:0.2.0", [(Version.parse("0.2.0"), False)], 5 + mocker.patch.object( + repo, + "_find_packages", + wraps=lambda _, name: [ + Package( + "cachy", + Version.parse("0.2.0"), + source_type="legacy", + source_reference=repo.name, + source_url=repo._url, + yanked=False, + ) + ], ) poetry.pool.add_repository(repo) @@ -1809,11 +1824,23 @@ def test_add_constraint_with_source_old_installer( poetry: Poetry, installer: NoopInstaller, old_tester: CommandTester, + mocker: MockerFixture, ): repo = LegacyRepository(name="my-index", url="https://my-index.fake") repo.add_package(get_package("cachy", "0.2.0")) - repo._cache.store("matches").put( - "cachy:0.2.0", [(Version.parse("0.2.0"), False)], 5 + mocker.patch.object( + repo, + "_find_packages", + wraps=lambda _, name: [ + Package( + "cachy", + Version.parse("0.2.0"), + source_type="legacy", + source_reference=repo.name, + source_url=repo._url, + yanked=False, + ) + ], ) poetry.pool.add_repository(repo) From 2c190ab1dc6e6a245f85f4ec3990762a0dab9a8e Mon Sep 17 00:00:00 2001 From: Chad Crawford Date: Tue, 11 Oct 2022 14:58:01 -0700 Subject: [PATCH 22/25] fix: Add post init check for `FileCache` hash type. --- src/poetry/utils/cache.py | 6 ++++++ tests/utils/test_cache.py | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/src/poetry/utils/cache.py b/src/poetry/utils/cache.py index 9eb7872b09f..ba88a077055 100644 --- a/src/poetry/utils/cache.py +++ b/src/poetry/utils/cache.py @@ -100,6 +100,12 @@ class FileCache(Generic[T]): path: Path hash_type: str = "sha256" + def __post_init__(self) -> None: + if self.hash_type not in _HASHES: + raise ValueError( + f"FileCache.hash_type is unknown value: '{self.hash_type}'." + ) + def get(self, key: str) -> T | None: return self._get_payload(key) diff --git a/tests/utils/test_cache.py b/tests/utils/test_cache.py index 21079ee42e0..c1bbae5071a 100644 --- a/tests/utils/test_cache.py +++ b/tests/utils/test_cache.py @@ -85,6 +85,12 @@ def cachy_dict_cache() -> CacheManager: return patch_cachy(cache) +def test_cache_validates(repository_cache_dir: Path) -> None: + with pytest.raises(ValueError) as e: + FileCache(repository_cache_dir / "cache", hash_type="unknown") + assert str(e.value) == "FileCache.hash_type is unknown value: 'unknown'." + + @pytest.mark.parametrize("cache_name", ["cachy_file_cache", "poetry_file_cache"]) def test_cache_get_put_has(cache_name: str, request: FixtureRequest) -> None: cache = request.getfixturevalue(cache_name) From 0ffe91c0c8e2edde987cf2213d402614a9c2a6a6 Mon Sep 17 00:00:00 2001 From: Mathieu Kniewallner Date: Wed, 12 Oct 2022 02:50:47 +0200 Subject: [PATCH 23/25] ci: upgrade `pip` for 3.11 on all OSes --- .github/workflows/main.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 225af8ebc64..c4e363a2d8f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -84,9 +84,9 @@ jobs: timeout 10s poetry run pip --version || rm -rf .venv # XXX: https://github.com/pypa/pip/issues/11352 causes random failures -- remove once fixed in a release. - - name: Upgrade pip on 3.11 for macOS - if: ${{ matrix.python-version == '3.11-dev' && matrix.os == 'macOS' }} - run: poetry run pip install git+https://github.com/pypa/pip.git@fbb7f0b293f1d9289d8c76a556540325b9a172b2 + - name: Upgrade pip on Python 3.11 + if: ${{ matrix.python-version == '3.11-dev' }} + run: poetry run pip install git+https://github.com/pypa/pip.git@f8a25921e5c443b07483017b0ffdeb08b9ba2fdf - name: Install dependencies run: poetry install --with github-actions From 0b71e4381445bf6b6c239bf0d88c8f5dfb4a32bd Mon Sep 17 00:00:00 2001 From: miles Date: Wed, 12 Oct 2022 21:57:52 +0800 Subject: [PATCH 24/25] fix(git): `experimental.system-git-client` can't be set using environment variable Resolves: #6722 --- src/poetry/vcs/git/backend.py | 4 ++-- tests/integration/test_utils_vcs_git.py | 7 +++++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/poetry/vcs/git/backend.py b/src/poetry/vcs/git/backend.py index 4eba0b37d60..7b37d259690 100644 --- a/src/poetry/vcs/git/backend.py +++ b/src/poetry/vcs/git/backend.py @@ -366,8 +366,8 @@ def _clone_submodules(cls, repo: Repo) -> None: def is_using_legacy_client() -> bool: from poetry.config.config import Config - legacy_client: bool = ( - Config.create().get("experimental", {}).get("system-git-client", False) + legacy_client: bool = Config.create().get( + "experimental.system-git-client", False ) return legacy_client diff --git a/tests/integration/test_utils_vcs_git.py b/tests/integration/test_utils_vcs_git.py index 334ae04a268..1ca7ea0c843 100644 --- a/tests/integration/test_utils_vcs_git.py +++ b/tests/integration/test_utils_vcs_git.py @@ -111,6 +111,13 @@ def remote_default_branch(remote_default_ref: bytes) -> str: return remote_default_ref.decode("utf-8").replace("refs/heads/", "") +# Regression test for https://github.com/python-poetry/poetry/issues/6722 +def test_use_system_git_client_from_environment_variables(): + os.environ["POETRY_EXPERIMENTAL_SYSTEM_GIT_CLIENT"] = "true" + + assert Git.is_using_legacy_client() + + def test_git_local_info( source_url: str, remote_refs: FetchPackResult, remote_default_ref: bytes ) -> None: From 16046d9ac9b72a49e1bc4618fb686695cc64821c Mon Sep 17 00:00:00 2001 From: doolio Date: Thu, 13 Oct 2022 15:04:35 +0200 Subject: [PATCH 25/25] docs(intro): Add missing dash to command enabling completion in bash (#6793) --- docs/_index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/_index.md b/docs/_index.md index 450505dd3f4..c383fdac017 100644 --- a/docs/_index.md +++ b/docs/_index.md @@ -367,7 +367,7 @@ poetry completions bash >> ~/.bash_completion #### Lazy-loaded ```bash -poetry completions bash > ${XDG_DATA_HOME:~/.local/share}/bash-completion/completions/poetry +poetry completions bash > ${XDG_DATA_HOME:-~/.local/share}/bash-completion/completions/poetry ``` ### Fish