Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add --platform option to facilitate installing wheels for platforms other than the host system #123

Open
wants to merge 20 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 18 additions & 1 deletion src/poetry_plugin_bundle/bundlers/venv_bundler.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
from cleo.io.outputs.section_output import SectionOutput
from poetry.poetry import Poetry
from poetry.repositories.lockfile_repository import LockfileRepository
from poetry.utils.env import Env


class VenvBundler(Bundler):
Expand All @@ -23,6 +24,7 @@ def __init__(self) -> None:
self._remove: bool = False
self._activated_groups: set[str] | None = None
self._compile: bool = False
self._platform: str | None = None

def set_path(self, path: Path) -> VenvBundler:
self._path = path
Expand All @@ -49,6 +51,11 @@ def set_compile(self, compile: bool = False) -> VenvBundler:

return self

def set_platform(self, platform: str | None) -> VenvBundler:
self._platform = platform

return self

def bundle(self, poetry: Poetry, io: IO) -> bool:
from pathlib import Path
from tempfile import TemporaryDirectory
Expand All @@ -67,7 +74,6 @@ def bundle(self, poetry: Poetry, io: IO) -> bool:
from poetry.installation.installer import Installer
from poetry.installation.operations.install import Install
from poetry.packages.locker import Locker
from poetry.utils.env import Env
from poetry.utils.env import EnvManager
from poetry.utils.env import InvalidCurrentPythonVersionError

Expand Down Expand Up @@ -138,6 +144,9 @@ def create_venv_at_path(
self._path, executable=executable, force=True
)

if self._platform:
self.constrain_env_platform(env, self._platform)

self._write(io, f"{message}: <info>Installing dependencies</info>")

class CustomLocker(Locker):
Expand Down Expand Up @@ -248,3 +257,11 @@ def _write(self, io: IO | SectionOutput, message: str) -> None:
return

io.overwrite(message)

def constrain_env_platform(self, env: Env, platform: str) -> None:
"""
Set the argument environment's supported tags based on the configured platform override.
"""
from poetry_plugin_bundle.utils.platforms import create_supported_tags

env._supported_tags = create_supported_tags(platform, env)
9 changes: 9 additions & 0 deletions src/poetry_plugin_bundle/console/commands/bundle/venv.py
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,14 @@ class BundleVenvCommand(BundleCommand):
" because the old installer always compiles.)",
flag=True,
),
option(
"platform",
None,
"Only use wheels compatible with the specified platform. Otherwise the default behavior uses the platform "
" of the running system.",
flag=False,
value_required=True,
),
]

bundler_name = "venv"
Expand All @@ -54,4 +62,5 @@ def configure_bundler(self, bundler: VenvBundler) -> None: # type: ignore[overr
bundler.set_executable(self.option("python"))
bundler.set_remove(self.option("clear"))
bundler.set_compile(self.option("compile"))
bundler.set_platform(self.option("platform"))
bundler.set_activated_groups(self.activated_groups)
Empty file.
157 changes: 157 additions & 0 deletions src/poetry_plugin_bundle/utils/platforms.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,157 @@
from __future__ import annotations

from dataclasses import dataclass
from typing import TYPE_CHECKING


if TYPE_CHECKING:
from packaging.tags import Tag
from poetry.utils.env import Env


@dataclass
class PlatformTagParseResult:
platform: str
version_major: int
version_minor: int
arch: str

@staticmethod
def parse(tag: str) -> PlatformTagParseResult:
import re

match = re.match("([a-z]+)_([0-9]+)_([0-9]+)_(.*)", tag)
if not match:
raise ValueError(f"Invalid platform tag: {tag}")
platform, version_major_str, version_minor_str, arch = match.groups()
return PlatformTagParseResult(
platform=platform,
version_major=int(version_major_str),
version_minor=int(version_minor_str),
arch=arch,
)

def to_tag(self) -> str:
return "_".join(
[self.platform, str(self.version_major), str(self.version_minor), self.arch]
)


def create_supported_tags(platform: str, env: Env) -> list[Tag]:
"""
Given a platform specifier string, generate a list of compatible tags for the argument environment's interpreter.

Refer to:
https://packaging.python.org/en/latest/specifications/platform-compatibility-tags/#platform-tag
https://pip.pypa.io/en/stable/cli/pip_install/#cmdoption-platform
"""
from packaging.tags import INTERPRETER_SHORT_NAMES
from packaging.tags import compatible_tags
from packaging.tags import cpython_tags
from packaging.tags import generic_tags

if platform.startswith("manylinux"):
supported_platforms = create_supported_manylinux_platforms(platform)
elif platform.startswith("musllinux"):
supported_platforms = create_supported_musllinux_platforms(platform)
elif platform.startswith("macosx"):
supported_platforms = create_supported_macosx_platforms(platform)
else:
raise NotImplementedError(f"Platform {platform} not supported")

python_implementation = env.python_implementation.lower()
python_version = env.version_info[:2]
interpreter_name = INTERPRETER_SHORT_NAMES.get(
python_implementation, python_implementation
)
interpreter = None

if interpreter_name == "cp":
tags = list(
cpython_tags(python_version=python_version, platforms=supported_platforms)
)
interpreter = f"{interpreter_name}{python_version[0]}{python_version[1]}"
else:
tags = list(
generic_tags(
interpreter=interpreter, abis=[], platforms=supported_platforms
)
)

tags.extend(
compatible_tags(
interpreter=interpreter,
python_version=python_version,
platforms=supported_platforms,
)
)

return tags


def create_supported_manylinux_platforms(platform: str) -> list[str]:
"""
https://peps.python.org/pep-0600/
manylinux_${GLIBCMAJOR}_${GLIBCMINOR}_${ARCH}

For now, only GLIBCMAJOR "2" is supported. It is unclear if there will be a need to support a future major
version like "3" and if specified, how generate the compatible 2.x version tags.
"""
# Implementation based on https://peps.python.org/pep-0600/#package-installers

tag = normalize_legacy_manylinux_alias(platform)

parsed = PlatformTagParseResult.parse(tag)
return [
f"{parsed.platform}_{parsed.version_major}_{tag_minor}_{parsed.arch}"
for tag_minor in range(parsed.version_minor, -1, -1)
]


LEGACY_MANYLINUX_ALIASES = {
"manylinux1": "manylinux_2_5",
"manylinux2010": "manylinux_2_12",
"manylinux2014": "manylinux_2_17",
}


def normalize_legacy_manylinux_alias(tag: str) -> str:
tag_os_index_end = tag.index("_")
tag_os = tag[:tag_os_index_end]
tag_arch_suffix = tag[tag_os_index_end:]
os_replacement = LEGACY_MANYLINUX_ALIASES.get(tag_os)
if not os_replacement:
return tag

return os_replacement + tag_arch_suffix


def create_supported_macosx_platforms(platform: str) -> list[str]:
import re

from packaging.tags import mac_platforms

match = re.match("macosx_([0-9]+)_([0-9]+)_(.*)", platform)
if not match:
raise ValueError(f"Invalid macosx tag: {platform}")
tag_major_str, tag_minor_str, tag_arch = match.groups()
tag_major_max = int(tag_major_str)
tag_minor_max = int(tag_minor_str)

return list(mac_platforms(version=(tag_major_max, tag_minor_max), arch=tag_arch))


def create_supported_musllinux_platforms(platform: str) -> list[str]:
import re

match = re.match("musllinux_([0-9]+)_([0-9]+)_(.*)", platform)
if not match:
raise ValueError(f"Invalid musllinux tag: {platform}")
tag_major_str, tag_minor_str, tag_arch = match.groups()
tag_major_max = int(tag_major_str)
tag_minor_max = int(tag_minor_str)

return [
f"musllinux_{tag_major_max}_{minor}_{tag_arch}"
for minor in range(tag_minor_max, -1, -1)
]
77 changes: 77 additions & 0 deletions tests/bundlers/test_venv_bundler.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
from cleo.formatters.style import Style
from cleo.io.buffered_io import BufferedIO
from poetry.core.packages.package import Package
from poetry.core.packages.utils.link import Link
from poetry.factory import Factory
from poetry.installation.operations.install import Install
from poetry.puzzle.exceptions import SolverProblemError
Expand Down Expand Up @@ -422,3 +423,79 @@ def test_bundler_non_package_mode(
• Bundled simple-project-non-package-mode (1.2.3) into {path}
"""
assert expected == io.fetch_output()


def test_bundler_platform_override(
io: BufferedIO, tmpdir: str, mocker: MockerFixture, config: Config
) -> None:
poetry = Factory().create_poetry(
Path(__file__).parent.parent / "fixtures" / "project_with_binary_wheel"
)
poetry.set_config(config)

def get_links_fake(package: Package) -> list[Link]:
return [Link(f"https://example.com/{file['file']}") for file in package.files]

mocker.patch(
"poetry.installation.chooser.Chooser._get_links", side_effect=get_links_fake
)
mocker.patch("poetry.installation.executor.Executor._execute_uninstall")
mocker.patch("poetry.installation.executor.Executor._execute_update")
mock_download_link = mocker.patch(
"poetry.installation.executor.Executor._download_link"
)
mocker.patch("poetry.installation.wheel_installer.WheelInstaller.install")

def get_installed_links() -> dict[str, str]:
return {
call[0][0].package.name: call[0][1].filename
for call in mock_download_link.call_args_list
}

bundler = VenvBundler()
bundler.set_path(Path(tmpdir))
bundler.set_remove(True)

bundler.set_platform("manylinux_2_28_x86_64")
bundler.bundle(poetry, io)
installed_link_by_package = get_installed_links()
assert "manylinux_2_28_x86_64" in installed_link_by_package["cryptography"]
assert "manylinux_2_17_x86_64" in installed_link_by_package["cffi"]
assert "py3-none-any.whl" in installed_link_by_package["pycparser"]

bundler.set_platform("manylinux2014_x86_64")
bundler.bundle(poetry, io)
installed_link_by_package = get_installed_links()
assert "manylinux2014_x86_64" in installed_link_by_package["cryptography"]
assert "manylinux_2_17_x86_64" in installed_link_by_package["cffi"]
assert "py3-none-any.whl" in installed_link_by_package["pycparser"]

bundler.set_platform("macosx_10_9_x86_64")
bundler.bundle(poetry, io)
installed_link_by_package = get_installed_links()
assert "macosx_10_9_universal2" in installed_link_by_package["cryptography"]
expected_cffi_platform = (
"macosx_10_9_x86_64" if sys.version_info < (3, 13) else "cffi-1.17.1.tar.gz"
)
assert expected_cffi_platform in installed_link_by_package["cffi"]
assert "py3-none-any.whl" in installed_link_by_package["pycparser"]

bundler.set_platform("macosx_11_0_arm64")
bundler.bundle(poetry, io)
installed_link_by_package = get_installed_links()
assert "macosx_10_9_universal2" in installed_link_by_package["cryptography"]
expected_cffi_platform = (
"macosx_11_0_arm64" if sys.version_info >= (3, 9) else "cffi-1.17.1.tar.gz"
)
assert expected_cffi_platform in installed_link_by_package["cffi"]
assert "py3-none-any.whl" in installed_link_by_package["pycparser"]

bundler.set_platform("musllinux_1_2_aarch64")
bundler.bundle(poetry, io)
installed_link_by_package = get_installed_links()
assert "musllinux_1_2_aarch64" in installed_link_by_package["cryptography"]
expected_cffi_platform = (
"musllinux_1_1_aarch64" if sys.version_info >= (3, 9) else "cffi-1.17.1.tar.gz"
)
assert expected_cffi_platform in installed_link_by_package["cffi"]
assert "py3-none-any.whl" in installed_link_by_package["pycparser"]
Empty file.
Loading
Loading