diff --git a/news/6908.removal b/news/6908.removal new file mode 100644 index 00000000000..aca9d590ab0 --- /dev/null +++ b/news/6908.removal @@ -0,0 +1,2 @@ +Remove wheel tag calculation from pip and use ``packaging.tags``. This +should provide more tags ordered better than in prior releases. diff --git a/src/pip/_internal/pep425tags.py b/src/pip/_internal/pep425tags.py index 7b7a3551b71..a2386ee75b8 100644 --- a/src/pip/_internal/pep425tags.py +++ b/src/pip/_internal/pep425tags.py @@ -1,235 +1,37 @@ """Generate and work with PEP 425 Compatibility Tags.""" from __future__ import absolute_import -import distutils.util import logging -import platform import re -import sys -import sysconfig from pip._vendor.packaging.tags import ( Tag, + compatible_tags, + cpython_tags, + generic_tags, interpreter_name, interpreter_version, mac_platforms, ) -from pip._vendor.six import PY2 -import pip._internal.utils.glibc from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: - from typing import ( - Callable, List, Optional, Tuple, Union - ) + from typing import List, Optional, Tuple + + from pip._vendor.packaging.tags import PythonVersion logger = logging.getLogger(__name__) _osx_arch_pat = re.compile(r'(.+)_(\d+)_(\d+)_(.+)') -def get_config_var(var): - # type: (str) -> Optional[str] - return sysconfig.get_config_var(var) - - def version_info_to_nodot(version_info): # type: (Tuple[int, ...]) -> str # Only use up to the first two numbers. return ''.join(map(str, version_info[:2])) -def get_impl_version_info(): - # type: () -> Tuple[int, ...] - """Return sys.version_info-like tuple for use in decrementing the minor - version.""" - if interpreter_name() == 'pp': - # as per https://github.com/pypa/pip/issues/2882 - # attrs exist only on pypy - return (sys.version_info[0], - sys.pypy_version_info.major, # type: ignore - sys.pypy_version_info.minor) # type: ignore - else: - return sys.version_info[0], sys.version_info[1] - - -def get_flag(var, fallback, expected=True, warn=True): - # type: (str, Callable[..., bool], Union[bool, int], bool) -> bool - """Use a fallback method for determining SOABI flags if the needed config - var is unset or unavailable.""" - val = get_config_var(var) - if val is None: - if warn: - logger.debug("Config variable '%s' is unset, Python ABI tag may " - "be incorrect", var) - return fallback() - return val == expected - - -def get_abi_tag(): - # type: () -> Optional[str] - """Return the ABI tag based on SOABI (if available) or emulate SOABI - (CPython 2, PyPy).""" - soabi = get_config_var('SOABI') - impl = interpreter_name() - abi = None # type: Optional[str] - - if not soabi and impl in {'cp', 'pp'} and hasattr(sys, 'maxunicode'): - d = '' - m = '' - u = '' - is_cpython = (impl == 'cp') - if get_flag( - 'Py_DEBUG', lambda: hasattr(sys, 'gettotalrefcount'), - warn=is_cpython): - d = 'd' - if sys.version_info < (3, 8) and get_flag( - 'WITH_PYMALLOC', lambda: is_cpython, warn=is_cpython): - m = 'm' - if sys.version_info < (3, 3) and get_flag( - 'Py_UNICODE_SIZE', lambda: sys.maxunicode == 0x10ffff, - expected=4, warn=is_cpython): - u = 'u' - abi = '%s%s%s%s%s' % (impl, interpreter_version(), d, m, u) - elif soabi and soabi.startswith('cpython-'): - abi = 'cp' + soabi.split('-')[1] - elif soabi: - abi = soabi.replace('.', '_').replace('-', '_') - - return abi - - -def _is_running_32bit(): - # type: () -> bool - return sys.maxsize == 2147483647 - - -def get_platform(): - # type: () -> str - """Return our platform name 'win32', 'linux_x86_64'""" - if sys.platform == 'darwin': - # distutils.util.get_platform() returns the release based on the value - # of MACOSX_DEPLOYMENT_TARGET on which Python was built, which may - # be significantly older than the user's current machine. - release, _, machine = platform.mac_ver() - split_ver = release.split('.') - - if machine == "x86_64" and _is_running_32bit(): - machine = "i386" - elif machine == "ppc64" and _is_running_32bit(): - machine = "ppc" - - return 'macosx_{}_{}_{}'.format(split_ver[0], split_ver[1], machine) - - # XXX remove distutils dependency - result = distutils.util.get_platform().replace('.', '_').replace('-', '_') - if result == "linux_x86_64" and _is_running_32bit(): - # 32 bit Python program (running on a 64 bit Linux): pip should only - # install and run 32 bit compiled extensions in that case. - result = "linux_i686" - - return result - - -def is_linux_armhf(): - # type: () -> bool - if get_platform() != "linux_armv7l": - return False - # hard-float ABI can be detected from the ELF header of the running - # process - try: - with open(sys.executable, 'rb') as f: - elf_header_raw = f.read(40) # read 40 first bytes of ELF header - except (IOError, OSError, TypeError): - return False - if elf_header_raw is None or len(elf_header_raw) < 40: - return False - if isinstance(elf_header_raw, str): - elf_header = [ord(c) for c in elf_header_raw] - else: - elf_header = [b for b in elf_header_raw] - result = elf_header[0:4] == [0x7f, 0x45, 0x4c, 0x46] # ELF magic number - result &= elf_header[4:5] == [1] # 32-bit ELF - result &= elf_header[5:6] == [1] # little-endian - result &= elf_header[18:20] == [0x28, 0] # ARM machine - result &= elf_header[39:40] == [5] # ARM EABIv5 - result &= (elf_header[37:38][0] & 4) == 4 # EF_ARM_ABI_FLOAT_HARD - return result - - -def is_manylinux1_compatible(): - # type: () -> bool - # Only Linux, and only x86-64 / i686 - if get_platform() not in {"linux_x86_64", "linux_i686"}: - return False - - # Check for presence of _manylinux module - try: - import _manylinux - return bool(_manylinux.manylinux1_compatible) - except (ImportError, AttributeError): - # Fall through to heuristic check below - pass - - # Check glibc version. CentOS 5 uses glibc 2.5. - return pip._internal.utils.glibc.have_compatible_glibc(2, 5) - - -def is_manylinux2010_compatible(): - # type: () -> bool - # Only Linux, and only x86-64 / i686 - if get_platform() not in {"linux_x86_64", "linux_i686"}: - return False - - # Check for presence of _manylinux module - try: - import _manylinux - return bool(_manylinux.manylinux2010_compatible) - except (ImportError, AttributeError): - # Fall through to heuristic check below - pass - - # Check glibc version. CentOS 6 uses glibc 2.12. - return pip._internal.utils.glibc.have_compatible_glibc(2, 12) - - -def is_manylinux2014_compatible(): - # type: () -> bool - # Only Linux, and only supported architectures - platform = get_platform() - if platform not in {"linux_x86_64", "linux_i686", "linux_aarch64", - "linux_armv7l", "linux_ppc64", "linux_ppc64le", - "linux_s390x"}: - return False - - # check for hard-float ABI in case we're running linux_armv7l not to - # install hard-float ABI wheel in a soft-float ABI environment - if platform == "linux_armv7l" and not is_linux_armhf(): - return False - - # Check for presence of _manylinux module - try: - import _manylinux - return bool(_manylinux.manylinux2014_compatible) - except (ImportError, AttributeError): - # Fall through to heuristic check below - pass - - # Check glibc version. CentOS 7 uses glibc 2.17. - return pip._internal.utils.glibc.have_compatible_glibc(2, 17) - - -def get_all_minor_versions_as_strings(version_info): - # type: (Tuple[int, ...]) -> List[str] - versions = [] - major = version_info[:-1] - # Support all previous minor Python versions. - for minor in range(version_info[-1], -1, -1): - versions.append(''.join(map(str, major + (minor,)))) - return versions - - def _mac_platforms(arch): # type: (str) -> List[str] match = _osx_arch_pat.match(arch) @@ -273,27 +75,35 @@ def _custom_manylinux_platforms(arch): return arches -def _get_custom_platforms(arch, platform): - # type: (str, Optional[str]) -> List[str] +def _get_custom_platforms(arch): + # type: (str) -> List[str] arch_prefix, arch_sep, arch_suffix = arch.partition('_') if arch.startswith('macosx'): arches = _mac_platforms(arch) elif arch_prefix in ['manylinux2014', 'manylinux2010']: arches = _custom_manylinux_platforms(arch) - elif platform is None: - arches = [] - if is_manylinux2014_compatible(): - arches.append('manylinux2014' + arch_sep + arch_suffix) - if is_manylinux2010_compatible(): - arches.append('manylinux2010' + arch_sep + arch_suffix) - if is_manylinux1_compatible(): - arches.append('manylinux1' + arch_sep + arch_suffix) - arches.append(arch) else: arches = [arch] return arches +def _get_python_version(version): + # type: (str) -> PythonVersion + if len(version) > 1: + return int(version[0]), int(version[1:]) + else: + return (int(version[0]),) + + +def _get_custom_interpreter(implementation=None, version=None): + # type: (Optional[str], Optional[str]) -> str + if implementation is None: + implementation = interpreter_name() + if version is None: + version = interpreter_version() + return "{}{}".format(implementation, version) + + def get_supported( version=None, # type: Optional[str] platform=None, # type: Optional[str] @@ -313,59 +123,45 @@ def get_supported( :param abi: specify the exact abi you want valid tags for, or None. If None, use the local interpreter abi. """ - supported = [] - - # Versions must be given with respect to the preference - if version is None: - version_info = get_impl_version_info() - versions = get_all_minor_versions_as_strings(version_info) + supported = [] # type: List[Tag] + + python_version = None # type: Optional[PythonVersion] + if version is not None: + python_version = _get_python_version(version) + + interpreter = _get_custom_interpreter(impl, version) + + abis = None # type: Optional[List[str]] + if abi is not None: + abis = [abi] + + platforms = None # type: Optional[List[str]] + if platform is not None: + platforms = _get_custom_platforms(platform) + + is_cpython = (impl or interpreter_name()) == "cp" + if is_cpython: + supported.extend( + cpython_tags( + python_version=python_version, + abis=abis, + platforms=platforms, + ) + ) else: - versions = [version] - current_version = versions[0] - other_versions = versions[1:] - - impl = impl or interpreter_name() - - abis = [] # type: List[str] - - abi = abi or get_abi_tag() - if abi: - abis[0:0] = [abi] - - supports_abi3 = not PY2 and impl == "cp" - - if supports_abi3: - abis.append("abi3") - - abis.append('none') - - arches = _get_custom_platforms(platform or get_platform(), platform) - - # Current version, current API (built specifically for our Python): - for abi in abis: - for arch in arches: - supported.append(('%s%s' % (impl, current_version), abi, arch)) - - # abi3 modules compatible with older version of Python - if supports_abi3: - for version in other_versions: - # abi3 was introduced in Python 3.2 - if version in {'31', '30'}: - break - for arch in arches: - supported.append(("%s%s" % (impl, version), "abi3", arch)) - - # Has binaries, does not use the Python API: - for arch in arches: - supported.append(('py%s' % (current_version[0]), 'none', arch)) - - # No abi / arch, but requires our implementation: - supported.append(('%s%s' % (impl, current_version), 'none', 'any')) - - # No abi / arch, generic Python - supported.append(('py%s' % (current_version,), 'none', 'any')) - supported.append(('py%s' % (current_version[0]), 'none', 'any')) - for version in other_versions: - supported.append(('py%s' % (version,), 'none', 'any')) + supported.extend( + generic_tags( + interpreter=interpreter, + abis=abis, + platforms=platforms, + ) + ) + supported.extend( + compatible_tags( + python_version=python_version, + interpreter=interpreter, + platforms=platforms, + ) + ) - return [Tag(*parts) for parts in supported] + return supported diff --git a/src/pip/_internal/utils/glibc.py b/src/pip/_internal/utils/glibc.py index 42b1d3919a3..36104244138 100644 --- a/src/pip/_internal/utils/glibc.py +++ b/src/pip/_internal/utils/glibc.py @@ -4,9 +4,7 @@ from __future__ import absolute_import import os -import re import sys -import warnings from pip._internal.utils.typing import MYPY_CHECK_RUNNING @@ -69,32 +67,6 @@ def glibc_version_string_ctypes(): return version_str -# Separated out from have_compatible_glibc for easier unit testing -def check_glibc_version(version_str, required_major, minimum_minor): - # type: (str, int, int) -> bool - # Parse string and check against requested version. - # - # We use a regexp instead of str.split because we want to discard any - # random junk that might come after the minor version -- this might happen - # in patched/forked versions of glibc (e.g. Linaro's version of glibc - # uses version strings like "2.20-2014.11"). See gh-3588. - m = re.match(r"(?P[0-9]+)\.(?P[0-9]+)", version_str) - if not m: - warnings.warn("Expected glibc version with 2 components major.minor," - " got: %s" % version_str, RuntimeWarning) - return False - return (int(m.group("major")) == required_major and - int(m.group("minor")) >= minimum_minor) - - -def have_compatible_glibc(required_major, minimum_minor): - # type: (int, int) -> bool - version_str = glibc_version_string() - if version_str is None: - return False - return check_glibc_version(version_str, required_major, minimum_minor) - - # platform.libc_ver regularly returns completely nonsensical glibc # versions. E.g. on my computer, platform says: # diff --git a/tests/unit/test_pep425tags.py b/tests/unit/test_pep425tags.py index eaa88888fb0..71d0aefe4f9 100644 --- a/tests/unit/test_pep425tags.py +++ b/tests/unit/test_pep425tags.py @@ -1,8 +1,7 @@ -import sys +import sysconfig import pytest from mock import patch -from pip._vendor.packaging.tags import interpreter_name, interpreter_version from pip._internal import pep425tags @@ -28,9 +27,7 @@ def mock_get_config_var(self, **kwd): """ Patch sysconfig.get_config_var for arbitrary keys. """ - import pip._internal.pep425tags - - get_config_var = pip._internal.pep425tags.sysconfig.get_config_var + get_config_var = sysconfig.get_config_var def _mock_get_config_var(var): if var in kwd: @@ -38,45 +35,6 @@ def _mock_get_config_var(var): return get_config_var(var) return _mock_get_config_var - def abi_tag_unicode(self, flags, config_vars): - """ - Used to test ABI tags, verify correct use of the `u` flag - """ - import pip._internal.pep425tags - - config_vars.update({'SOABI': None}) - base = interpreter_name() + interpreter_version() - - if sys.version_info >= (3, 8): - # Python 3.8 removes the m flag, so don't look for it. - flags = flags.replace('m', '') - - if sys.version_info < (3, 3): - config_vars.update({'Py_UNICODE_SIZE': 2}) - mock_gcf = self.mock_get_config_var(**config_vars) - with patch('pip._internal.pep425tags.sysconfig.get_config_var', - mock_gcf): - abi_tag = pip._internal.pep425tags.get_abi_tag() - assert abi_tag == base + flags - - config_vars.update({'Py_UNICODE_SIZE': 4}) - mock_gcf = self.mock_get_config_var(**config_vars) - with patch('pip._internal.pep425tags.sysconfig.get_config_var', - mock_gcf): - abi_tag = pip._internal.pep425tags.get_abi_tag() - assert abi_tag == base + flags + 'u' - - else: - # On Python >= 3.3, UCS-4 is essentially permanently enabled, and - # Py_UNICODE_SIZE is None. SOABI on these builds does not include - # the 'u' so manual SOABI detection should not do so either. - config_vars.update({'Py_UNICODE_SIZE': None}) - mock_gcf = self.mock_get_config_var(**config_vars) - with patch('pip._internal.pep425tags.sysconfig.get_config_var', - mock_gcf): - abi_tag = pip._internal.pep425tags.get_abi_tag() - assert abi_tag == base + flags - def test_no_hyphen_tag(self): """ Test that no tag contains a hyphen. @@ -85,8 +43,7 @@ def test_no_hyphen_tag(self): mock_gcf = self.mock_get_config_var(SOABI='cpython-35m-darwin') - with patch('pip._internal.pep425tags.sysconfig.get_config_var', - mock_gcf): + with patch('sysconfig.get_config_var', mock_gcf): supported = pip._internal.pep425tags.get_supported() for tag in supported: @@ -94,142 +51,9 @@ def test_no_hyphen_tag(self): assert '-' not in tag.abi assert '-' not in tag.platform - def test_manual_abi_noflags(self): - """ - Test that no flags are set on a non-PyDebug, non-Pymalloc ABI tag. - """ - self.abi_tag_unicode('', {'Py_DEBUG': False, 'WITH_PYMALLOC': False}) - - def test_manual_abi_d_flag(self): - """ - Test that the `d` flag is set on a PyDebug, non-Pymalloc ABI tag. - """ - self.abi_tag_unicode('d', {'Py_DEBUG': True, 'WITH_PYMALLOC': False}) - - def test_manual_abi_m_flag(self): - """ - Test that the `m` flag is set on a non-PyDebug, Pymalloc ABI tag. - """ - self.abi_tag_unicode('m', {'Py_DEBUG': False, 'WITH_PYMALLOC': True}) - - def test_manual_abi_dm_flags(self): - """ - Test that the `dm` flags are set on a PyDebug, Pymalloc ABI tag. - """ - self.abi_tag_unicode('dm', {'Py_DEBUG': True, 'WITH_PYMALLOC': True}) - - -@pytest.mark.parametrize('is_manylinux_compatible', [ - pep425tags.is_manylinux1_compatible, - pep425tags.is_manylinux2010_compatible, - pep425tags.is_manylinux2014_compatible, -]) -class TestManylinuxTags(object): - """ - Tests common to all manylinux tags (e.g. manylinux1, manylinux2010, - ...) - """ - @patch('pip._internal.pep425tags.get_platform', lambda: 'linux_x86_64') - @patch('pip._internal.utils.glibc.have_compatible_glibc', - lambda major, minor: True) - def test_manylinux_compatible_on_linux_x86_64(self, - is_manylinux_compatible): - """ - Test that manylinuxes are enabled on linux_x86_64 - """ - assert is_manylinux_compatible() - - @patch('pip._internal.pep425tags.get_platform', lambda: 'linux_i686') - @patch('pip._internal.utils.glibc.have_compatible_glibc', - lambda major, minor: True) - def test_manylinux_compatible_on_linux_i686(self, - is_manylinux_compatible): - """ - Test that manylinuxes are enabled on linux_i686 - """ - assert is_manylinux_compatible() - - @patch('pip._internal.pep425tags.get_platform', lambda: 'linux_x86_64') - @patch('pip._internal.utils.glibc.have_compatible_glibc', - lambda major, minor: False) - def test_manylinux_2(self, is_manylinux_compatible): - """ - Test that manylinuxes are disabled with incompatible glibc - """ - assert not is_manylinux_compatible() - - @patch('pip._internal.pep425tags.get_platform', lambda: 'arm6vl') - @patch('pip._internal.utils.glibc.have_compatible_glibc', - lambda major, minor: True) - def test_manylinux_3(self, is_manylinux_compatible): - """ - Test that manylinuxes are disabled on arm6vl - """ - assert not is_manylinux_compatible() - - -class TestManylinux1Tags(object): - - @patch('pip._internal.pep425tags.is_manylinux2010_compatible', - lambda: False) - @patch('pip._internal.pep425tags.is_manylinux2014_compatible', - lambda: False) - @patch('pip._internal.pep425tags.get_platform', lambda: 'linux_x86_64') - @patch('pip._internal.utils.glibc.have_compatible_glibc', - lambda major, minor: True) - @patch('sys.platform', 'linux2') - def test_manylinux1_tag_is_first(self): - """ - Test that the more specific tag manylinux1 comes first. - """ - groups = {} - for tag in pep425tags.get_supported(): - groups.setdefault( - (tag.interpreter, tag.abi), [] - ).append(tag.platform) - - for arches in groups.values(): - if arches == ['any']: - continue - # Expect the most specific arch first: - if len(arches) == 3: - assert arches == ['manylinux1_x86_64', 'linux_x86_64', 'any'] - else: - assert arches == ['manylinux1_x86_64', 'linux_x86_64'] - class TestManylinux2010Tags(object): - @patch('pip._internal.pep425tags.is_manylinux2014_compatible', - lambda: False) - @patch('pip._internal.pep425tags.get_platform', lambda: 'linux_x86_64') - @patch('pip._internal.utils.glibc.have_compatible_glibc', - lambda major, minor: True) - @patch('sys.platform', 'linux2') - def test_manylinux2010_tag_is_first(self): - """ - Test that the more specific tag manylinux2010 comes first. - """ - groups = {} - for tag in pep425tags.get_supported(): - groups.setdefault( - (tag.interpreter, tag.abi), [] - ).append(tag.platform) - - for arches in groups.values(): - if arches == ['any']: - continue - # Expect the most specific arch first: - if len(arches) == 4: - assert arches == ['manylinux2010_x86_64', - 'manylinux1_x86_64', - 'linux_x86_64', - 'any'] - else: - assert arches == ['manylinux2010_x86_64', - 'manylinux1_x86_64', - 'linux_x86_64'] - @pytest.mark.parametrize("manylinux2010,manylinux1", [ ("manylinux2010_x86_64", "manylinux1_x86_64"), ("manylinux2010_i686", "manylinux1_i686"), @@ -253,36 +77,6 @@ def test_manylinux2010_implies_manylinux1(self, manylinux2010, manylinux1): class TestManylinux2014Tags(object): - @patch('pip._internal.pep425tags.get_platform', lambda: 'linux_x86_64') - @patch('pip._internal.utils.glibc.have_compatible_glibc', - lambda major, minor: True) - @patch('sys.platform', 'linux2') - def test_manylinux2014_tag_is_first(self): - """ - Test that the more specific tag manylinux2014 comes first. - """ - groups = {} - for tag in pep425tags.get_supported(): - groups.setdefault( - (tag.interpreter, tag.abi), [] - ).append(tag.platform) - - for arches in groups.values(): - if arches == ['any']: - continue - # Expect the most specific arch first: - if len(arches) == 5: - assert arches == ['manylinux2014_x86_64', - 'manylinux2010_x86_64', - 'manylinux1_x86_64', - 'linux_x86_64', - 'any'] - else: - assert arches == ['manylinux2014_x86_64', - 'manylinux2010_x86_64', - 'manylinux1_x86_64', - 'linux_x86_64'] - @pytest.mark.parametrize("manylinuxA,manylinuxB", [ ("manylinux2014_x86_64", ["manylinux2010_x86_64", "manylinux1_x86_64"]), diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 64c8aabf505..011543bdcc5 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -10,7 +10,6 @@ import stat import sys import time -import warnings from io import BytesIO import pytest @@ -24,7 +23,6 @@ from pip._internal.utils.deprecation import PipDeprecationWarning, deprecated from pip._internal.utils.encoding import BOMS, auto_decode from pip._internal.utils.glibc import ( - check_glibc_version, glibc_version_string, glibc_version_string_confstr, glibc_version_string_ctypes, @@ -538,38 +536,6 @@ def raises(error): class TestGlibc(object): - def test_manylinux_check_glibc_version(self): - """ - Test that the check_glibc_version function is robust against weird - glibc version strings. - """ - for two_twenty in ["2.20", - # used by "linaro glibc", see gh-3588 - "2.20-2014.11", - # weird possibilities that I just made up - "2.20+dev", - "2.20-custom", - "2.20.1", - ]: - assert check_glibc_version(two_twenty, 2, 15) - assert check_glibc_version(two_twenty, 2, 20) - assert not check_glibc_version(two_twenty, 2, 21) - assert not check_glibc_version(two_twenty, 3, 15) - assert not check_glibc_version(two_twenty, 1, 15) - - # For strings that we just can't parse at all, we should warn and - # return false - for bad_string in ["asdf", "", "foo.bar"]: - with warnings.catch_warnings(record=True) as ws: - warnings.filterwarnings("always") - assert not check_glibc_version(bad_string, 2, 5) - for w in ws: - if "Expected glibc version with" in str(w.message): - break - else: - # Didn't find the warning we were expecting - assert False - @pytest.mark.skipif("sys.platform == 'win32'") def test_glibc_version_string(self, monkeypatch): monkeypatch.setattr(