diff --git a/public/2.6/get-pip.py b/public/2.6/get-pip.py index b601fccb..b83a2ee6 100644 --- a/public/2.6/get-pip.py +++ b/public/2.6/get-pip.py @@ -147,11 +147,11 @@ def parse_args(self, args): # Add any implicit installations to the end of our args if implicit_pip: - args += ["pip<10"] + args += ["pip"] if implicit_setuptools: - args += ["setuptools<37"] + args += ["setuptools"] if implicit_wheel: - args += ["wheel<0.30"] + args += ["wheel"] delete_tmpdir = False try: diff --git a/public/2.7/get-pip.py b/public/2.7/get-pip.py index 3ef3d705..c2064943 100644 --- a/public/2.7/get-pip.py +++ b/public/2.7/get-pip.py @@ -149,9 +149,9 @@ def cert_parse_args(self, args): # Add any implicit installations to the end of our args if implicit_pip: - args += ["pip<21.0"] + args += ["pip"] if implicit_setuptools: - args += ["setuptools<45"] + args += ["setuptools"] if implicit_wheel: args += ["wheel"] diff --git a/public/3.3/get-pip.py b/public/3.3/get-pip.py index 77be6b0f..c5fb4a97 100644 --- a/public/3.3/get-pip.py +++ b/public/3.3/get-pip.py @@ -147,11 +147,11 @@ def parse_args(self, args): # Add any implicit installations to the end of our args if implicit_pip: - args += ["pip<18"] + args += ["pip"] if implicit_setuptools: args += ["setuptools"] if implicit_wheel: - args += ["wheel<0.30"] + args += ["wheel"] # Add our default arguments args = ["install", "--upgrade", "--force-reinstall"] + args diff --git a/public/3.4/get-pip.py b/public/3.4/get-pip.py index a52adebc..759d72e6 100644 --- a/public/3.4/get-pip.py +++ b/public/3.4/get-pip.py @@ -147,7 +147,7 @@ def parse_args(self, args): # Add any implicit installations to the end of our args if implicit_pip: - args += ["pip<19.2"] + args += ["pip"] if implicit_setuptools: args += ["setuptools"] if implicit_wheel: diff --git a/public/3.5/get-pip.py b/public/3.5/get-pip.py index 0761be8a..c2064943 100644 --- a/public/3.5/get-pip.py +++ b/public/3.5/get-pip.py @@ -149,7 +149,7 @@ def cert_parse_args(self, args): # Add any implicit installations to the end of our args if implicit_pip: - args += ["pip<21.0"] + args += ["pip"] if implicit_setuptools: args += ["setuptools"] if implicit_wheel: diff --git a/public/3.6/get-pip.py b/public/3.6/get-pip.py index efc3a26c..6ee0c6af 100644 --- a/public/3.6/get-pip.py +++ b/public/3.6/get-pip.py @@ -71,7 +71,7 @@ def determine_pip_install_arguments(): pre_parser.add_argument("--no-wheel", action="store_true") pre, args = pre_parser.parse_known_args() - args.append("pip<22.0") + args.append("pip") if include_setuptools(pre): args.append("setuptools") diff --git a/public/3.7/get-pip.py b/public/3.7/get-pip.py index ca5622ec..0d1fc4e1 100644 --- a/public/3.7/get-pip.py +++ b/public/3.7/get-pip.py @@ -71,7 +71,7 @@ def determine_pip_install_arguments(): pre_parser.add_argument("--no-wheel", action="store_true") pre, args = pre_parser.parse_known_args() - args.append("pip<24.1") + args.append("pip") if include_setuptools(pre): args.append("setuptools") diff --git a/scripts/generate.py b/scripts/generate.py index d6fd75b7..36993540 100644 --- a/scripts/generate.py +++ b/scripts/generate.py @@ -23,48 +23,34 @@ SCRIPT_CONSTRAINTS = { "default": { "pip": "", - "setuptools": "", - "wheel": "", }, "2.6": { "pip": "<10", - "setuptools": "<37", - "wheel": "<0.30", }, "2.7": { "pip": "<21.0", - "setuptools": "<45", - "wheel": "", }, "3.2": { + # Pip older than v9.0.0 does not support Requires-Python so we have to manually + # constrain the pip, setuptools and wheel versions that are installed at runtime. "pip": "<8", "setuptools": "<30", "wheel": "<0.30", }, "3.3": { "pip": "<18", - "setuptools": "", - "wheel": "<0.30", }, "3.4": { "pip": "<19.2", - "setuptools": "", - "wheel": "", }, "3.5": { "pip": "<21.0", - "setuptools": "", - "wheel": "", }, "3.6": { "pip": "<22.0", - "setuptools": "", - "wheel": "", }, "3.7": { "pip": "<24.1", - "setuptools": "", - "wheel": "", }, } @@ -248,7 +234,7 @@ def detect_newline(f: TextIO) -> str: def generate_one(variant, mapping, *, console, pip_versions): - # Determing the correct wheel to download + # Determine the correct wheel to download pip_version = determine_latest(pip_versions.keys(), constraint=mapping["pip"]) wheel_url, wheel_hash = pip_versions[pip_version] @@ -264,10 +250,11 @@ def generate_one(variant, mapping, *, console, pip_versions): newline = detect_newline(f) rendered_template = f.read().format( zipfile=encoded_wheel, - installed_version=pip_version, - pip_version=mapping["pip"], - setuptools_version=mapping["setuptools"], - wheel_version=mapping["wheel"], + bundled_pip_version=pip_version, + # These constraints are only used for pip versions that don't support Requires-Python. + pip_version_constraint=mapping.get("pip"), + setuptools_version_constraint=mapping.get("setuptools"), + wheel_version_constraint=mapping.get("wheel"), minimum_supported_version=mapping["minimum_supported_version"], ) # Write the script to the correct location diff --git a/templates/default.py b/templates/default.py index 01f27a33..c90cc3d5 100644 --- a/templates/default.py +++ b/templates/default.py @@ -5,7 +5,7 @@ # You may be wondering what this giant blob of binary data here is, you might # even be worried that we're up to something nefarious (good for you for being # paranoid!). This is a base85 encoding of a zip file, this zip file contains -# an entire copy of pip (version {installed_version}). +# an entire copy of pip (version {bundled_pip_version}). # # Pip is a thing that installs packages, pip itself is a package that someone # might want to install, especially if they're looking to run this get-pip.py @@ -71,13 +71,13 @@ def determine_pip_install_arguments(): pre_parser.add_argument("--no-wheel", action="store_true") pre, args = pre_parser.parse_known_args() - args.append("pip{pip_version}") + args.append("pip") if include_setuptools(pre): - args.append("setuptools{setuptools_version}") + args.append("setuptools") if include_wheel(pre): - args.append("wheel{wheel_version}") + args.append("wheel") return ["install", "--upgrade", "--force-reinstall"] + args diff --git a/templates/pre-10.py b/templates/pre-10.py index acbea9d8..280a1d8c 100644 --- a/templates/pre-10.py +++ b/templates/pre-10.py @@ -5,7 +5,7 @@ # You may be wondering what this giant blob of binary data here is, you might # even be worried that we're up to something nefarious (good for you for being # paranoid!). This is a base85 encoding of a zip file, this zip file contains -# an entire copy of pip (version {installed_version}). +# an entire copy of pip (version {bundled_pip_version}). # # Pip is a thing that installs packages, pip itself is a package that someone # might want to install, especially if they're looking to run this get-pip.py @@ -147,11 +147,11 @@ def parse_args(self, args): # Add any implicit installations to the end of our args if implicit_pip: - args += ["pip{pip_version}"] + args += ["pip"] if implicit_setuptools: - args += ["setuptools{setuptools_version}"] + args += ["setuptools"] if implicit_wheel: - args += ["wheel{wheel_version}"] + args += ["wheel"] delete_tmpdir = False try: diff --git a/templates/pre-18.1.py b/templates/pre-18.1.py index dd10d083..dfe760f8 100644 --- a/templates/pre-18.1.py +++ b/templates/pre-18.1.py @@ -5,7 +5,7 @@ # You may be wondering what this giant blob of binary data here is, you might # even be worried that we're up to something nefarious (good for you for being # paranoid!). This is a base85 encoding of a zip file, this zip file contains -# an entire copy of pip (version {installed_version}). +# an entire copy of pip (version {bundled_pip_version}). # # Pip is a thing that installs packages, pip itself is a package that someone # might want to install, especially if they're looking to run this get-pip.py @@ -147,11 +147,11 @@ def parse_args(self, args): # Add any implicit installations to the end of our args if implicit_pip: - args += ["pip{pip_version}"] + args += ["pip"] if implicit_setuptools: - args += ["setuptools{setuptools_version}"] + args += ["setuptools"] if implicit_wheel: - args += ["wheel{wheel_version}"] + args += ["wheel"] # Add our default arguments args = ["install", "--upgrade", "--force-reinstall"] + args diff --git a/templates/pre-19.3.py b/templates/pre-19.3.py index 38fb75c4..1d198f4c 100644 --- a/templates/pre-19.3.py +++ b/templates/pre-19.3.py @@ -5,7 +5,7 @@ # You may be wondering what this giant blob of binary data here is, you might # even be worried that we're up to something nefarious (good for you for being # paranoid!). This is a base85 encoding of a zip file, this zip file contains -# an entire copy of pip (version {installed_version}). +# an entire copy of pip (version {bundled_pip_version}). # # Pip is a thing that installs packages, pip itself is a package that someone # might want to install, especially if they're looking to run this get-pip.py @@ -147,11 +147,11 @@ def parse_args(self, args): # Add any implicit installations to the end of our args if implicit_pip: - args += ["pip{pip_version}"] + args += ["pip"] if implicit_setuptools: - args += ["setuptools{setuptools_version}"] + args += ["setuptools"] if implicit_wheel: - args += ["wheel{wheel_version}"] + args += ["wheel"] # Add our default arguments args = ["install", "--upgrade", "--force-reinstall"] + args diff --git a/templates/pre-21.0.py b/templates/pre-21.0.py index 038c6249..6006bfdf 100644 --- a/templates/pre-21.0.py +++ b/templates/pre-21.0.py @@ -5,7 +5,7 @@ # You may be wondering what this giant blob of binary data here is, you might # even be worried that we're up to something nefarious (good for you for being # paranoid!). This is a base85 encoding of a zip file, this zip file contains -# an entire copy of pip (version {installed_version}). +# an entire copy of pip (version {bundled_pip_version}). # # Pip is a thing that installs packages, pip itself is a package that someone # might want to install, especially if they're looking to run this get-pip.py @@ -149,11 +149,11 @@ def cert_parse_args(self, args): # Add any implicit installations to the end of our args if implicit_pip: - args += ["pip{pip_version}"] + args += ["pip"] if implicit_setuptools: - args += ["setuptools{setuptools_version}"] + args += ["setuptools"] if implicit_wheel: - args += ["wheel{wheel_version}"] + args += ["wheel"] # Add our default arguments args = ["install", "--upgrade", "--force-reinstall"] + args diff --git a/templates/pre-9.py b/templates/pre-9.py new file mode 100644 index 00000000..232f48c6 --- /dev/null +++ b/templates/pre-9.py @@ -0,0 +1,207 @@ +#!/usr/bin/env python +# +# Hi There! +# +# You may be wondering what this giant blob of binary data here is, you might +# even be worried that we're up to something nefarious (good for you for being +# paranoid!). This is a base85 encoding of a zip file, this zip file contains +# an entire copy of pip (version {bundled_pip_version}). +# +# Pip is a thing that installs packages, pip itself is a package that someone +# might want to install, especially if they're looking to run this get-pip.py +# script. Pip has a lot of code to deal with the security of installing +# packages, various edge cases on various platforms, and other such sort of +# "tribal knowledge" that has been encoded in its code base. Because of this +# we basically include an entire copy of pip inside this blob. We do this +# because the alternatives are attempt to implement a "minipip" that probably +# doesn't do things correctly and has weird edge cases, or compress pip itself +# down into a single file. +# +# If you're wondering how this is created, it is generated using +# `scripts/generate.py` in https://github.com/pypa/get-pip. + +import os.path +import pkgutil +import shutil +import sys +import struct +import tempfile + +# Useful for very coarse version differentiation. +PY2 = sys.version_info[0] == 2 +PY3 = sys.version_info[0] == 3 + +if PY3: + iterbytes = iter +else: + def iterbytes(buf): + return (ord(byte) for byte in buf) + +try: + from base64 import b85decode +except ImportError: + _b85alphabet = (b"0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ" + b"abcdefghijklmnopqrstuvwxyz!#$%&()*+-;<=>?@^_`{{|}}~") + + def b85decode(b): + _b85dec = [None] * 256 + for i, c in enumerate(iterbytes(_b85alphabet)): + _b85dec[c] = i + + padding = (-len(b)) % 5 + b = b + b'~' * padding + out = [] + packI = struct.Struct('!I').pack + for i in range(0, len(b), 5): + chunk = b[i:i + 5] + acc = 0 + try: + for c in iterbytes(chunk): + acc = acc * 85 + _b85dec[c] + except TypeError: + for j, c in enumerate(iterbytes(chunk)): + if _b85dec[c] is None: + raise ValueError( + 'bad base85 character at position %d' % (i + j) + ) + raise + try: + out.append(packI(acc)) + except struct.error: + raise ValueError('base85 overflow in hunk starting at byte %d' + % i) + + result = b''.join(out) + if padding: + result = result[:-padding] + return result + + +def bootstrap(tmpdir=None): + # Import pip so we can use it to install pip and maybe setuptools too + import pip + from pip.commands.install import InstallCommand + from pip.req import InstallRequirement + + # Wrapper to provide default certificate with the lowest priority + class CertInstallCommand(InstallCommand): + def parse_args(self, args): + # If cert isn't specified in config or environment, we provide our + # own certificate through defaults. + # This allows user to specify custom cert anywhere one likes: + # config, environment variable or argv. + if not self.parser.get_default_values().cert: + self.parser.defaults["cert"] = cert_path # calculated below + return super(CertInstallCommand, self).parse_args(args) + + pip.commands_dict["install"] = CertInstallCommand + + implicit_pip = True + implicit_setuptools = True + implicit_wheel = True + + # Check if the user has requested us not to install setuptools + if "--no-setuptools" in sys.argv or os.environ.get("PIP_NO_SETUPTOOLS"): + args = [x for x in sys.argv[1:] if x != "--no-setuptools"] + implicit_setuptools = False + else: + args = sys.argv[1:] + + # Check if the user has requested us not to install wheel + if "--no-wheel" in args or os.environ.get("PIP_NO_WHEEL"): + args = [x for x in args if x != "--no-wheel"] + implicit_wheel = False + + # We only want to implicitly install setuptools and wheel if they don't + # already exist on the target platform. + if implicit_setuptools: + try: + import setuptools # noqa + implicit_setuptools = False + except ImportError: + pass + if implicit_wheel: + try: + import wheel # noqa + implicit_wheel = False + except ImportError: + pass + + # We want to support people passing things like 'pip<8' to get-pip.py which + # will let them install a specific version. However because of the dreaded + # DoubleRequirement error if any of the args look like they might be a + # specific for one of our packages, then we'll turn off the implicit + # install of them. + for arg in args: + try: + req = InstallRequirement.from_line(arg) + except Exception: + continue + + if implicit_pip and req.name == "pip": + implicit_pip = False + elif implicit_setuptools and req.name == "setuptools": + implicit_setuptools = False + elif implicit_wheel and req.name == "wheel": + implicit_wheel = False + + # Add any implicit installations to the end of our args + if implicit_pip: + args += ["pip{pip_version_constraint}"] + if implicit_setuptools: + args += ["setuptools{setuptools_version_constraint}"] + if implicit_wheel: + args += ["wheel{wheel_version_constraint}"] + + delete_tmpdir = False + try: + # Create a temporary directory to act as a working directory if we were + # not given one. + if tmpdir is None: + tmpdir = tempfile.mkdtemp() + delete_tmpdir = True + + # We need to extract the SSL certificates from requests so that they + # can be passed to --cert + cert_path = os.path.join(tmpdir, "cacert.pem") + with open(cert_path, "wb") as cert: + cert.write(pkgutil.get_data("pip._vendor.requests", "cacert.pem")) + + # Execute the included pip and use it to install the latest pip and + # setuptools from PyPI + sys.exit(pip.main(["install", "--upgrade"] + args)) + finally: + # Remove our temporary directory + if delete_tmpdir and tmpdir: + shutil.rmtree(tmpdir, ignore_errors=True) + + +def main(): + tmpdir = None + try: + # Create a temporary working directory + tmpdir = tempfile.mkdtemp() + + # Unpack the zipfile into the temporary directory + pip_zip = os.path.join(tmpdir, "pip.zip") + with open(pip_zip, "wb") as fp: + fp.write(b85decode(DATA.replace(b"\n", b""))) + + # Add the zipfile to sys.path so that we can import it + sys.path.insert(0, pip_zip) + + # Run the bootstrap + bootstrap(tmpdir=tmpdir) + finally: + # Clean up our temporary working directory + if tmpdir: + shutil.rmtree(tmpdir, ignore_errors=True) + + +DATA = b""" +{zipfile} +""" + + +if __name__ == "__main__": + main()