From bef2244d295ea5bf979261513b31bf38b941ac28 Mon Sep 17 00:00:00 2001 From: UebelAndre Date: Sun, 5 Sep 2021 15:22:44 -0700 Subject: [PATCH] `pip_parse` and `pip_install` can now parse entry points from wheels (#523) --- docs/pip.md | 98 +++++++++++++++- examples/pip_install/BUILD | 20 +++- examples/pip_install/entry_point_test.py | 20 ++++ examples/pip_install/requirements.in | 1 + examples/pip_install/requirements.txt | 62 ++++++++-- examples/pip_parse/BUILD | 29 ++++- examples/pip_parse/entry_point_test.py | 20 ++++ examples/pip_parse/requirements.txt | 1 + examples/pip_parse/requirements_lock.txt | 46 +++++++- python/pip.bzl | 110 +++++++++++++++++- .../pip_install/extract_wheels/lib/bazel.py | 110 ++++++++++++++++-- .../pip_install/extract_wheels/lib/wheel.py | 27 +++++ .../parse_requirements_to_bzl/__init__.py | 10 +- 13 files changed, 525 insertions(+), 29 deletions(-) mode change 100755 => 100644 docs/pip.md create mode 100644 examples/pip_install/entry_point_test.py create mode 100644 examples/pip_parse/entry_point_test.py diff --git a/docs/pip.md b/docs/pip.md old mode 100755 new mode 100644 index 6df1794fd5..90b1bbee4f --- a/docs/pip.md +++ b/docs/pip.md @@ -51,6 +51,36 @@ py_library( ) ``` +In addition to the `requirement` macro, which is used to access the generated `py_library` +target generated from a package's wheel, The generated `requirements.bzl` file contains +functionality for exposing [entry points][whl_ep] as `py_binary` targets as well. + +[whl_ep]: https://packaging.python.org/specifications/entry-points/ + +```python +load("@pip_deps//:requirements.bzl", "entry_point") + +alias( + name = "pip-compile", + actual = entry_point( + pkg = "pip-tools", + script = "pip-compile", + ), +) +``` + +Note that for packages who's name and script are the same, only the name of the package +is needed when calling the `entry_point` macro. + +```python +load("@pip_deps//:requirements.bzl", "entry_point") + +alias( + name = "flake8", + actual = entry_point("flake8"), +) +``` + **PARAMETERS** @@ -70,6 +100,68 @@ py_library( pip_parse(requirements_lock, name, kwargs) +Imports a locked/compiled requirements file and generates a new `requirements.bzl` file. + +This is used via the `WORKSPACE` pattern: + +```python +load("@rules_python//python:pip.bzl", "pip_parse") + +pip_parse( + name = "pip_deps", + requirements_lock = ":requirements.txt", +) + +load("@pip_deps//:requirements.bzl", "install_deps") + +install_deps() +``` + +You can then reference imported dependencies from your `BUILD` file with: + +```python +load("@pip_deps//:requirements.bzl", "requirement") + +py_library( + name = "bar", + ... + deps = [ + "//my/other:dep", + requirement("requests"), + requirement("numpy"), + ], +) +``` + +In addition to the `requirement` macro, which is used to access the generated `py_library` +target generated from a package's wheel, The generated `requirements.bzl` file contains +functionality for exposing [entry points][whl_ep] as `py_binary` targets as well. + +[whl_ep]: https://packaging.python.org/specifications/entry-points/ + +```python +load("@pip_deps//:requirements.bzl", "entry_point") + +alias( + name = "pip-compile", + actual = entry_point( + pkg = "pip-tools", + script = "pip-compile", + ), +) +``` + +Note that for packages who's name and script are the same, only the name of the package +is needed when calling the `entry_point` macro. + +```python +load("@pip_deps//:requirements.bzl", "entry_point") + +alias( + name = "flake8", + actual = entry_point("flake8"), +) +``` **PARAMETERS** @@ -77,9 +169,9 @@ pip_parse(requirements_lock, -

| none | -| name |

-

| "pip_parsed_deps" | -| kwargs |

-

| none | +| requirements_lock | A fully resolved 'requirements.txt' pip requirement file containing the transitive set of your dependencies. If this file is passed instead of 'requirements' no resolve will take place and pip_repository will create individual repositories for each of your dependencies so that wheels are fetched/built only for the targets specified by 'build/run/test'. | none | +| name | The name of the generated repository. | "pip_parsed_deps" | +| kwargs | Additional keyword arguments for the underlying pip_repository rule. | none | diff --git a/examples/pip_install/BUILD b/examples/pip_install/BUILD index c8fbc0bbee..c57ffbdcb5 100644 --- a/examples/pip_install/BUILD +++ b/examples/pip_install/BUILD @@ -1,4 +1,8 @@ -load("@pip//:requirements.bzl", "requirement") +load( + "@pip//:requirements.bzl", + "entry_point", + "requirement", +) load("@rules_python//python:defs.bzl", "py_binary", "py_test") load("@rules_python//python/pip_install:requirements.bzl", "compile_pip_requirements") @@ -42,7 +46,21 @@ py_test( deps = [":main"], ) +# For pip dependencies which have entry points, the `entry_point` macro can be +# used from the generated `pip_install` repository to access a runnable binary. +alias( + name = "yamllint", + actual = entry_point("yamllint"), +) + +py_test( + name = "entry_point_test", + srcs = ["entry_point_test.py"], + data = [":yamllint"], +) + # Check that our compiled requirements are up-to-date compile_pip_requirements( name = "requirements", + extra_args = ["--allow-unsafe"], ) diff --git a/examples/pip_install/entry_point_test.py b/examples/pip_install/entry_point_test.py new file mode 100644 index 0000000000..b6b589a7b6 --- /dev/null +++ b/examples/pip_install/entry_point_test.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python3 + +from pathlib import Path +import subprocess +import unittest + + +class PipParseEntryPointTest(unittest.TestCase): + def test_output(self): + self.maxDiff = None + + entry_point = Path("external/pip/pypi__yamllint/rules_python_wheel_entry_point_yamllint") + self.assertTrue(entry_point.exists()) + + proc = subprocess.run([entry_point, "--version"], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + self.assertEqual(proc.stdout.decode("utf-8").strip(), "yamllint 1.26.3") + + +if __name__ == "__main__": + unittest.main() diff --git a/examples/pip_install/requirements.in b/examples/pip_install/requirements.in index cbc5542582..6ecac0dc9a 100644 --- a/examples/pip_install/requirements.in +++ b/examples/pip_install/requirements.in @@ -1 +1,2 @@ boto3==1.14.51 +yamllint==1.26.3 diff --git a/examples/pip_install/requirements.txt b/examples/pip_install/requirements.txt index 78b024682b..267c37002a 100644 --- a/examples/pip_install/requirements.txt +++ b/examples/pip_install/requirements.txt @@ -25,19 +25,63 @@ jmespath==0.10.0 \ # via # boto3 # botocore -python-dateutil==2.8.1 \ - --hash=sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c \ - --hash=sha256:75bb3f31ea686f1197762692a9ee6a7550b59fc6ca3a1f4b5d7e32fb98e2da2a +pathspec==0.9.0 \ + --hash=sha256:7d15c4ddb0b5c802d161efc417ec1a2558ea2653c2e8ad9c19098201dc1c993a \ + --hash=sha256:e564499435a2673d586f6b2130bb5b95f04a3ba06f81b8f895b651a3c76aabb1 + # via yamllint +python-dateutil==2.8.2 \ + --hash=sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86 \ + --hash=sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9 # via botocore -s3transfer==0.3.3 \ - --hash=sha256:2482b4259524933a022d59da830f51bd746db62f047d6eb213f2f8855dcb8a13 \ - --hash=sha256:921a37e2aefc64145e7b73d50c71bb4f26f46e4c9f414dc648c6245ff92cf7db +pyyaml==5.4.1 \ + --hash=sha256:08682f6b72c722394747bddaf0aa62277e02557c0fd1c42cb853016a38f8dedf \ + --hash=sha256:0f5f5786c0e09baddcd8b4b45f20a7b5d61a7e7e99846e3c799b05c7c53fa696 \ + --hash=sha256:129def1b7c1bf22faffd67b8f3724645203b79d8f4cc81f674654d9902cb4393 \ + --hash=sha256:294db365efa064d00b8d1ef65d8ea2c3426ac366c0c4368d930bf1c5fb497f77 \ + --hash=sha256:3b2b1824fe7112845700f815ff6a489360226a5609b96ec2190a45e62a9fc922 \ + --hash=sha256:3bd0e463264cf257d1ffd2e40223b197271046d09dadf73a0fe82b9c1fc385a5 \ + --hash=sha256:4465124ef1b18d9ace298060f4eccc64b0850899ac4ac53294547536533800c8 \ + --hash=sha256:49d4cdd9065b9b6e206d0595fee27a96b5dd22618e7520c33204a4a3239d5b10 \ + --hash=sha256:4e0583d24c881e14342eaf4ec5fbc97f934b999a6828693a99157fde912540cc \ + --hash=sha256:5accb17103e43963b80e6f837831f38d314a0495500067cb25afab2e8d7a4018 \ + --hash=sha256:607774cbba28732bfa802b54baa7484215f530991055bb562efbed5b2f20a45e \ + --hash=sha256:6c78645d400265a062508ae399b60b8c167bf003db364ecb26dcab2bda048253 \ + --hash=sha256:72a01f726a9c7851ca9bfad6fd09ca4e090a023c00945ea05ba1638c09dc3347 \ + --hash=sha256:74c1485f7707cf707a7aef42ef6322b8f97921bd89be2ab6317fd782c2d53183 \ + --hash=sha256:895f61ef02e8fed38159bb70f7e100e00f471eae2bc838cd0f4ebb21e28f8541 \ + --hash=sha256:8c1be557ee92a20f184922c7b6424e8ab6691788e6d86137c5d93c1a6ec1b8fb \ + --hash=sha256:bb4191dfc9306777bc594117aee052446b3fa88737cd13b7188d0e7aa8162185 \ + --hash=sha256:bfb51918d4ff3d77c1c856a9699f8492c612cde32fd3bcd344af9be34999bfdc \ + --hash=sha256:c20cfa2d49991c8b4147af39859b167664f2ad4561704ee74c1de03318e898db \ + --hash=sha256:cb333c16912324fd5f769fff6bc5de372e9e7a202247b48870bc251ed40239aa \ + --hash=sha256:d2d9808ea7b4af864f35ea216be506ecec180628aced0704e34aca0b040ffe46 \ + --hash=sha256:d483ad4e639292c90170eb6f7783ad19490e7a8defb3e46f97dfe4bacae89122 \ + --hash=sha256:dd5de0646207f053eb0d6c74ae45ba98c3395a571a2891858e87df7c9b9bd51b \ + --hash=sha256:e1d4970ea66be07ae37a3c2e48b5ec63f7ba6804bdddfdbd3cfd954d25a82e63 \ + --hash=sha256:e4fac90784481d221a8e4b1162afa7c47ed953be40d31ab4629ae917510051df \ + --hash=sha256:fa5ae20527d8e831e8230cbffd9f8fe952815b2b7dae6ffec25318803a7528fc \ + --hash=sha256:fd7f6999a8070df521b6384004ef42833b9bd62cfee11a09bda1079b4b704247 \ + --hash=sha256:fdc842473cd33f45ff6bce46aea678a54e3d21f1b61a7750ce3c498eedfe25d6 \ + --hash=sha256:fe69978f3f768926cfa37b867e3843918e012cf83f680806599ddce33c2c68b0 + # via yamllint +s3transfer==0.3.7 \ + --hash=sha256:35627b86af8ff97e7ac27975fe0a98a312814b46c6333d8a6b889627bcd80994 \ + --hash=sha256:efa5bd92a897b6a8d5c1383828dca3d52d0790e0756d49740563a3fb6ed03246 # via boto3 -six==1.15.0 \ - --hash=sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259 \ - --hash=sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced +six==1.16.0 \ + --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926 \ + --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254 # via python-dateutil urllib3==1.25.11 \ --hash=sha256:8d7eaa5a82a1cac232164990f04874c594c9453ec55eef02eab885aa02fc17a2 \ --hash=sha256:f5321fbe4bf3fefa0efd0bfe7fb14e90909eb62a48ccda331726b4319897dd5e # via botocore +yamllint==1.26.3 \ + --hash=sha256:3934dcde484374596d6b52d8db412929a169f6d9e52e20f9ade5bf3523d9b96e + # via -r requirements.in + +# The following packages are considered to be unsafe in a requirements file: +setuptools==57.5.0 \ + --hash=sha256:60d78588f15b048f86e35cdab73003d8b21dd45108ee61a6693881a427f22073 \ + --hash=sha256:d9d3266d50f59c6967b9312844470babbdb26304fe740833a5f8d89829ba3a24 + # via yamllint diff --git a/examples/pip_parse/BUILD b/examples/pip_parse/BUILD index ca56af9c0d..2bc713b7ac 100644 --- a/examples/pip_parse/BUILD +++ b/examples/pip_parse/BUILD @@ -1,5 +1,6 @@ -load("@pip_parsed_deps//:requirements.bzl", "requirement") +load("@pip_parsed_deps//:requirements.bzl", "entry_point", "requirement") load("@rules_python//python:defs.bzl", "py_binary", "py_test") +load("@rules_python//python/pip_install:requirements.bzl", "compile_pip_requirements") # Toolchain setup, this is optional. # Demonstrate that we can use the same python interpreter for the toolchain and executing pip in pip install (see WORKSPACE). @@ -40,3 +41,29 @@ py_test( srcs = ["test.py"], deps = [":main"], ) + +# For pip dependencies which have entry points, the `entry_point` macro can be +# used from the generated `pip_parse` repository to access a runnable binary. +alias( + name = "yamllint", + # If `pkg` and `script` are the same, passing a single string to + # `entry_point` would work as well: `entry_point("yamllint")` + actual = entry_point( + pkg = "yamllint", + script = "yamllint", + ), +) + +py_test( + name = "entry_point_test", + srcs = ["entry_point_test.py"], + data = [":yamllint"], +) + +# This rule adds a convenient way to update the requiremenst file. +compile_pip_requirements( + name = "requirements", + extra_args = ["--allow-unsafe"], + requirements_in = "requirements.txt", + requirements_txt = "requirements_lock.txt", +) diff --git a/examples/pip_parse/entry_point_test.py b/examples/pip_parse/entry_point_test.py new file mode 100644 index 0000000000..7d2234377f --- /dev/null +++ b/examples/pip_parse/entry_point_test.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python3 + +from pathlib import Path +import subprocess +import unittest + + +class PipParseEntryPointTest(unittest.TestCase): + def test_output(self): + self.maxDiff = None + + entry_point = Path("external/pip_parsed_deps_pypi__yamllint/rules_python_wheel_entry_point_yamllint") + self.assertTrue(entry_point.exists()) + + proc = subprocess.run([entry_point, "--version"], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + self.assertEqual(proc.stdout.decode("utf-8").strip(), "yamllint 1.26.3") + + +if __name__ == "__main__": + unittest.main() diff --git a/examples/pip_parse/requirements.txt b/examples/pip_parse/requirements.txt index 9d84d35885..019562ae60 100644 --- a/examples/pip_parse/requirements.txt +++ b/examples/pip_parse/requirements.txt @@ -1 +1,2 @@ requests==2.25.1 +yamllint==1.26.3 diff --git a/examples/pip_parse/requirements_lock.txt b/examples/pip_parse/requirements_lock.txt index 7573a6f591..dd29e95011 100644 --- a/examples/pip_parse/requirements_lock.txt +++ b/examples/pip_parse/requirements_lock.txt @@ -2,7 +2,7 @@ # This file is autogenerated by pip-compile # To update, run: # -# pip-compile --generate-hashes --output-file=requirements_lock.txt requirements.txt +# bazel run //:requirements.update # certifi==2020.12.5 \ --hash=sha256:1a4995114262bffbc2413b159f2a1a480c969de6e6eb13ee966d470af86af59c \ @@ -16,6 +16,41 @@ idna==2.10 \ --hash=sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6 \ --hash=sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0 # via requests +pathspec==0.9.0 \ + --hash=sha256:7d15c4ddb0b5c802d161efc417ec1a2558ea2653c2e8ad9c19098201dc1c993a \ + --hash=sha256:e564499435a2673d586f6b2130bb5b95f04a3ba06f81b8f895b651a3c76aabb1 + # via yamllint +pyyaml==5.4.1 \ + --hash=sha256:08682f6b72c722394747bddaf0aa62277e02557c0fd1c42cb853016a38f8dedf \ + --hash=sha256:0f5f5786c0e09baddcd8b4b45f20a7b5d61a7e7e99846e3c799b05c7c53fa696 \ + --hash=sha256:129def1b7c1bf22faffd67b8f3724645203b79d8f4cc81f674654d9902cb4393 \ + --hash=sha256:294db365efa064d00b8d1ef65d8ea2c3426ac366c0c4368d930bf1c5fb497f77 \ + --hash=sha256:3b2b1824fe7112845700f815ff6a489360226a5609b96ec2190a45e62a9fc922 \ + --hash=sha256:3bd0e463264cf257d1ffd2e40223b197271046d09dadf73a0fe82b9c1fc385a5 \ + --hash=sha256:4465124ef1b18d9ace298060f4eccc64b0850899ac4ac53294547536533800c8 \ + --hash=sha256:49d4cdd9065b9b6e206d0595fee27a96b5dd22618e7520c33204a4a3239d5b10 \ + --hash=sha256:4e0583d24c881e14342eaf4ec5fbc97f934b999a6828693a99157fde912540cc \ + --hash=sha256:5accb17103e43963b80e6f837831f38d314a0495500067cb25afab2e8d7a4018 \ + --hash=sha256:607774cbba28732bfa802b54baa7484215f530991055bb562efbed5b2f20a45e \ + --hash=sha256:6c78645d400265a062508ae399b60b8c167bf003db364ecb26dcab2bda048253 \ + --hash=sha256:72a01f726a9c7851ca9bfad6fd09ca4e090a023c00945ea05ba1638c09dc3347 \ + --hash=sha256:74c1485f7707cf707a7aef42ef6322b8f97921bd89be2ab6317fd782c2d53183 \ + --hash=sha256:895f61ef02e8fed38159bb70f7e100e00f471eae2bc838cd0f4ebb21e28f8541 \ + --hash=sha256:8c1be557ee92a20f184922c7b6424e8ab6691788e6d86137c5d93c1a6ec1b8fb \ + --hash=sha256:bb4191dfc9306777bc594117aee052446b3fa88737cd13b7188d0e7aa8162185 \ + --hash=sha256:bfb51918d4ff3d77c1c856a9699f8492c612cde32fd3bcd344af9be34999bfdc \ + --hash=sha256:c20cfa2d49991c8b4147af39859b167664f2ad4561704ee74c1de03318e898db \ + --hash=sha256:cb333c16912324fd5f769fff6bc5de372e9e7a202247b48870bc251ed40239aa \ + --hash=sha256:d2d9808ea7b4af864f35ea216be506ecec180628aced0704e34aca0b040ffe46 \ + --hash=sha256:d483ad4e639292c90170eb6f7783ad19490e7a8defb3e46f97dfe4bacae89122 \ + --hash=sha256:dd5de0646207f053eb0d6c74ae45ba98c3395a571a2891858e87df7c9b9bd51b \ + --hash=sha256:e1d4970ea66be07ae37a3c2e48b5ec63f7ba6804bdddfdbd3cfd954d25a82e63 \ + --hash=sha256:e4fac90784481d221a8e4b1162afa7c47ed953be40d31ab4629ae917510051df \ + --hash=sha256:fa5ae20527d8e831e8230cbffd9f8fe952815b2b7dae6ffec25318803a7528fc \ + --hash=sha256:fd7f6999a8070df521b6384004ef42833b9bd62cfee11a09bda1079b4b704247 \ + --hash=sha256:fdc842473cd33f45ff6bce46aea678a54e3d21f1b61a7750ce3c498eedfe25d6 \ + --hash=sha256:fe69978f3f768926cfa37b867e3843918e012cf83f680806599ddce33c2c68b0 + # via yamllint requests==2.25.1 \ --hash=sha256:27973dd4a904a4f13b263a19c866c13b92a39ed1c964655f025f3f8d3d75b804 \ --hash=sha256:c210084e36a42ae6b9219e00e48287def368a26d03a048ddad7bfee44f75871e @@ -24,3 +59,12 @@ urllib3==1.26.5 \ --hash=sha256:753a0374df26658f99d826cfe40394a686d05985786d946fbe4165b5148f5a7c \ --hash=sha256:a7acd0977125325f516bda9735fa7142b909a8d01e8b2e4c8108d0984e6e0098 # via requests +yamllint==1.26.3 \ + --hash=sha256:3934dcde484374596d6b52d8db412929a169f6d9e52e20f9ade5bf3523d9b96e + # via -r requirements.txt + +# The following packages are considered to be unsafe in a requirements file: +setuptools==57.5.0 \ + --hash=sha256:60d78588f15b048f86e35cdab73003d8b21dd45108ee61a6693881a427f22073 \ + --hash=sha256:d9d3266d50f59c6967b9312844470babbdb26304fe740833a5f8d89829ba3a24 + # via yamllint diff --git a/python/pip.bzl b/python/pip.bzl index 5027666806..785156f4b7 100644 --- a/python/pip.bzl +++ b/python/pip.bzl @@ -42,10 +42,40 @@ def pip_install(requirements, name = "pip", **kwargs): ) ``` + In addition to the `requirement` macro, which is used to access the generated `py_library` + target generated from a package's wheel, The generated `requirements.bzl` file contains + functionality for exposing [entry points][whl_ep] as `py_binary` targets as well. + + [whl_ep]: https://packaging.python.org/specifications/entry-points/ + + ```python + load("@pip_deps//:requirements.bzl", "entry_point") + + alias( + name = "pip-compile", + actual = entry_point( + pkg = "pip-tools", + script = "pip-compile", + ), + ) + ``` + + Note that for packages who's name and script are the same, only the name of the package + is needed when calling the `entry_point` macro. + + ```python + load("@pip_deps//:requirements.bzl", "entry_point") + + alias( + name = "flake8", + actual = entry_point("flake8"), + ) + ``` + Args: - requirements: A 'requirements.txt' pip requirements file. - name: A unique name for the created external repository (default 'pip'). - **kwargs: Keyword arguments passed directly to the `pip_repository` repository rule. + requirements: A 'requirements.txt' pip requirements file. + name: A unique name for the created external repository (default 'pip'). + **kwargs: Keyword arguments passed directly to the `pip_repository` repository rule. """ # Just in case our dependencies weren't already fetched @@ -58,6 +88,80 @@ def pip_install(requirements, name = "pip", **kwargs): ) def pip_parse(requirements_lock, name = "pip_parsed_deps", **kwargs): + """Imports a locked/compiled requirements file and generates a new `requirements.bzl` file. + + This is used via the `WORKSPACE` pattern: + + ```python + load("@rules_python//python:pip.bzl", "pip_parse") + + pip_parse( + name = "pip_deps", + requirements_lock = ":requirements.txt", + ) + + load("@pip_deps//:requirements.bzl", "install_deps") + + install_deps() + ``` + + You can then reference imported dependencies from your `BUILD` file with: + + ```python + load("@pip_deps//:requirements.bzl", "requirement") + + py_library( + name = "bar", + ... + deps = [ + "//my/other:dep", + requirement("requests"), + requirement("numpy"), + ], + ) + ``` + + In addition to the `requirement` macro, which is used to access the generated `py_library` + target generated from a package's wheel, The generated `requirements.bzl` file contains + functionality for exposing [entry points][whl_ep] as `py_binary` targets as well. + + [whl_ep]: https://packaging.python.org/specifications/entry-points/ + + ```python + load("@pip_deps//:requirements.bzl", "entry_point") + + alias( + name = "pip-compile", + actual = entry_point( + pkg = "pip-tools", + script = "pip-compile", + ), + ) + ``` + + Note that for packages who's name and script are the same, only the name of the package + is needed when calling the `entry_point` macro. + + ```python + load("@pip_deps//:requirements.bzl", "entry_point") + + alias( + name = "flake8", + actual = entry_point("flake8"), + ) + ``` + + Args: + requirements_lock (Label): A fully resolved 'requirements.txt' pip requirement file + containing the transitive set of your dependencies. If this file is passed instead + of 'requirements' no resolve will take place and pip_repository will create + individual repositories for each of your dependencies so that wheels are + fetched/built only for the targets specified by 'build/run/test'. + name (str, optional): The name of the generated repository. + **kwargs (dict): Additional keyword arguments for the underlying + `pip_repository` rule. + """ + # Just in case our dependencies weren't already fetched pip_install_dependencies() diff --git a/python/pip_install/extract_wheels/lib/bazel.py b/python/pip_install/extract_wheels/lib/bazel.py index 0dbc560d86..a51a41ad5e 100644 --- a/python/pip_install/extract_wheels/lib/bazel.py +++ b/python/pip_install/extract_wheels/lib/bazel.py @@ -4,6 +4,7 @@ import json from typing import Iterable, List, Dict, Set, Optional import shutil +from pathlib import Path from python.pip_install.extract_wheels.lib import namespace_pkgs, wheel, purelib @@ -12,10 +13,73 @@ PY_LIBRARY_LABEL = "pkg" DATA_LABEL = "data" DIST_INFO_LABEL = "dist_info" +WHEEL_ENTRY_POINT_PREFIX = "rules_python_wheel_entry_point" + + +def generate_entry_point_contents(entry_point: str, shebang: str = "#!/usr/bin/env python3") -> str: + """Generate the contents of an entry point script. + + Args: + entry_point (str): The name of the entry point as show in the + `console_scripts` section of `entry_point.txt`. + shebang (str, optional): The shebang to use for the entry point python + file. + + Returns: + str: A string of python code. + """ + module, method = entry_point.split(":", 1) + return textwrap.dedent("""\ + {shebang} + if __name__ == "__main__": + from {module} import {method} + {method}() + """.format( + shebang=shebang, + module=module, + method=method + )) + + +def generate_entry_point_rule(script: str, pkg: str) -> str: + """Generate a Bazel `py_binary` rule for an entry point script. + + Note that the script is used to determine the name of the target. The name of + entry point targets should be uniuqe to avoid conflicts with existing sources or + directories within a wheel. + + Args: + script (str): The path to the entry point's python file. + pkg (str): The package owning the entry point. This is expected to + match up with the `py_library` defined for each repository. + + + Returns: + str: A `py_binary` instantiation. + """ + name = os.path.splitext(script)[0] + return textwrap.dedent("""\ + py_binary( + name = "{name}", + srcs = ["{src}"], + # This makes this directory a top-level in the python import + # search path for anything that depends on this. + imports = ["."], + deps = ["{pkg}"], + ) + """.format( + name=name, + src=str(script).replace("\\", "/"), + pkg=pkg + )) def generate_build_file_contents( - name: str, dependencies: List[str], whl_file_deps: List[str], pip_data_exclude: List[str], + name: str, + dependencies: List[str], + whl_file_deps: List[str], + pip_data_exclude: List[str], + additional_targets: List[str] = [], ) -> str: """Generate a BUILD file for an unzipped Wheel @@ -23,6 +87,7 @@ def generate_build_file_contents( name: the target name of the py_library dependencies: a list of Bazel labels pointing to dependencies of the library whl_file_deps: a list of Bazel labels pointing to wheel file dependencies of this wheel. + additional_targets: A list of additional targets to append to the BUILD file contents. Returns: A complete BUILD file as a string @@ -31,12 +96,19 @@ def generate_build_file_contents( there may be no Python sources whatsoever (e.g. packages written in Cython: like `pymssql`). """ - data_exclude = ["*.whl", "**/*.py", "**/* *", "BUILD.bazel", "WORKSPACE"] + pip_data_exclude + data_exclude = [ + "*.whl", + "**/*.py", + f"{WHEEL_ENTRY_POINT_PREFIX}*.py", + "**/* *", + "BUILD.bazel", + "WORKSPACE", + ] + pip_data_exclude - return textwrap.dedent( + return "\n".join([textwrap.dedent( """\ - load("@rules_python//python:defs.bzl", "py_library") - + load("@rules_python//python:defs.bzl", "py_library", "py_binary") + package(default_visibility = ["//visibility:public"]) filegroup( @@ -57,7 +129,7 @@ def generate_build_file_contents( py_library( name = "{name}", - srcs = glob(["**/*.py"], allow_empty = True), + srcs = glob(["**/*.py"], exclude=["{entry_point_prefix}*.py"], allow_empty = True), data = glob(["**/*"], exclude={data_exclude}), # This makes this directory a top-level in the python import # search path for anything that depends on this. @@ -72,7 +144,8 @@ def generate_build_file_contents( whl_file_deps=",".join(whl_file_deps), data_label=DATA_LABEL, dist_info_label=DIST_INFO_LABEL, - ) + entry_point_prefix=WHEEL_ENTRY_POINT_PREFIX, + ))] + additional_targets ) @@ -114,6 +187,11 @@ def data_requirement(name): def dist_info_requirement(name): return requirement(name) + ":{dist_info_label}" + def entry_point(pkg, script = None): + if not script: + script = pkg + return requirement(pkg) + ":{entry_point_prefix}_" + script + def install_deps(): fail("install_deps() only works if you are creating an incremental repo. Did you mean to use pip_parse()?") """.format( @@ -123,6 +201,7 @@ def install_deps(): whl_file_label=WHEEL_FILE_LABEL, data_label=DATA_LABEL, dist_info_label=DIST_INFO_LABEL, + entry_point_prefix=WHEEL_ENTRY_POINT_PREFIX, ) ) @@ -262,12 +341,25 @@ def extract_wheel( sanitised_file_label(d) for d in whl_deps ] + library_name = PY_LIBRARY_LABEL if incremental else sanitise_name(whl.name) + + directory_path = Path(directory) + entry_points = [] + for name, entry_point in sorted(whl.entry_points().items()): + entry_point_script = f"{WHEEL_ENTRY_POINT_PREFIX}_{name}.py" + (directory_path / entry_point_script).write_text(generate_entry_point_contents(entry_point)) + entry_points.append(generate_entry_point_rule( + entry_point_script, + library_name, + )) + with open(os.path.join(directory, "BUILD.bazel"), "w") as build_file: contents = generate_build_file_contents( - PY_LIBRARY_LABEL if incremental else sanitise_name(whl.name), + library_name, sanitised_dependencies, sanitised_wheel_file_dependencies, - pip_data_exclude + pip_data_exclude, + entry_points, ) build_file.write(contents) diff --git a/python/pip_install/extract_wheels/lib/wheel.py b/python/pip_install/extract_wheels/lib/wheel.py index c13f4e8621..aa5b0ca2d3 100644 --- a/python/pip_install/extract_wheels/lib/wheel.py +++ b/python/pip_install/extract_wheels/lib/wheel.py @@ -1,4 +1,5 @@ """Utility class to inspect an extracted wheel directory""" +import configparser import glob import os import stat @@ -42,6 +43,32 @@ def name(self) -> str: def metadata(self) -> pkginfo.Wheel: return pkginfo.get_metadata(self.path) + def entry_points(self) -> Dict[str, str]: + """Returns the entrypoints defined in the current wheel + + See https://packaging.python.org/specifications/entry-points/ for more info + + Returns: + Dict[str, str]: A mappying of the entry point's name to it's method + """ + with zipfile.ZipFile(self.path, "r") as whl: + # Calculate the location of the entry_points.txt file + metadata = self.metadata + name = "{}-{}".format(metadata.name.replace("-", "_"), metadata.version) + entry_points_path = os.path.join("{}.dist-info".format(name), "entry_points.txt") + + # If this file does not exist in the wheel, there are no entry points + if entry_points_path not in whl.namelist(): + return dict() + + # Parse the avaialble entry points + config = configparser.ConfigParser() + config.read_string(whl.read(entry_points_path).decode("utf-8")) + if "console_scripts" in config.sections(): + return dict(config["console_scripts"]) + + return dict() + def dependencies(self, extras_requested: Optional[Set[str]] = None) -> Set[str]: dependency_set = set() diff --git a/python/pip_install/parse_requirements_to_bzl/__init__.py b/python/pip_install/parse_requirements_to_bzl/__init__.py index a77dd858f2..aa0f1b6a21 100644 --- a/python/pip_install/parse_requirements_to_bzl/__init__.py +++ b/python/pip_install/parse_requirements_to_bzl/__init__.py @@ -90,10 +90,15 @@ def whl_requirement(name): return "@{repo_prefix}" + _clean_name(name) + "//:{wheel_file_label}" def data_requirement(name): - return requirement(name) + ":{data_label}" + return "@{repo_prefix}" + _clean_name(name) + "//:{data_label}" def dist_info_requirement(name): - return requirement(name) + ":{dist_info_label}" + return "@{repo_prefix}" + _clean_name(name) + "//:{dist_info_label}" + + def entry_point(pkg, script = None): + if not script: + script = pkg + return "@{repo_prefix}" + _clean_name(pkg) + "//:{entry_point_prefix}_" + script def install_deps(): for name, requirement in _packages: @@ -112,6 +117,7 @@ def install_deps(): wheel_file_label=bazel.WHEEL_FILE_LABEL, data_label=bazel.DATA_LABEL, dist_info_label=bazel.DIST_INFO_LABEL, + entry_point_prefix=bazel.WHEEL_ENTRY_POINT_PREFIX, ) )