From 087fb575418b9fbdab4a6491a24cef517b8c2506 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sat, 25 Jan 2025 16:47:26 +0900 Subject: [PATCH] feat(uv): allow specifying any uv version to use This is using the `dist-manifest.json` on the GH releases page so that we can get the expected `sha256` value of each available file and download all of the usable archives. This means that `rules_python` no longer needs to be updated for `uv` version bumps. The remaining bits for closing the ticket: - [ ] Finalize the `lock` interface. - [ ] Add it to the `pip.parse` hub repo if `pyproject.toml` is passed in. - [ ] Add a rule/target for `venv` creation. Work towards #1975. --- CHANGELOG.md | 5 + MODULE.bazel | 80 +++++++++++- python/uv/private/BUILD.bazel | 7 -- python/uv/private/lock.bzl | 8 +- python/uv/private/uv.bzl | 173 ++++++++++++++++++++++++-- python/uv/private/uv_repositories.bzl | 67 +++++----- python/uv/private/versions.bzl | 94 -------------- 7 files changed, 281 insertions(+), 153 deletions(-) delete mode 100644 python/uv/private/versions.bzl diff --git a/CHANGELOG.md b/CHANGELOG.md index 24b83e3228..26c70a978e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -57,6 +57,11 @@ Unreleased changes template. * (rules) deprecation warnings for deprecated symbols have been turned off by default for now and can be enabled with `RULES_PYTHON_DEPRECATION_WARNINGS` env var. +* (uv) Now the extension can be fully configured via `bzlmod` APIs without the + need to patch `rules_python`. The documentation has been added to `rules_python` + docs but usage of the extension may result in your setup breaking without any + notice. What is more, the URLs and SHA256 values will be retrieved from the + GitHub releases page metadata published by the `uv` project. {#v0-0-0-fixed} ### Fixed diff --git a/MODULE.bazel b/MODULE.bazel index 7034357f61..5105344be3 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -173,14 +173,86 @@ use_repo( "build_bazel_bazel_self", ) -# EXPERIMENTAL: This is experimental and may be removed without notice -uv = use_extension( +uv = use_extension("//python/uv:uv.bzl", "uv") + +# Here is how we can define platforms for the `uv` binaries - this will affect +# all of the downstream callers because we are using the extension without +# `dev_dependency = True`. +uv.platform( + name = "aarch64-apple-darwin", + compatible_with = [ + "@platforms//os:macos", + "@platforms//cpu:aarch64", + ], + flag_values = {}, +) +uv.platform( + name = "aarch64-unknown-linux-gnu", + compatible_with = [ + "@platforms//os:linux", + "@platforms//cpu:aarch64", + ], + flag_values = {}, +) +uv.platform( + name = "powerpc64-unknown-linux-gnu", + compatible_with = [ + "@platforms//os:linux", + "@platforms//cpu:ppc", + ], + flag_values = {}, +) +uv.platform( + name = "powerpc64le-unknown-linux-gnu", + compatible_with = [ + "@platforms//os:linux", + "@platforms//cpu:ppc64le", + ], + flag_values = {}, +) +uv.platform( + name = "s390x-unknown-linux-gnu", + compatible_with = [ + "@platforms//os:linux", + "@platforms//cpu:s390x", + ], + flag_values = {}, +) +uv.platform( + name = "x86_64-apple-darwin", + compatible_with = [ + "@platforms//os:macos", + "@platforms//cpu:x86_64", + ], + flag_values = {}, +) +uv.platform( + name = "x86_64-pc-windows-msvc", + compatible_with = [ + "@platforms//os:windows", + "@platforms//cpu:x86_64", + ], + flag_values = {}, +) +uv.platform( + name = "x86_64-unknown-linux-gnu", + compatible_with = [ + "@platforms//os:linux", + "@platforms//cpu:x86_64", + ], + flag_values = {}, +) + +uv_dev = use_extension( "//python/uv:uv.bzl", "uv", dev_dependency = True, ) -uv.toolchain(uv_version = "0.4.25") -use_repo(uv, "uv_toolchains") +uv_dev.toolchain( + name = "uv_toolchains", + version = "0.5.24", +) +use_repo(uv_dev, "uv_toolchains") register_toolchains( "@uv_toolchains//:all", diff --git a/python/uv/private/BUILD.bazel b/python/uv/private/BUILD.bazel index 006c856d02..9a69f04a41 100644 --- a/python/uv/private/BUILD.bazel +++ b/python/uv/private/BUILD.bazel @@ -57,7 +57,6 @@ bzl_library( deps = [ ":toolchain_types_bzl", ":uv_toolchains_repo_bzl", - ":versions_bzl", ], ) @@ -82,9 +81,3 @@ bzl_library( "//python/private:text_util_bzl", ], ) - -bzl_library( - name = "versions_bzl", - srcs = ["versions.bzl"], - visibility = ["//python/uv:__subpackages__"], -) diff --git a/python/uv/private/lock.bzl b/python/uv/private/lock.bzl index e0491b282c..9378f180db 100644 --- a/python/uv/private/lock.bzl +++ b/python/uv/private/lock.bzl @@ -30,9 +30,11 @@ def lock(*, name, srcs, out, upgrade = False, universal = True, args = [], **kwa """Pin the requirements based on the src files. Differences with the current {obj}`compile_pip_requirements` rule: - - This is implemented in shell and uv. + - This is implemented in shell and `uv`. - This does not error out if the output file does not exist yet. - Supports transitions out of the box. + - The execution of the lock file generation is happening inside of a build + action in a `genrule`. Args: name: The name of the target to run for updating the requirements. @@ -41,8 +43,8 @@ def lock(*, name, srcs, out, upgrade = False, universal = True, args = [], **kwa upgrade: Tell `uv` to always upgrade the dependencies instead of keeping them as they are. universal: Tell `uv` to generate a universal lock file. - args: Extra args to pass to `uv`. - **kwargs: Extra kwargs passed to the {obj}`py_binary` rule. + args: Extra args to pass to the rule. + **kwargs: Extra kwargs passed to the binary rule. """ pkg = native.package_name() update_target = name + ".update" diff --git a/python/uv/private/uv.bzl b/python/uv/private/uv.bzl index 886e7fe748..4c0a33f4e7 100644 --- a/python/uv/private/uv.bzl +++ b/python/uv/private/uv.bzl @@ -22,32 +22,181 @@ load(":uv_repositories.bzl", "uv_repositories") _DOC = """\ A module extension for working with uv. + +Use it in your own setup by: +```starlark +uv = use_extension( + "@rules_python//python/uv:uv.bzl", + "uv", + dev_dependency = True, +) +uv.toolchain( + name = "uv_toolchains", + version = "0.5.24", +) +use_repo(uv, "uv_toolchains") + +register_toolchains( + "@uv_toolchains//:all", + dev_dependency = True, +) +``` + +Since this is only for locking the requirements files, it should be always +marked as a `dev_dependency`. """ +_DIST_MANIFEST_JSON = "dist-manifest.json" +_DEFAULT_BASE_URL = "https://github.com/astral-sh/uv/releases/download" + +config = tag_class( + doc = "Configure where the binaries are going to be downloaded from.", + attrs = { + "base_url": attr.string( + doc = "Base URL to download metadata about the binaries and the binaries themselves.", + default = _DEFAULT_BASE_URL, + ), + }, +) + +platform = tag_class( + doc = "Configure the available platforms for lock file generation.", + attrs = { + "compatible_with": attr.label_list( + doc = "The compatible with constraint values for toolchain resolution", + ), + "flag_values": attr.label_keyed_string_dict( + doc = "The flag values for toolchain resolution", + ), + "name": attr.string( + doc = "The platform string used in the UV repository to denote the platform triple.", + mandatory = True, + ), + }, +) + uv_toolchain = tag_class( doc = "Configure uv toolchain for lock file generation.", attrs = { - "uv_version": attr.string(doc = "Explicit version of uv.", mandatory = True), + "name": attr.string( + doc = "The name of the toolchain repo", + default = "uv_toolchains", + ), + "version": attr.string( + doc = "Explicit version of uv.", + mandatory = True, + ), }, ) def _uv_toolchain_extension(module_ctx): + config = { + "platforms": {}, + } + for mod in module_ctx.modules: + if not mod.is_root and not mod.name == "rules_python": + # Only rules_python and the root module can configure this. + # + # Ignore any attempts to configure the `uv` toolchain elsewhere + # + # Only the root module may configure the uv toolchain. + # This prevents conflicting registrations with any other modules. + # + # NOTE: We may wish to enforce a policy where toolchain configuration is only allowed in the root module, or in rules_python. See https://github.com/bazelbuild/bazel/discussions/22024 + continue + + # Note, that the first registration will always win, givin priority to + # the root module. + + for platform_attr in mod.tags.platform: + config["platforms"].setdefault(platform_attr.name, struct( + name = platform_attr.name.replace("-", "_").lower(), + compatible_with = platform_attr.compatible_with, + flag_values = platform_attr.flag_values, + )) + + for config_attr in mod.tags.config: + config.setdefault("base_url", config_attr.base_url) + for toolchain in mod.tags.toolchain: - if not mod.is_root: - fail( - "Only the root module may configure the uv toolchain.", - "This prevents conflicting registrations with any other modules.", - "NOTE: We may wish to enforce a policy where toolchain configuration is only allowed in the root module, or in rules_python. See https://github.com/bazelbuild/bazel/discussions/22024", - ) - - uv_repositories( - uv_version = toolchain.uv_version, - register_toolchains = False, + config.setdefault("version", toolchain.version) + config.setdefault("name", toolchain.name) + + if not config["version"]: + return + + config.setdefault("base_url", _DEFAULT_BASE_URL) + config["urls"] = _get_tool_urls_from_dist_manifest( + module_ctx, + base_url = "{base_url}/{version}".format(**config), + ) + uv_repositories( + name = config["name"], + platforms = config["platforms"], + urls = config["urls"], + version = config["version"], + ) + +def _get_tool_urls_from_dist_manifest(module_ctx, *, base_url): + """Download the results about remote tool sources. + + This relies on the tools using the cargo packaging to infer the actual + sha256 values for each binary. + """ + dist_manifest = module_ctx.path(_DIST_MANIFEST_JSON) + module_ctx.download(base_url + "/" + _DIST_MANIFEST_JSON, output = dist_manifest) + dist_manifest = json.decode(module_ctx.read(dist_manifest)) + + artifacts = dist_manifest["artifacts"] + tool_sources = {} + downloads = {} + for fname, artifact in artifacts.items(): + if artifact.get("kind") != "executable-zip": + continue + + checksum = artifacts[artifact["checksum"]] + checksum_fname = checksum["name"] + checksum_path = module_ctx.path(checksum_fname) + downloads[checksum_path] = struct( + download = module_ctx.download( + "{}/{}".format(base_url, checksum_fname), + output = checksum_path, + block = False, + ), + archive_fname = fname, + platforms = checksum["target_triples"], + ) + + for checksum_path, download in downloads.items(): + result = download.download.wait() + if not result.success: + fail(result) + + archive_fname = download.archive_fname + + sha256, _, checksummed_fname = module_ctx.read(checksum_path).partition(" ") + checksummed_fname = checksummed_fname.strip(" *\n") + if archive_fname != checksummed_fname: + fail("The checksum is for a different file, expected '{}' but got '{}'".format( + archive_fname, + checksummed_fname, + )) + + for platform in download.platforms: + tool_sources[platform] = struct( + urls = ["{}/{}".format(base_url, archive_fname)], + sha256 = sha256, ) + return tool_sources + uv = module_extension( doc = _DOC, implementation = _uv_toolchain_extension, - tag_classes = {"toolchain": uv_toolchain}, + tag_classes = { + "config": config, + "platform": platform, + "toolchain": uv_toolchain, + }, ) diff --git a/python/uv/private/uv_repositories.bzl b/python/uv/private/uv_repositories.bzl index 24fb9c2447..0ea66d4e79 100644 --- a/python/uv/private/uv_repositories.bzl +++ b/python/uv/private/uv_repositories.bzl @@ -20,7 +20,6 @@ Create repositories for uv toolchain dependencies load(":toolchain_types.bzl", "UV_TOOLCHAIN_TYPE") load(":uv_toolchains_repo.bzl", "uv_toolchains_repo") -load(":versions.bzl", "UV_PLATFORMS", "UV_TOOL_VERSIONS") UV_BUILD_TMPL = """\ # Generated by repositories.bzl @@ -35,27 +34,17 @@ uv_toolchain( def _uv_repo_impl(repository_ctx): platform = repository_ctx.attr.platform - uv_version = repository_ctx.attr.uv_version is_windows = "windows" in platform - - suffix = ".zip" if is_windows else ".tar.gz" - filename = "uv-{platform}{suffix}".format( - platform = platform, - suffix = suffix, - ) - url = "https://github.com/astral-sh/uv/releases/download/{version}/{filename}".format( - version = uv_version, - filename = filename, - ) + _, _, filename = repository_ctx.attr.urls[0].rpartition("/") if filename.endswith(".tar.gz"): strip_prefix = filename[:-len(".tar.gz")] else: strip_prefix = "" - repository_ctx.download_and_extract( - url = url, - sha256 = UV_TOOL_VERSIONS[repository_ctx.attr.uv_version][repository_ctx.attr.platform].sha256, + result = repository_ctx.download_and_extract( + url = repository_ctx.attr.urls, + sha256 = repository_ctx.attr.sha256, stripPrefix = strip_prefix, ) @@ -64,49 +53,64 @@ def _uv_repo_impl(repository_ctx): "BUILD.bazel", UV_BUILD_TMPL.format( binary = binary, - version = uv_version, + version = repository_ctx.attr.version, ), ) + return { + "name": repository_ctx.attr.name, + "platform": repository_ctx.attr.platform, + "sha256": result.sha256, + "urls": repository_ctx.attr.urls, + "version": repository_ctx.attr.version, + } + uv_repository = repository_rule( _uv_repo_impl, doc = "Fetch external tools needed for uv toolchain", attrs = { - "platform": attr.string(mandatory = True, values = UV_PLATFORMS.keys()), - "uv_version": attr.string(mandatory = True, values = UV_TOOL_VERSIONS.keys()), + "platform": attr.string(mandatory = True), + "sha256": attr.string(mandatory = False), + "urls": attr.string_list(mandatory = True), + "version": attr.string(mandatory = True), }, ) -def uv_repositories(name = "uv_toolchains", uv_version = None, register_toolchains = True): +def uv_repositories(*, name, version, platforms, urls): """Convenience macro which does typical toolchain setup Skip this macro if you need more control over the toolchain setup. Args: - name: {type}`str` The name of the toolchains repo. - uv_version: The uv toolchain version to download. - register_toolchains: If true, repositories will be generated to produce and register `uv_toolchain` targets. + name: The name of the toolchains repo, + version: The uv toolchain version to download. + platforms: The platforms to register uv for. + urls: The urls with sha256 values to register uv for. """ - if not uv_version: - fail("uv_version is required") + if not version: + fail("version is required") toolchain_names = [] toolchain_labels_by_toolchain = {} toolchain_compatible_with_by_toolchain = {} - for platform in UV_PLATFORMS.keys(): - uv_repository_name = UV_PLATFORMS[platform].default_repo_name - + for platform_name, platform in platforms.items(): + uv_repository_name = "{}_{}".format(name, platform_name.lower().replace("-", "_")) uv_repository( name = uv_repository_name, - uv_version = uv_version, - platform = platform, + version = version, + platform = platform_name, + urls = urls[platform_name].urls, + sha256 = urls[platform_name].sha256, ) toolchain_name = uv_repository_name + "_toolchain" toolchain_names.append(toolchain_name) toolchain_labels_by_toolchain[toolchain_name] = "@{}//:uv_toolchain".format(uv_repository_name) - toolchain_compatible_with_by_toolchain[toolchain_name] = UV_PLATFORMS[platform].compatible_with + toolchain_compatible_with_by_toolchain[toolchain_name] = [ + str(label) + for label in platform.compatible_with + ] uv_toolchains_repo( name = name, @@ -115,6 +119,3 @@ def uv_repositories(name = "uv_toolchains", uv_version = None, register_toolchai toolchain_labels = toolchain_labels_by_toolchain, toolchain_compatible_with = toolchain_compatible_with_by_toolchain, ) - - if register_toolchains: - native.register_toolchains("@{}/:all".format(name)) diff --git a/python/uv/private/versions.bzl b/python/uv/private/versions.bzl deleted file mode 100644 index 1d68302c74..0000000000 --- a/python/uv/private/versions.bzl +++ /dev/null @@ -1,94 +0,0 @@ -# Copyright 2024 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Version and integrity information for downloaded artifacts""" - -UV_PLATFORMS = { - "aarch64-apple-darwin": struct( - default_repo_name = "uv_darwin_aarch64", - compatible_with = [ - "@platforms//os:macos", - "@platforms//cpu:aarch64", - ], - ), - "aarch64-unknown-linux-gnu": struct( - default_repo_name = "uv_linux_aarch64", - compatible_with = [ - "@platforms//os:linux", - "@platforms//cpu:aarch64", - ], - ), - "powerpc64le-unknown-linux-gnu": struct( - default_repo_name = "uv_linux_ppc", - compatible_with = [ - "@platforms//os:linux", - "@platforms//cpu:ppc", - ], - ), - "s390x-unknown-linux-gnu": struct( - default_repo_name = "uv_linux_s390x", - compatible_with = [ - "@platforms//os:linux", - "@platforms//cpu:s390x", - ], - ), - "x86_64-apple-darwin": struct( - default_repo_name = "uv_darwin_x86_64", - compatible_with = [ - "@platforms//os:macos", - "@platforms//cpu:x86_64", - ], - ), - "x86_64-pc-windows-msvc": struct( - default_repo_name = "uv_windows_x86_64", - compatible_with = [ - "@platforms//os:windows", - "@platforms//cpu:x86_64", - ], - ), - "x86_64-unknown-linux-gnu": struct( - default_repo_name = "uv_linux_x86_64", - compatible_with = [ - "@platforms//os:linux", - "@platforms//cpu:x86_64", - ], - ), -} - -# From: https://github.com/astral-sh/uv/releases -UV_TOOL_VERSIONS = { - "0.4.25": { - "aarch64-apple-darwin": struct( - sha256 = "bb2ff4348114ef220ca52e44d5086640c4a1a18f797a5f1ab6f8559fc37b1230", - ), - "aarch64-unknown-linux-gnu": struct( - sha256 = "4485852eb8013530c4275cd222c0056ce123f92742321f012610f1b241463f39", - ), - "powerpc64le-unknown-linux-gnu": struct( - sha256 = "32421c61e8d497243171b28c7efd74f039251256ae9e57ce4a457fdd7d045e24", - ), - "s390x-unknown-linux-gnu": struct( - sha256 = "9afa342d87256f5178a592d3eeb44ece8a93e9359db37e31be1b092226338469", - ), - "x86_64-apple-darwin": struct( - sha256 = "f0ec1f79f4791294382bff242691c6502e95853acef080ae3f7c367a8e1beb6f", - ), - "x86_64-pc-windows-msvc": struct( - sha256 = "c5c7fa084ae4e8ac9e3b0b6c4c7b61e9355eb0c86801c4c7728c0cb142701f38", - ), - "x86_64-unknown-linux-gnu": struct( - sha256 = "6cb6eaf711cd7ce5fb1efaa539c5906374c762af547707a2041c9f6fd207769a", - ), - }, -}