From ea03ac262ac0a00852ae496cc38692ca7ce9b5f7 Mon Sep 17 00:00:00 2001 From: Cameron Martin Date: Thu, 25 Aug 2022 18:54:51 +0100 Subject: [PATCH 01/12] Support bzlmod This is a rough proof of concept of making rules_rust load dependencies via bzlmod. Currently it can only compile and run rust crates, and cannot load crates from Cargo.toml. There is one new module, `examples/bzlmod/hello_world`, that depends on the root `rules_rust` module. This example can be built and run using: ``` cd examples/bzlmod/hello_world bazel run //:hello_world ``` A "hub" repository for toolchains is defined, and can be registered using: ``` register_toolchains("@rust_toolchains//:all") ``` --- MODULE.bazel | 23 ++++ examples/bzlmod/hello_world/.bazelrc | 1 + examples/bzlmod/hello_world/.gitignore | 1 + examples/bzlmod/hello_world/BUILD.bazel | 13 +++ examples/bzlmod/hello_world/MODULE.bazel | 22 ++++ examples/bzlmod/hello_world/WORKSPACE | 0 examples/bzlmod/hello_world/src/main.rs | 17 +++ extensions.bzl | 45 ++++++++ rust/private/repository_utils.bzl | 87 ++++++++++++--- rust/repositories.bzl | 130 +++++++++++++++-------- 10 files changed, 280 insertions(+), 59 deletions(-) create mode 100644 MODULE.bazel create mode 100644 examples/bzlmod/hello_world/.bazelrc create mode 100644 examples/bzlmod/hello_world/.gitignore create mode 100644 examples/bzlmod/hello_world/BUILD.bazel create mode 100644 examples/bzlmod/hello_world/MODULE.bazel create mode 100644 examples/bzlmod/hello_world/WORKSPACE create mode 100644 examples/bzlmod/hello_world/src/main.rs create mode 100644 extensions.bzl diff --git a/MODULE.bazel b/MODULE.bazel new file mode 100644 index 0000000000..c67cd5dd72 --- /dev/null +++ b/MODULE.bazel @@ -0,0 +1,23 @@ +module( + name = "rules_rust", + version = "0.19.0", +) + +bazel_dep(name = "platforms", version = "0.0.5") +bazel_dep(name = "rules_cc", version = "0.0.1") +bazel_dep(name = "bazel_skylib", version = "1.2.0") +bazel_dep(name = "apple_support", version = "0.13.0") + +# Load tinyjson without using the process wrapper +raw_cargo = use_extension("//:extensions.bzl", "raw_cargo") + +raw_cargo.crate( + name = "tinyjson", + sha256 = "1a8304da9f9370f6a6f9020b7903b044aa9ce3470f300a1fba5bc77c78145a16", + version = "2.3.0", +) + +use_repo( + raw_cargo, + rules_rust_tinyjson = "raw_cargo_tinyjson_2_3_0", +) diff --git a/examples/bzlmod/hello_world/.bazelrc b/examples/bzlmod/hello_world/.bazelrc new file mode 100644 index 0000000000..c8428d9570 --- /dev/null +++ b/examples/bzlmod/hello_world/.bazelrc @@ -0,0 +1 @@ +build --experimental_enable_bzlmod \ No newline at end of file diff --git a/examples/bzlmod/hello_world/.gitignore b/examples/bzlmod/hello_world/.gitignore new file mode 100644 index 0000000000..65e8edca79 --- /dev/null +++ b/examples/bzlmod/hello_world/.gitignore @@ -0,0 +1 @@ +/bazel-* \ No newline at end of file diff --git a/examples/bzlmod/hello_world/BUILD.bazel b/examples/bzlmod/hello_world/BUILD.bazel new file mode 100644 index 0000000000..3e2ec1932e --- /dev/null +++ b/examples/bzlmod/hello_world/BUILD.bazel @@ -0,0 +1,13 @@ +load("@rules_rust//rust:defs.bzl", "rust_binary", "rust_doc") + +package(default_visibility = ["//visibility:public"]) + +rust_binary( + name = "hello_world", + srcs = ["src/main.rs"], +) + +rust_doc( + name = "hello_world_doc", + crate = ":hello_world", +) diff --git a/examples/bzlmod/hello_world/MODULE.bazel b/examples/bzlmod/hello_world/MODULE.bazel new file mode 100644 index 0000000000..e34a5592e0 --- /dev/null +++ b/examples/bzlmod/hello_world/MODULE.bazel @@ -0,0 +1,22 @@ +module( + name = "hello_world", + version = "1.0", +) + +bazel_dep(name = "rules_rust", version = "0.9.0") + +local_path_override( + module_name = "rules_rust", + path = "../../..", +) + +rust_toolchains = use_extension("@rules_rust//:extensions.bzl", "toolchains") + +rust_toolchains.toolchain(edition = "2021") + +use_repo( + rust_toolchains, + "rust_toolchains", +) + +register_toolchains("@rust_toolchains//:all") diff --git a/examples/bzlmod/hello_world/WORKSPACE b/examples/bzlmod/hello_world/WORKSPACE new file mode 100644 index 0000000000..e69de29bb2 diff --git a/examples/bzlmod/hello_world/src/main.rs b/examples/bzlmod/hello_world/src/main.rs new file mode 100644 index 0000000000..317f564583 --- /dev/null +++ b/examples/bzlmod/hello_world/src/main.rs @@ -0,0 +1,17 @@ +// Copyright 2015 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +fn main() { + println!("Hello, world!"); +} diff --git a/extensions.bzl b/extensions.bzl new file mode 100644 index 0000000000..7c5e04a8e1 --- /dev/null +++ b/extensions.bzl @@ -0,0 +1,45 @@ +"Module extensions for using rules_rust with bzlmod" + +load("//rust:repositories.bzl", "rust_register_toolchains") +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") + +def _toolchains_impl(ctx): + mod = ctx.modules[0] + for toolchain in mod.tags.toolchain: + rust_register_toolchains(edition = toolchain.edition, register_toolchains = False) + +toolchains_toolchain = tag_class(attrs = {"edition": attr.string()}) +toolchains = module_extension( + implementation = _toolchains_impl, + tag_classes = {"toolchain": toolchains_toolchain}, +) + +def _create_build_file_content(name): + return """ +load("@rules_rust//rust/private:rust.bzl", "rust_library_without_process_wrapper") + +rust_library_without_process_wrapper( + name = "{}", + srcs = glob(["src/*.rs"]), + edition = "2018", + visibility = ["@rules_rust//util/process_wrapper:__pkg__"], +) +""".format(name) + +def _raw_cargo_impl(ctx): + for mod in ctx.modules: + for crate in mod.tags.crate: + http_archive( + name = "raw_cargo_{}_{}".format(crate.name, crate.version.replace(".", "_")), + sha256 = crate.sha256, + url = "https://crates.io/api/v1/crates/{}/{}/download".format(crate.name, crate.version), + strip_prefix = "{}-{}".format(crate.name, crate.version), + type = "tar.gz", + build_file_content = _create_build_file_content(crate.name), + ) + +raw_cargo_crate = tag_class(attrs = {"name": attr.string(), "version": attr.string(), "sha256": attr.string()}) +raw_cargo = module_extension( + implementation = _raw_cargo_impl, + tag_classes = {"crate": raw_cargo_crate}, +) diff --git a/rust/private/repository_utils.bzl b/rust/private/repository_utils.bzl index 9bf63cd5a9..6a66e2ba5c 100644 --- a/rust/private/repository_utils.bzl +++ b/rust/private/repository_utils.bzl @@ -220,15 +220,15 @@ load("@rules_rust//rust:toolchain.bzl", "rust_toolchain") rust_toolchain( name = "{toolchain_name}", - rust_doc = "@{workspace_name}//:rustdoc", - rust_std = "@{workspace_name}//:rust_std-{target_triple}", - rustc = "@{workspace_name}//:rustc", + rust_doc = "//:rustdoc", + rust_std = "//:rust_std-{target_triple}", + rustc = "//:rustc", rustfmt = {rustfmt_label}, - cargo = "@{workspace_name}//:cargo", - clippy_driver = "@{workspace_name}//:clippy_driver_bin", + cargo = "//:cargo", + clippy_driver = "//:clippy_driver_bin", llvm_cov = {llvm_cov_label}, llvm_profdata = {llvm_profdata_label}, - rustc_lib = "@{workspace_name}//:rustc_lib", + rustc_lib = "//:rustc_lib", allocator_library = {allocator_library}, binary_ext = "{binary_ext}", staticlib_ext = "{staticlib_ext}", @@ -243,7 +243,6 @@ rust_toolchain( """ def BUILD_for_rust_toolchain( - workspace_name, name, exec_triple, target_triple, @@ -255,7 +254,6 @@ def BUILD_for_rust_toolchain( """Emits a toolchain declaration to match an existing compiler and stdlib. Args: - workspace_name (str): The name of the workspace that this toolchain resides in name (str): The name of the toolchain declaration exec_triple (triple): The rust-style target that this compiler runs on target_triple (triple): The rust-style target triple of the tool @@ -276,19 +274,18 @@ def BUILD_for_rust_toolchain( rustfmt_label = "None" if include_rustfmt: - rustfmt_label = "\"@{workspace_name}//:rustfmt_bin\"".format(workspace_name = workspace_name) + rustfmt_label = "\"//:rustfmt_bin\"" llvm_cov_label = "None" llvm_profdata_label = "None" if include_llvm_tools: - llvm_cov_label = "\"@{workspace_name}//:llvm_cov_bin\"".format(workspace_name = workspace_name) - llvm_profdata_label = "\"@{workspace_name}//:llvm_profdata_bin\"".format(workspace_name = workspace_name) + llvm_cov_label = "\"//:llvm_cov_bin\"" + llvm_profdata_label = "\"//:llvm_profdata_bin\"" allocator_library_label = "None" if allocator_library: allocator_library_label = "\"{allocator_library}\"".format(allocator_library = allocator_library) return _build_file_for_rust_toolchain_template.format( toolchain_name = name, - workspace_name = workspace_name, binary_ext = system_to_binary_ext(target_triple.system), staticlib_ext = system_to_staticlib_ext(target_triple.system), dylib_ext = system_to_dylib_ext(target_triple.system), @@ -310,6 +307,7 @@ toolchain( target_compatible_with = {target_constraint_sets_serialized}, toolchain = "{toolchain}", toolchain_type = "{toolchain_type}", + visibility = ["//visibility:public"], {target_settings} ) """ @@ -774,3 +772,68 @@ def select_rust_version(versions): current = ver return current + +_build_file_for_toolchain_hub_template = """ +toolchain( + name = "{name}", + exec_compatible_with = {exec_constraint_sets_serialized}, + target_compatible_with = {target_constraint_sets_serialized}, + toolchain = "{toolchain}", + toolchain_type = "{toolchain_type}", + visibility = ["//visibility:public"], +) +""" + +def BUILD_for_toolchain_hub( + toolchain_names, + toolchain_labels, + toolchain_types, + target_compatible_with, + exec_compatible_with): + return "\n".join([_build_file_for_toolchain_hub_template.format( + name = toolchain_name, + exec_constraint_sets_serialized = json.encode(exec_compatible_with[toolchain_name]), + target_constraint_sets_serialized = json.encode(target_compatible_with[toolchain_name]), + toolchain = toolchain_labels[toolchain_name], + toolchain_type = toolchain_types[toolchain_name], + ) for toolchain_name in toolchain_names]) + +def _toolchain_repository_hub_impl(repository_ctx): + repository_ctx.file("WORKSPACE.bazel", """workspace(name = "{}")""".format( + repository_ctx.name, + )) + + repository_ctx.file("BUILD.bazel", BUILD_for_toolchain_hub( + toolchain_names = repository_ctx.attr.toolchain_names, + toolchain_labels = repository_ctx.attr.toolchain_labels, + toolchain_types = repository_ctx.attr.toolchain_types, + target_compatible_with = repository_ctx.attr.target_compatible_with, + exec_compatible_with = repository_ctx.attr.exec_compatible_with, + )) + +toolchain_repository_hub = repository_rule( + doc = ( + "Generates a toolchain-bearing repository that declares a set of other toolchains from other " + + "repositories. This exists to allow registering a set of toolchains in one go with the `:all` target." + ), + attrs = { + "toolchain_names": attr.string_list(mandatory = True), + "exec_compatible_with": attr.string_list_dict( + doc = "A list of constraints for the execution platform for this toolchain, keyed by toolchain name.", + mandatory = True, + ), + "target_compatible_with": attr.string_list_dict( + doc = "A list of constraints for the target platform for this toolchain, keyed by toolchain name.", + mandatory = True, + ), + "toolchain_labels": attr.string_dict( + doc = "The name of the toolchain implementation target, keyed by toolchain name.", + mandatory = True, + ), + "toolchain_types": attr.string_dict( + doc = "The toolchain type of the toolchain to declare, keyed by toolchain name.", + mandatory = True, + ), + }, + implementation = _toolchain_repository_hub_impl, +) diff --git a/rust/repositories.bzl b/rust/repositories.bzl index 5b67333b55..6a94fec8f2 100644 --- a/rust/repositories.bzl +++ b/rust/repositories.bzl @@ -24,6 +24,7 @@ load( "load_rustc_dev_nightly", "load_rustfmt", "select_rust_version", + "toolchain_repository_hub", _load_arbitrary_tool = "load_arbitrary_tool", ) @@ -180,6 +181,12 @@ def rust_register_toolchains( if rust_analyzer_version.startswith(("beta", "nightly")): rust_analyzer_version, _, rust_analyzer_iso_date = rustfmt_version.partition("/") + toolchain_names = [] + toolchain_labels = {} + toolchain_types = {} + exec_compatible_with_by_toolchain = {} + target_compatible_with_by_toolchain = {} + maybe( rust_analyzer_toolchain_repository, name = rust_analyzer_repo_name, @@ -189,6 +196,14 @@ def rust_register_toolchains( iso_date = rust_analyzer_iso_date, ) + toolchain_names.append(rust_analyzer_repo_name) + toolchain_labels[rust_analyzer_repo_name] = "@{}_srcs//:rust_analyzer_toolchain".format( + rust_analyzer_repo_name, + ) + exec_compatible_with_by_toolchain[rust_analyzer_repo_name] = [] + target_compatible_with_by_toolchain[rust_analyzer_repo_name] = [] + toolchain_types[rust_analyzer_repo_name] = "@rules_rust//rust/rust_analyzer:toolchain_type" + if register_toolchains: native.register_toolchains("@{}//:toolchain".format( rust_analyzer_repo_name, @@ -234,6 +249,22 @@ def rust_register_toolchains( rustfmt_repo_name, )) + for toolchain in get_toolchain_repositories(name, exec_triple, extra_target_triples, versions, iso_date): + toolchain_names.append(toolchain.name) + toolchain_labels[toolchain.name] = "@{}//:{}".format(toolchain.name + "_tools", "rust_toolchain") + exec_compatible_with_by_toolchain[toolchain.name] = triple_to_constraint_set(exec_triple) + target_compatible_with_by_toolchain[toolchain.name] = triple_to_constraint_set(toolchain.target_triple) + toolchain_types[toolchain.name] = "@rules_rust//rust:toolchain" + + toolchain_repository_hub( + name = "rust_toolchains", + toolchain_names = toolchain_names, + toolchain_labels = toolchain_labels, + toolchain_types = toolchain_types, + exec_compatible_with = exec_compatible_with_by_toolchain, + target_compatible_with = target_compatible_with_by_toolchain, + ) + # buildifier: disable=unnamed-macro def rust_repositories(**kwargs): """**Deprecated**: Use [rules_rust_dependencies](#rules_rust_dependencies) \ @@ -317,7 +348,6 @@ def _rust_toolchain_tools_repository_impl(ctx): allocator_library = ctx.attr.allocator_library, target_triple = target_triple, stdlib_linkflags = stdlib_linkflags, - workspace_name = ctx.attr.name, default_edition = ctx.attr.edition, include_rustfmt = not (not ctx.attr.rustfmt_version), include_llvm_tools = include_llvm_tools, @@ -475,9 +505,6 @@ def rust_toolchain_repository( urls (list, optional): A list of mirror urls containing the tools from the Rust-lang static file server. These must contain the '{}' used to substitute the tool being fetched (using .format). Defaults to ['https://static.rust-lang.org/dist/{}.tar.gz'] auth (dict): Auth object compatible with repository_ctx.download to use when downloading files. See [repository_ctx.download](https://docs.bazel.build/versions/main/skylark/lib/repository_ctx.html#download) for more details. - - Returns: - str: The name of the registerable toolchain created by this rule. """ if rustfmt_version in ("nightly", "beta"): @@ -519,10 +546,6 @@ def rust_toolchain_repository( target_compatible_with = target_compatible_with, ) - return "@{name}//:toolchain".format( - name = name, - ) - def _rust_analyzer_toolchain_tools_repository_impl(repository_ctx): load_rust_src( ctx = repository_ctx, @@ -783,6 +806,40 @@ rust_toolchain_set_repository = repository_rule( implementation = _rust_toolchain_set_repository_impl, ) +def get_toolchain_repositories(name, exec_triple, extra_target_triples, versions, iso_date): + toolchain_repos = [] + + for target_triple in [exec_triple] + extra_target_triples: + # Parse all provided versions while checking for duplicates + channels = {} + for version in versions: + if version.startswith(("beta", "nightly")): + channel, _, date = version.partition("/") + ver = channel + else: + channel = "stable" + date = iso_date + ver = version + + if channel in channels: + fail("Duplicate {} channels provided for {}: {}".format(channel, name, versions)) + + channels.update({channel: struct( + name = channel, + iso_date = date, + version = ver, + )}) + + # Define toolchains for each requested version + for channel in channels.values(): + toolchain_repos.append(struct( + name = "{}__{}__{}".format(name, target_triple, channel.name), + target_triple = target_triple, + channel = channel, + )) + + return toolchain_repos + def rust_repository_set( name, exec_triple, @@ -847,46 +904,25 @@ def rust_repository_set( versions = [version] all_toolchain_names = [] - for target_triple in [exec_triple] + extra_target_triples: - # Parse all provided versions while checking for duplicates - channels = {} - for version in versions: - if version.startswith(("beta", "nightly")): - channel, _, date = version.partition("/") - ver = channel - else: - channel = "stable" - date = iso_date - ver = version - - if channel in channels: - fail("Duplicate {} channels provided for {}: {}".format(channel, name, versions)) - - channels.update({channel: struct( - iso_date = date, - version = ver, - )}) - - # Define toolchains for each requested version - for channel, info in channels.items(): - toolchain_name = "{}__{}__{}".format(name, target_triple, channel) + for toolchain in get_toolchain_repositories(name, exec_triple, extra_target_triples, versions, iso_date): + rust_toolchain_repository( + name = toolchain.name, + allocator_library = allocator_library, + auth = auth, + channel = toolchain.channel.name, + dev_components = dev_components, + edition = edition, + exec_triple = exec_triple, + target_settings = target_settings, + iso_date = toolchain.channel.iso_date, + rustfmt_version = rustfmt_version, + sha256s = sha256s, + target_triple = toolchain.target_triple, + urls = urls, + version = toolchain.channel.version, + ) - all_toolchain_names.append(rust_toolchain_repository( - name = toolchain_name, - allocator_library = allocator_library, - auth = auth, - channel = channel, - dev_components = dev_components, - edition = edition, - exec_triple = exec_triple, - target_settings = target_settings, - iso_date = info.iso_date, - rustfmt_version = rustfmt_version, - sha256s = sha256s, - target_triple = target_triple, - urls = urls, - version = info.version, - )) + all_toolchain_names.append("@{name}//:toolchain".format(name = toolchain.name)) # This repository exists to allow `rust_repository_set` to work with the `maybe` wrapper. rust_toolchain_set_repository( From e9caaaf90247e9f1e6955de27bf62e046d9a7bce Mon Sep 17 00:00:00 2001 From: Cameron Martin Date: Sat, 18 Feb 2023 14:53:08 +0000 Subject: [PATCH 02/12] Move raw_cargo into private package --- MODULE.bazel | 2 +- extensions.bzl | 31 ------------------------------- rust/private/extensions.bzl | 31 +++++++++++++++++++++++++++++++ 3 files changed, 32 insertions(+), 32 deletions(-) create mode 100644 rust/private/extensions.bzl diff --git a/MODULE.bazel b/MODULE.bazel index c67cd5dd72..0d80ee0dc0 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -9,7 +9,7 @@ bazel_dep(name = "bazel_skylib", version = "1.2.0") bazel_dep(name = "apple_support", version = "0.13.0") # Load tinyjson without using the process wrapper -raw_cargo = use_extension("//:extensions.bzl", "raw_cargo") +raw_cargo = use_extension("//rust/private:extensions.bzl", "raw_cargo") raw_cargo.crate( name = "tinyjson", diff --git a/extensions.bzl b/extensions.bzl index 7c5e04a8e1..9116a4f43a 100644 --- a/extensions.bzl +++ b/extensions.bzl @@ -1,7 +1,6 @@ "Module extensions for using rules_rust with bzlmod" load("//rust:repositories.bzl", "rust_register_toolchains") -load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") def _toolchains_impl(ctx): mod = ctx.modules[0] @@ -13,33 +12,3 @@ toolchains = module_extension( implementation = _toolchains_impl, tag_classes = {"toolchain": toolchains_toolchain}, ) - -def _create_build_file_content(name): - return """ -load("@rules_rust//rust/private:rust.bzl", "rust_library_without_process_wrapper") - -rust_library_without_process_wrapper( - name = "{}", - srcs = glob(["src/*.rs"]), - edition = "2018", - visibility = ["@rules_rust//util/process_wrapper:__pkg__"], -) -""".format(name) - -def _raw_cargo_impl(ctx): - for mod in ctx.modules: - for crate in mod.tags.crate: - http_archive( - name = "raw_cargo_{}_{}".format(crate.name, crate.version.replace(".", "_")), - sha256 = crate.sha256, - url = "https://crates.io/api/v1/crates/{}/{}/download".format(crate.name, crate.version), - strip_prefix = "{}-{}".format(crate.name, crate.version), - type = "tar.gz", - build_file_content = _create_build_file_content(crate.name), - ) - -raw_cargo_crate = tag_class(attrs = {"name": attr.string(), "version": attr.string(), "sha256": attr.string()}) -raw_cargo = module_extension( - implementation = _raw_cargo_impl, - tag_classes = {"crate": raw_cargo_crate}, -) diff --git a/rust/private/extensions.bzl b/rust/private/extensions.bzl new file mode 100644 index 0000000000..7d46b846f5 --- /dev/null +++ b/rust/private/extensions.bzl @@ -0,0 +1,31 @@ +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") + +def _create_build_file_content(name): + return """ +load("@rules_rust//rust/private:rust.bzl", "rust_library_without_process_wrapper") + +rust_library_without_process_wrapper( + name = "{}", + srcs = glob(["src/*.rs"]), + edition = "2018", + visibility = ["@rules_rust//util/process_wrapper:__pkg__"], +) +""".format(name) + +def _raw_cargo_impl(ctx): + for mod in ctx.modules: + for crate in mod.tags.crate: + http_archive( + name = "raw_cargo_{}_{}".format(crate.name, crate.version.replace(".", "_")), + sha256 = crate.sha256, + url = "https://crates.io/api/v1/crates/{}/{}/download".format(crate.name, crate.version), + strip_prefix = "{}-{}".format(crate.name, crate.version), + type = "tar.gz", + build_file_content = _create_build_file_content(crate.name), + ) + +raw_cargo_crate = tag_class(attrs = {"name": attr.string(), "version": attr.string(), "sha256": attr.string()}) +raw_cargo = module_extension( + implementation = _raw_cargo_impl, + tag_classes = {"crate": raw_cargo_crate}, +) From 3525fce7cdeb0a47f8f858efd4fd2ebc7b3d55ea Mon Sep 17 00:00:00 2001 From: Cameron Martin Date: Sat, 18 Feb 2023 15:33:04 +0000 Subject: [PATCH 03/12] Support all toolchain parameters in module extension --- extensions.bzl | 31 +++++++++++++++++++++++++++++-- rust/private/repository_utils.bzl | 3 +++ rust/repositories.bzl | 10 +++++----- 3 files changed, 37 insertions(+), 7 deletions(-) diff --git a/extensions.bzl b/extensions.bzl index 9116a4f43a..fda329f1df 100644 --- a/extensions.bzl +++ b/extensions.bzl @@ -1,13 +1,40 @@ "Module extensions for using rules_rust with bzlmod" load("//rust:repositories.bzl", "rust_register_toolchains") +load( + "//rust/private:repository_utils.bzl", + "DEFAULT_EXTRA_TARGET_TRIPLES", + "DEFAULT_NIGHTLY_VERSION", + "DEFAULT_STATIC_RUST_URL_TEMPLATES", +) def _toolchains_impl(ctx): mod = ctx.modules[0] for toolchain in mod.tags.toolchain: - rust_register_toolchains(edition = toolchain.edition, register_toolchains = False) + rust_register_toolchains( + dev_components = toolchain.dev_components, + edition = toolchain.edition, + allocator_library = toolchain.allocator_library, + rustfmt_version = toolchain.rustfmt_version, + rust_analyzer_version = toolchain.rust_analyzer_version, + sha256s = toolchain.sha256s, + extra_target_triples = toolchain.extra_target_triples, + urls = toolchain.urls, + versions = toolchain.versions, + register_toolchains = False, + ) -toolchains_toolchain = tag_class(attrs = {"edition": attr.string()}) +toolchains_toolchain = tag_class(attrs = { + "dev_components": attr.bool(default = False), + "edition": attr.string(), + "allocator_library": attr.string(), + "rustfmt_version": attr.string(default = DEFAULT_NIGHTLY_VERSION), + "rust_analyzer_version": attr.string(), + "sha256s": attr.string_dict(), + "extra_target_triples": attr.string_list(default = DEFAULT_EXTRA_TARGET_TRIPLES), + "urls": attr.string_list(default = DEFAULT_STATIC_RUST_URL_TEMPLATES), + "versions": attr.string_list(default = []), +}) toolchains = module_extension( implementation = _toolchains_impl, tag_classes = {"toolchain": toolchains_toolchain}, diff --git a/rust/private/repository_utils.bzl b/rust/private/repository_utils.bzl index 6a66e2ba5c..d160b72c20 100644 --- a/rust/private/repository_utils.bzl +++ b/rust/private/repository_utils.bzl @@ -1,5 +1,6 @@ """Utility macros for use in rules_rust repository rules""" +load("//rust/private:common.bzl", "DEFAULT_NIGHTLY_ISO_DATE") load("//rust:known_shas.bzl", "FILE_KEY_TO_SHA") load( "//rust/platform:triple_mappings.bzl", @@ -11,6 +12,8 @@ load( DEFAULT_TOOLCHAIN_NAME_PREFIX = "toolchain_for" DEFAULT_STATIC_RUST_URL_TEMPLATES = ["https://static.rust-lang.org/dist/{}.tar.gz"] +DEFAULT_NIGHTLY_VERSION = "nightly/{}".format(DEFAULT_NIGHTLY_ISO_DATE) +DEFAULT_EXTRA_TARGET_TRIPLES = ["wasm32-unknown-unknown", "wasm32-wasi"] _build_file_for_compiler_template = """\ filegroup( diff --git a/rust/repositories.bzl b/rust/repositories.bzl index 6a94fec8f2..6abb7c6334 100644 --- a/rust/repositories.bzl +++ b/rust/repositories.bzl @@ -12,6 +12,8 @@ load( "BUILD_for_rust_toolchain", "BUILD_for_rustfmt_toolchain", "BUILD_for_toolchain", + "DEFAULT_EXTRA_TARGET_TRIPLES", + "DEFAULT_NIGHTLY_VERSION", "DEFAULT_STATIC_RUST_URL_TEMPLATES", "check_version_valid", "includes_rust_analyzer_proc_macro_srv", @@ -91,11 +93,9 @@ def rules_rust_dependencies(): build_file = "@rules_rust//util/process_wrapper:BUILD.tinyjson.bazel", ) -_DEFAULT_NIGHTLY_VERSION = "nightly/{}".format(DEFAULT_NIGHTLY_ISO_DATE) - _RUST_TOOLCHAIN_VERSIONS = [ rust_common.default_version, - _DEFAULT_NIGHTLY_VERSION, + DEFAULT_NIGHTLY_VERSION, ] # buildifier: disable=unnamed-macro @@ -105,10 +105,10 @@ def rust_register_toolchains( allocator_library = None, iso_date = None, register_toolchains = True, - rustfmt_version = _DEFAULT_NIGHTLY_VERSION, + rustfmt_version = DEFAULT_NIGHTLY_VERSION, rust_analyzer_version = None, sha256s = None, - extra_target_triples = ["wasm32-unknown-unknown", "wasm32-wasi"], + extra_target_triples = DEFAULT_EXTRA_TARGET_TRIPLES, urls = DEFAULT_STATIC_RUST_URL_TEMPLATES, version = None, versions = []): From ccc446ba3bd5a649df431af256bb9ea3da61fadc Mon Sep 17 00:00:00 2001 From: Cameron Martin Date: Sat, 18 Feb 2023 15:39:52 +0000 Subject: [PATCH 04/12] Fix lint warnings --- rust/private/extensions.bzl | 2 ++ rust/repositories.bzl | 6 +++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/rust/private/extensions.bzl b/rust/private/extensions.bzl index 7d46b846f5..4d42ffc699 100644 --- a/rust/private/extensions.bzl +++ b/rust/private/extensions.bzl @@ -1,3 +1,5 @@ +"""Bzlmod module extensions that are only used internally""" + load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") def _create_build_file_content(name): diff --git a/rust/repositories.bzl b/rust/repositories.bzl index 6abb7c6334..c8efe84cb7 100644 --- a/rust/repositories.bzl +++ b/rust/repositories.bzl @@ -249,7 +249,7 @@ def rust_register_toolchains( rustfmt_repo_name, )) - for toolchain in get_toolchain_repositories(name, exec_triple, extra_target_triples, versions, iso_date): + for toolchain in _get_toolchain_repositories(name, exec_triple, extra_target_triples, versions, iso_date): toolchain_names.append(toolchain.name) toolchain_labels[toolchain.name] = "@{}//:{}".format(toolchain.name + "_tools", "rust_toolchain") exec_compatible_with_by_toolchain[toolchain.name] = triple_to_constraint_set(exec_triple) @@ -806,7 +806,7 @@ rust_toolchain_set_repository = repository_rule( implementation = _rust_toolchain_set_repository_impl, ) -def get_toolchain_repositories(name, exec_triple, extra_target_triples, versions, iso_date): +def _get_toolchain_repositories(name, exec_triple, extra_target_triples, versions, iso_date): toolchain_repos = [] for target_triple in [exec_triple] + extra_target_triples: @@ -904,7 +904,7 @@ def rust_repository_set( versions = [version] all_toolchain_names = [] - for toolchain in get_toolchain_repositories(name, exec_triple, extra_target_triples, versions, iso_date): + for toolchain in _get_toolchain_repositories(name, exec_triple, extra_target_triples, versions, iso_date): rust_toolchain_repository( name = toolchain.name, allocator_library = allocator_library, From 43b2e97925d5700fab767531aba34e01c9e01771 Mon Sep 17 00:00:00 2001 From: Cameron Martin Date: Sat, 18 Feb 2023 16:02:50 +0000 Subject: [PATCH 05/12] Undo change where rust_toolchain_repository no longer returns the toolchain name. --- rust/repositories.bzl | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/rust/repositories.bzl b/rust/repositories.bzl index c8efe84cb7..8e0e9d6d74 100644 --- a/rust/repositories.bzl +++ b/rust/repositories.bzl @@ -505,6 +505,9 @@ def rust_toolchain_repository( urls (list, optional): A list of mirror urls containing the tools from the Rust-lang static file server. These must contain the '{}' used to substitute the tool being fetched (using .format). Defaults to ['https://static.rust-lang.org/dist/{}.tar.gz'] auth (dict): Auth object compatible with repository_ctx.download to use when downloading files. See [repository_ctx.download](https://docs.bazel.build/versions/main/skylark/lib/repository_ctx.html#download) for more details. + + Returns: + str: The name of the registerable toolchain created by this rule. """ if rustfmt_version in ("nightly", "beta"): @@ -546,6 +549,10 @@ def rust_toolchain_repository( target_compatible_with = target_compatible_with, ) + return "@{name}//:toolchain".format( + name = name, + ) + def _rust_analyzer_toolchain_tools_repository_impl(repository_ctx): load_rust_src( ctx = repository_ctx, @@ -905,7 +912,7 @@ def rust_repository_set( all_toolchain_names = [] for toolchain in _get_toolchain_repositories(name, exec_triple, extra_target_triples, versions, iso_date): - rust_toolchain_repository( + all_toolchain_names.append(rust_toolchain_repository( name = toolchain.name, allocator_library = allocator_library, auth = auth, @@ -920,9 +927,7 @@ def rust_repository_set( target_triple = toolchain.target_triple, urls = urls, version = toolchain.channel.version, - ) - - all_toolchain_names.append("@{name}//:toolchain".format(name = toolchain.name)) + )) # This repository exists to allow `rust_repository_set` to work with the `maybe` wrapper. rust_toolchain_set_repository( From aa775fc6d9dd0e5f664698282e1e01eefc8ee27f Mon Sep 17 00:00:00 2001 From: Cameron Martin Date: Sat, 18 Feb 2023 16:24:00 +0000 Subject: [PATCH 06/12] Fix buildifier errors --- MODULE.bazel | 2 -- examples/bzlmod/hello_world/MODULE.bazel | 5 +---- extensions.bzl => rust/extensions.bzl | 8 ++++---- rust/private/extensions.bzl | 2 +- rust/private/repository_utils.bzl | 4 ++-- rust/repositories.bzl | 2 +- 6 files changed, 9 insertions(+), 14 deletions(-) rename extensions.bzl => rust/extensions.bzl (96%) diff --git a/MODULE.bazel b/MODULE.bazel index 0d80ee0dc0..18a501b250 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -10,13 +10,11 @@ bazel_dep(name = "apple_support", version = "0.13.0") # Load tinyjson without using the process wrapper raw_cargo = use_extension("//rust/private:extensions.bzl", "raw_cargo") - raw_cargo.crate( name = "tinyjson", sha256 = "1a8304da9f9370f6a6f9020b7903b044aa9ce3470f300a1fba5bc77c78145a16", version = "2.3.0", ) - use_repo( raw_cargo, rules_rust_tinyjson = "raw_cargo_tinyjson_2_3_0", diff --git a/examples/bzlmod/hello_world/MODULE.bazel b/examples/bzlmod/hello_world/MODULE.bazel index e34a5592e0..d78b33d9ab 100644 --- a/examples/bzlmod/hello_world/MODULE.bazel +++ b/examples/bzlmod/hello_world/MODULE.bazel @@ -4,16 +4,13 @@ module( ) bazel_dep(name = "rules_rust", version = "0.9.0") - local_path_override( module_name = "rules_rust", path = "../../..", ) -rust_toolchains = use_extension("@rules_rust//:extensions.bzl", "toolchains") - +rust_toolchains = use_extension("@rules_rust//rust:extensions.bzl", "toolchains") rust_toolchains.toolchain(edition = "2021") - use_repo( rust_toolchains, "rust_toolchains", diff --git a/extensions.bzl b/rust/extensions.bzl similarity index 96% rename from extensions.bzl rename to rust/extensions.bzl index fda329f1df..df2155b913 100644 --- a/extensions.bzl +++ b/rust/extensions.bzl @@ -1,12 +1,12 @@ "Module extensions for using rules_rust with bzlmod" -load("//rust:repositories.bzl", "rust_register_toolchains") load( "//rust/private:repository_utils.bzl", "DEFAULT_EXTRA_TARGET_TRIPLES", "DEFAULT_NIGHTLY_VERSION", "DEFAULT_STATIC_RUST_URL_TEMPLATES", ) +load(":repositories.bzl", "rust_register_toolchains") def _toolchains_impl(ctx): mod = ctx.modules[0] @@ -25,13 +25,13 @@ def _toolchains_impl(ctx): ) toolchains_toolchain = tag_class(attrs = { + "allocator_library": attr.string(), "dev_components": attr.bool(default = False), "edition": attr.string(), - "allocator_library": attr.string(), - "rustfmt_version": attr.string(default = DEFAULT_NIGHTLY_VERSION), + "extra_target_triples": attr.string_list(default = DEFAULT_EXTRA_TARGET_TRIPLES), "rust_analyzer_version": attr.string(), + "rustfmt_version": attr.string(default = DEFAULT_NIGHTLY_VERSION), "sha256s": attr.string_dict(), - "extra_target_triples": attr.string_list(default = DEFAULT_EXTRA_TARGET_TRIPLES), "urls": attr.string_list(default = DEFAULT_STATIC_RUST_URL_TEMPLATES), "versions": attr.string_list(default = []), }) diff --git a/rust/private/extensions.bzl b/rust/private/extensions.bzl index 4d42ffc699..b5f7a158a7 100644 --- a/rust/private/extensions.bzl +++ b/rust/private/extensions.bzl @@ -26,7 +26,7 @@ def _raw_cargo_impl(ctx): build_file_content = _create_build_file_content(crate.name), ) -raw_cargo_crate = tag_class(attrs = {"name": attr.string(), "version": attr.string(), "sha256": attr.string()}) +raw_cargo_crate = tag_class(attrs = {"name": attr.string(), "sha256": attr.string(), "version": attr.string()}) raw_cargo = module_extension( implementation = _raw_cargo_impl, tag_classes = {"crate": raw_cargo_crate}, diff --git a/rust/private/repository_utils.bzl b/rust/private/repository_utils.bzl index d160b72c20..f3ebfee1ef 100644 --- a/rust/private/repository_utils.bzl +++ b/rust/private/repository_utils.bzl @@ -1,6 +1,5 @@ """Utility macros for use in rules_rust repository rules""" -load("//rust/private:common.bzl", "DEFAULT_NIGHTLY_ISO_DATE") load("//rust:known_shas.bzl", "FILE_KEY_TO_SHA") load( "//rust/platform:triple_mappings.bzl", @@ -9,6 +8,7 @@ load( "system_to_staticlib_ext", "system_to_stdlib_linkflags", ) +load("//rust/private:common.bzl", "DEFAULT_NIGHTLY_ISO_DATE") DEFAULT_TOOLCHAIN_NAME_PREFIX = "toolchain_for" DEFAULT_STATIC_RUST_URL_TEMPLATES = ["https://static.rust-lang.org/dist/{}.tar.gz"] @@ -820,7 +820,6 @@ toolchain_repository_hub = repository_rule( "repositories. This exists to allow registering a set of toolchains in one go with the `:all` target." ), attrs = { - "toolchain_names": attr.string_list(mandatory = True), "exec_compatible_with": attr.string_list_dict( doc = "A list of constraints for the execution platform for this toolchain, keyed by toolchain name.", mandatory = True, @@ -833,6 +832,7 @@ toolchain_repository_hub = repository_rule( doc = "The name of the toolchain implementation target, keyed by toolchain name.", mandatory = True, ), + "toolchain_names": attr.string_list(mandatory = True), "toolchain_types": attr.string_dict( doc = "The toolchain type of the toolchain to declare, keyed by toolchain name.", mandatory = True, diff --git a/rust/repositories.bzl b/rust/repositories.bzl index 8e0e9d6d74..bbffad7809 100644 --- a/rust/repositories.bzl +++ b/rust/repositories.bzl @@ -505,7 +505,7 @@ def rust_toolchain_repository( urls (list, optional): A list of mirror urls containing the tools from the Rust-lang static file server. These must contain the '{}' used to substitute the tool being fetched (using .format). Defaults to ['https://static.rust-lang.org/dist/{}.tar.gz'] auth (dict): Auth object compatible with repository_ctx.download to use when downloading files. See [repository_ctx.download](https://docs.bazel.build/versions/main/skylark/lib/repository_ctx.html#download) for more details. - + Returns: str: The name of the registerable toolchain created by this rule. """ From ae0cbc671e2075771b747286b74a736b1059cad1 Mon Sep 17 00:00:00 2001 From: Cameron Martin Date: Sat, 18 Feb 2023 16:44:13 +0000 Subject: [PATCH 07/12] Add bzlmod examples to CI --- .bazelci/presubmit.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.bazelci/presubmit.yml b/.bazelci/presubmit.yml index 85adb119e1..6892852f4b 100644 --- a/.bazelci/presubmit.yml +++ b/.bazelci/presubmit.yml @@ -342,6 +342,12 @@ tasks: - "//..." build_flags: *aspects_flags soft_fail: yes + ubuntu2004_examples_bzlmod: + name: Bzlmod Examples + platform: ubuntu2004 + working_directory: examples/bzlmod/hello_world + build_targets: + - "//..." rbe_ubuntu1604_examples: name: Examples platform: rbe_ubuntu1604 From 4db1894ba6be78ecbed3ba99bf926e173f2b99ae Mon Sep 17 00:00:00 2001 From: Matt Stark Date: Mon, 20 Feb 2023 14:17:40 +1100 Subject: [PATCH 08/12] Allow multiple toolchains in bzlmod. When multiple modules each define different toolchain configurations, at the moment, an arbitrary one wins and defines all the toolchains. This change allows each different module to define as many different toolchains as they want, each with a different set of configuration. If the configuration is the same between modules, they will share their toolchains. --- MODULE.bazel | 19 ++-- bzlmod/BUILD.bazel | 0 bzlmod/private/BUILD.bazel | 0 bzlmod/private/non_bzlmod_deps/BUILD.bazel | 0 .../non_bzlmod_deps/non_bzlmod_deps.bzl | 14 +++ bzlmod/private/toolchains/BUILD.bazel | 0 bzlmod/private/toolchains/toolchains.bzl | 87 +++++++++++++++++++ examples/bzlmod/hello_world/MODULE.bazel | 3 +- rust/extensions.bzl | 40 +-------- rust/private/extensions.bzl | 33 ------- rust/repositories.bzl | 46 ++++++---- 11 files changed, 145 insertions(+), 97 deletions(-) create mode 100644 bzlmod/BUILD.bazel create mode 100644 bzlmod/private/BUILD.bazel create mode 100644 bzlmod/private/non_bzlmod_deps/BUILD.bazel create mode 100644 bzlmod/private/non_bzlmod_deps/non_bzlmod_deps.bzl create mode 100644 bzlmod/private/toolchains/BUILD.bazel create mode 100644 bzlmod/private/toolchains/toolchains.bzl delete mode 100644 rust/private/extensions.bzl diff --git a/MODULE.bazel b/MODULE.bazel index 18a501b250..55e8013559 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -6,16 +6,15 @@ module( bazel_dep(name = "platforms", version = "0.0.5") bazel_dep(name = "rules_cc", version = "0.0.1") bazel_dep(name = "bazel_skylib", version = "1.2.0") -bazel_dep(name = "apple_support", version = "0.13.0") - -# Load tinyjson without using the process wrapper -raw_cargo = use_extension("//rust/private:extensions.bzl", "raw_cargo") -raw_cargo.crate( - name = "tinyjson", - sha256 = "1a8304da9f9370f6a6f9020b7903b044aa9ce3470f300a1fba5bc77c78145a16", - version = "2.3.0", +bazel_dep( + name = "apple_support", + repo_name = "build_bazel_apple_support", + version = "1.4.0" ) + +non_bzlmod_deps = use_extension("//bzlmod/private/non_bzlmod_deps:non_bzlmod_deps.bzl", "non_bzlmod_deps") +non_bzlmod_deps.non_bzlmod_deps() use_repo( - raw_cargo, - rules_rust_tinyjson = "raw_cargo_tinyjson_2_3_0", + non_bzlmod_deps, + "rules_rust_tinyjson" ) diff --git a/bzlmod/BUILD.bazel b/bzlmod/BUILD.bazel new file mode 100644 index 0000000000..e69de29bb2 diff --git a/bzlmod/private/BUILD.bazel b/bzlmod/private/BUILD.bazel new file mode 100644 index 0000000000..e69de29bb2 diff --git a/bzlmod/private/non_bzlmod_deps/BUILD.bazel b/bzlmod/private/non_bzlmod_deps/BUILD.bazel new file mode 100644 index 0000000000..e69de29bb2 diff --git a/bzlmod/private/non_bzlmod_deps/non_bzlmod_deps.bzl b/bzlmod/private/non_bzlmod_deps/non_bzlmod_deps.bzl new file mode 100644 index 0000000000..634943d7d0 --- /dev/null +++ b/bzlmod/private/non_bzlmod_deps/non_bzlmod_deps.bzl @@ -0,0 +1,14 @@ +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") +load("//rust:repositories.bzl", "TINYJSON_KWARGS") + +def _non_bzlmod_deps_impl(module_ctx): + http_archive(**TINYJSON_KWARGS) + +non_bzlmod_deps_non_bzlmod_deps = tag_class(attrs = {}) +non_bzlmod_deps = module_extension( + doc = "Dependencies for rules_rust", + implementation = _non_bzlmod_deps_impl, + tag_classes = dict( + non_bzlmod_deps = non_bzlmod_deps_non_bzlmod_deps, + ), +) diff --git a/bzlmod/private/toolchains/BUILD.bazel b/bzlmod/private/toolchains/BUILD.bazel new file mode 100644 index 0000000000..e69de29bb2 diff --git a/bzlmod/private/toolchains/toolchains.bzl b/bzlmod/private/toolchains/toolchains.bzl new file mode 100644 index 0000000000..ab62be1abc --- /dev/null +++ b/bzlmod/private/toolchains/toolchains.bzl @@ -0,0 +1,87 @@ +load("//crate_universe:deps_bootstrap.bzl", _cargo_bazel_bootstrap_repo_rule = "cargo_bazel_bootstrap") +load("//rust/private:common.bzl", "rust_common") +load( + "//rust/private:repository_utils.bzl", + "DEFAULT_EXTRA_TARGET_TRIPLES", + "DEFAULT_NIGHTLY_VERSION", + "DEFAULT_STATIC_RUST_URL_TEMPLATES", +) +load("//rust:repositories.bzl", "rust_register_toolchains", "rust_toolchain_tools_repository") + +def generate_uid(toolchain): + """ + Generates a UID representing a toolchain. This allows us to share toolchains + between different modules if they happen to use the same configuration. + """ + unique_config = [ + toolchain.allocator_library, + str(toolchain.dev_components), + toolchain.edition, + toolchain.rust_analyzer_version, + toolchain.rustfmt_version, + ] + unique_config.append("sha256s") + for k, v in toolchain.sha256s: + unique_config.append(k) + unique_config.append(v) + unique_config.append("extra_target_triples") + unique_config.extend(toolchain.extra_target_triples) + unique_config.append("urls") + unique_config.extend(toolchain.urls) + unique_config.append("versions") + unique_config.extend(toolchain.versions) + return hash("".join([str(hash(i)) for i in unique_config])) + +def _toolchains_impl(module_ctx): + toolchains = {} + for mod in module_ctx.modules: + for toolchain in mod.tags.toolchain: + repo_name = "%s_rust_toolchains" % mod.name + if toolchain.suffix: + repo_name += "_" + toolchain.suffix + + namespace = "internal_%d" % generate_uid(toolchain) + if namespace not in toolchains: + toolchains[namespace] = (toolchain, []) + toolchains[namespace][1].append("%s_rust_toolchains" % mod.name) + + for namespace, (toolchain, repos) in toolchains.items(): + rust_register_toolchains( + hub_repos = repos, + repo_namespace = namespace, + register_toolchains = False, + edition = toolchain.edition, + allocator_library = toolchain.allocator_library, + rustfmt_version = toolchain.rustfmt_version, + sha256s = toolchain.sha256s, + extra_target_triples = toolchain.extra_target_triples, + urls = toolchain.urls, + version = toolchain.versions, + ) + +toolchains_toolchain = tag_class( + doc = "Generates a repo '_rust_toolchain", + attrs = dict( + suffix = attr.string( + doc = "If provided, instead of generating _rust_toolchain, " + + "generates _rust_toolchain_.\n" + + "This is required to generate multiple toolchains in the same module.", + ), + allocator_library = attr.string(), + dev_components = attr.bool(default = False), + edition = attr.string(), + extra_target_triples = attr.string_list(default = DEFAULT_EXTRA_TARGET_TRIPLES), + rust_analyzer_version = attr.string(), + rustfmt_version = attr.string(default = DEFAULT_NIGHTLY_VERSION), + sha256s = attr.string_dict(), + urls = attr.string_list(default = DEFAULT_STATIC_RUST_URL_TEMPLATES), + versions = attr.string_list(default = []), + ), +) + +toolchains = module_extension( + implementation = _toolchains_impl, + tag_classes = dict( + toolchain = toolchains_toolchain, + ), +) diff --git a/examples/bzlmod/hello_world/MODULE.bazel b/examples/bzlmod/hello_world/MODULE.bazel index d78b33d9ab..937ef32f54 100644 --- a/examples/bzlmod/hello_world/MODULE.bazel +++ b/examples/bzlmod/hello_world/MODULE.bazel @@ -13,7 +13,8 @@ rust_toolchains = use_extension("@rules_rust//rust:extensions.bzl", "toolchains" rust_toolchains.toolchain(edition = "2021") use_repo( rust_toolchains, - "rust_toolchains", + # Rename it from _rust_toolchains. + rust_toolchains = "hello_world_rust_toolchains", ) register_toolchains("@rust_toolchains//:all") diff --git a/rust/extensions.bzl b/rust/extensions.bzl index df2155b913..675589e9ac 100644 --- a/rust/extensions.bzl +++ b/rust/extensions.bzl @@ -1,41 +1,5 @@ "Module extensions for using rules_rust with bzlmod" -load( - "//rust/private:repository_utils.bzl", - "DEFAULT_EXTRA_TARGET_TRIPLES", - "DEFAULT_NIGHTLY_VERSION", - "DEFAULT_STATIC_RUST_URL_TEMPLATES", -) -load(":repositories.bzl", "rust_register_toolchains") +load("//bzlmod/private/toolchains:toolchains.bzl", _toolchains = "toolchains") -def _toolchains_impl(ctx): - mod = ctx.modules[0] - for toolchain in mod.tags.toolchain: - rust_register_toolchains( - dev_components = toolchain.dev_components, - edition = toolchain.edition, - allocator_library = toolchain.allocator_library, - rustfmt_version = toolchain.rustfmt_version, - rust_analyzer_version = toolchain.rust_analyzer_version, - sha256s = toolchain.sha256s, - extra_target_triples = toolchain.extra_target_triples, - urls = toolchain.urls, - versions = toolchain.versions, - register_toolchains = False, - ) - -toolchains_toolchain = tag_class(attrs = { - "allocator_library": attr.string(), - "dev_components": attr.bool(default = False), - "edition": attr.string(), - "extra_target_triples": attr.string_list(default = DEFAULT_EXTRA_TARGET_TRIPLES), - "rust_analyzer_version": attr.string(), - "rustfmt_version": attr.string(default = DEFAULT_NIGHTLY_VERSION), - "sha256s": attr.string_dict(), - "urls": attr.string_list(default = DEFAULT_STATIC_RUST_URL_TEMPLATES), - "versions": attr.string_list(default = []), -}) -toolchains = module_extension( - implementation = _toolchains_impl, - tag_classes = {"toolchain": toolchains_toolchain}, -) +toolchains = _toolchains \ No newline at end of file diff --git a/rust/private/extensions.bzl b/rust/private/extensions.bzl deleted file mode 100644 index b5f7a158a7..0000000000 --- a/rust/private/extensions.bzl +++ /dev/null @@ -1,33 +0,0 @@ -"""Bzlmod module extensions that are only used internally""" - -load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") - -def _create_build_file_content(name): - return """ -load("@rules_rust//rust/private:rust.bzl", "rust_library_without_process_wrapper") - -rust_library_without_process_wrapper( - name = "{}", - srcs = glob(["src/*.rs"]), - edition = "2018", - visibility = ["@rules_rust//util/process_wrapper:__pkg__"], -) -""".format(name) - -def _raw_cargo_impl(ctx): - for mod in ctx.modules: - for crate in mod.tags.crate: - http_archive( - name = "raw_cargo_{}_{}".format(crate.name, crate.version.replace(".", "_")), - sha256 = crate.sha256, - url = "https://crates.io/api/v1/crates/{}/{}/download".format(crate.name, crate.version), - strip_prefix = "{}-{}".format(crate.name, crate.version), - type = "tar.gz", - build_file_content = _create_build_file_content(crate.name), - ) - -raw_cargo_crate = tag_class(attrs = {"name": attr.string(), "sha256": attr.string(), "version": attr.string()}) -raw_cargo = module_extension( - implementation = _raw_cargo_impl, - tag_classes = {"crate": raw_cargo_crate}, -) diff --git a/rust/repositories.bzl b/rust/repositories.bzl index bbffad7809..836abd41a7 100644 --- a/rust/repositories.bzl +++ b/rust/repositories.bzl @@ -44,6 +44,15 @@ DEFAULT_TOOLCHAIN_TRIPLES = { "x86_64-unknown-linux-gnu": "rust_linux_x86_64", } +TINYJSON_KWARGS = dict( + name = "rules_rust_tinyjson", + sha256 = "1a8304da9f9370f6a6f9020b7903b044aa9ce3470f300a1fba5bc77c78145a16", + url = "https://crates.io/api/v1/crates/tinyjson/2.3.0/download", + strip_prefix = "tinyjson-2.3.0", + type = "tar.gz", + build_file = "@rules_rust//util/process_wrapper:BUILD.tinyjson.bazel", +) + def rules_rust_dependencies(): """Dependencies used in the implementation of `rules_rust`.""" @@ -85,12 +94,7 @@ def rules_rust_dependencies(): # process_wrapper needs a low-dependency way to process json. maybe( http_archive, - name = "rules_rust_tinyjson", - sha256 = "1a8304da9f9370f6a6f9020b7903b044aa9ce3470f300a1fba5bc77c78145a16", - url = "https://crates.io/api/v1/crates/tinyjson/2.3.0/download", - strip_prefix = "tinyjson-2.3.0", - type = "tar.gz", - build_file = "@rules_rust//util/process_wrapper:BUILD.tinyjson.bazel", + **TINYJSON_KWARGS ) _RUST_TOOLCHAIN_VERSIONS = [ @@ -111,7 +115,9 @@ def rust_register_toolchains( extra_target_triples = DEFAULT_EXTRA_TARGET_TRIPLES, urls = DEFAULT_STATIC_RUST_URL_TEMPLATES, version = None, - versions = []): + versions = [], + hub_repos = ["rust_toolchains"], + repo_namespace = None): """Emits a default set of toolchains for Linux, MacOS, and Freebsd Skip this macro and call the `rust_repository_set` macros directly if you need a compiler for \ @@ -144,6 +150,8 @@ def rust_register_toolchains( version (str, optional): **Deprecated**: Use `versions` instead. versions (list, optional): A list of toolchain versions to download. This paramter only accepts one versions per channel. E.g. `["1.65.0", "nightly/2022-11-02", "beta/2020-12-30"]`. + hub_repos (list, optional): A list of hub repos to generate. For use with bzlmod. + repo_namespace (str, optional): If provided, all generated repos will be prefixed with repo_namespace. """ if version: # buildifier: disable=print @@ -177,6 +185,8 @@ def rust_register_toolchains( rust_analyzer_version = select_rust_version(versions) rust_analyzer_repo_name = "rust_analyzer_{}".format(rust_analyzer_version.replace("/", "-")) + if repo_namespace: + rust_analyzer_repo_name = "%s__%s" % (repo_namespace, rust_analyzer_repo_name) rust_analyzer_iso_date = None if rust_analyzer_version.startswith(("beta", "nightly")): rust_analyzer_version, _, rust_analyzer_iso_date = rustfmt_version.partition("/") @@ -215,6 +225,8 @@ def rust_register_toolchains( rustfmt_version_or_channel, _, rustfmt_iso_date = rustfmt_version.partition("/") for exec_triple, name in DEFAULT_TOOLCHAIN_TRIPLES.items(): + if repo_namespace: + name = "%s__%s" % (repo_namespace, name) maybe( rust_repository_set, name = name, @@ -256,14 +268,18 @@ def rust_register_toolchains( target_compatible_with_by_toolchain[toolchain.name] = triple_to_constraint_set(toolchain.target_triple) toolchain_types[toolchain.name] = "@rules_rust//rust:toolchain" - toolchain_repository_hub( - name = "rust_toolchains", - toolchain_names = toolchain_names, - toolchain_labels = toolchain_labels, - toolchain_types = toolchain_types, - exec_compatible_with = exec_compatible_with_by_toolchain, - target_compatible_with = target_compatible_with_by_toolchain, - ) + # See github.com/bazelbuild/bazel/issues/17493 + # Not all hub repos get the same "view" into spoke repos, but ones that + # share the same config do. + for repo_name in hub_repos: + toolchain_repository_hub( + name = repo_name, + toolchain_names = toolchain_names, + toolchain_labels = toolchain_labels, + toolchain_types = toolchain_types, + exec_compatible_with = exec_compatible_with_by_toolchain, + target_compatible_with = target_compatible_with_by_toolchain, + ) # buildifier: disable=unnamed-macro def rust_repositories(**kwargs): From 30dc06df750b89ea5feb78421fcaf026de0e68e7 Mon Sep 17 00:00:00 2001 From: Matt Stark Date: Mon, 20 Feb 2023 14:40:27 +1100 Subject: [PATCH 09/12] Create a host_tools repo. This is required for bootstrapping cargo_bazel. --- MODULE.bazel | 9 +++++ bzlmod/private/toolchains/toolchains.bzl | 49 +++++++++++++++++++----- 2 files changed, 48 insertions(+), 10 deletions(-) diff --git a/MODULE.bazel b/MODULE.bazel index 55e8013559..8b356a7173 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -18,3 +18,12 @@ use_repo( non_bzlmod_deps, "rules_rust_tinyjson" ) + +# Set up a default toolchain, so we can use it to bootstrap cargo bazel. +toolchains = use_extension("//rust:extensions.bzl", "toolchains") +toolchains.host_tools() +use_repo( + toolchains, + rust_host_tools = "rules_rust_rust_host_tools", +) + diff --git a/bzlmod/private/toolchains/toolchains.bzl b/bzlmod/private/toolchains/toolchains.bzl index ab62be1abc..d215b74b9d 100644 --- a/bzlmod/private/toolchains/toolchains.bzl +++ b/bzlmod/private/toolchains/toolchains.bzl @@ -1,4 +1,5 @@ load("//crate_universe:deps_bootstrap.bzl", _cargo_bazel_bootstrap_repo_rule = "cargo_bazel_bootstrap") +load("//rust/platform:triple.bzl", "get_host_triple") load("//rust/private:common.bzl", "rust_common") load( "//rust/private:repository_utils.bzl", @@ -33,8 +34,25 @@ def generate_uid(toolchain): return hash("".join([str(hash(i)) for i in unique_config])) def _toolchains_impl(module_ctx): + host_triple = get_host_triple(module_ctx) + toolchains = {} for mod in module_ctx.modules: + for toolchain in mod.tags.host_tools: + rust_toolchain_tools_repository( + name = "%s_rust_host_tools" % mod.name, + exec_triple = host_triple.str, + target_triple = host_triple.str, + allocator_library = toolchain.allocator_library, + version = toolchain.version, + rustfmt_version = toolchain.rustfmt_version, + edition = toolchain.edition, + sha256s = toolchain.sha256s, + urls = toolchain.urls, + dev_components = False, + auth = None, + ) + for toolchain in mod.tags.toolchain: repo_name = "%s_rust_toolchains" % mod.name if toolchain.suffix: @@ -59,29 +77,40 @@ def _toolchains_impl(module_ctx): version = toolchain.versions, ) +_COMMON_TOOLCHAIN_ATTRS = dict( + allocator_library = attr.string(), + dev_components = attr.bool(default = False), + edition = attr.string(), + extra_target_triples = attr.string_list(default = DEFAULT_EXTRA_TARGET_TRIPLES), + rust_analyzer_version = attr.string(), + rustfmt_version = attr.string(default = DEFAULT_NIGHTLY_VERSION), + sha256s = attr.string_dict(), + urls = attr.string_list(default = DEFAULT_STATIC_RUST_URL_TEMPLATES), +) + toolchains_toolchain = tag_class( doc = "Generates a repo '_rust_toolchain", attrs = dict( + versions = attr.string_list(default = []), suffix = attr.string( doc = "If provided, instead of generating _rust_toolchain, " + "generates _rust_toolchain_.\n" + "This is required to generate multiple toolchains in the same module.", ), - allocator_library = attr.string(), - dev_components = attr.bool(default = False), - edition = attr.string(), - extra_target_triples = attr.string_list(default = DEFAULT_EXTRA_TARGET_TRIPLES), - rust_analyzer_version = attr.string(), - rustfmt_version = attr.string(default = DEFAULT_NIGHTLY_VERSION), - sha256s = attr.string_dict(), - urls = attr.string_list(default = DEFAULT_STATIC_RUST_URL_TEMPLATES), - versions = attr.string_list(default = []), + **_COMMON_TOOLCHAIN_ATTRS + ), +) +host_tools = tag_class( + doc = "Generates a set of host tools suitable for bootstrapping cargo.", + attrs = dict( + version = attr.string(default = rust_common.default_version), + **_COMMON_TOOLCHAIN_ATTRS ), ) - toolchains = module_extension( implementation = _toolchains_impl, tag_classes = dict( toolchain = toolchains_toolchain, + host_tools = host_tools, ), ) From cca44f83f1cb34e6b1c46fc67a8906ffd89b9f73 Mon Sep 17 00:00:00 2001 From: Matt Stark Date: Mon, 20 Feb 2023 14:43:07 +1100 Subject: [PATCH 10/12] Bootstrap cargo_bazel with bzlmod. --- MODULE.bazel | 6 +++ .../private/cargo_bazel_bootstrap/BUILD.bazel | 0 .../cargo_bazel_bootstrap.bzl | 45 +++++++++++++++++++ rust/extensions.bzl | 4 +- 4 files changed, 54 insertions(+), 1 deletion(-) create mode 100644 bzlmod/private/cargo_bazel_bootstrap/BUILD.bazel create mode 100644 bzlmod/private/cargo_bazel_bootstrap/cargo_bazel_bootstrap.bzl diff --git a/MODULE.bazel b/MODULE.bazel index 8b356a7173..7c691ac214 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -27,3 +27,9 @@ use_repo( rust_host_tools = "rules_rust_rust_host_tools", ) +cargo_bazel_bootstrap = use_extension("//rust:extensions.bzl", "cargo_bazel_bootstrap") +cargo_bazel_bootstrap.cargo_bazel_bootstrap() +use_repo( + cargo_bazel_bootstrap, + "cargo_bazel_bootstrap" +) diff --git a/bzlmod/private/cargo_bazel_bootstrap/BUILD.bazel b/bzlmod/private/cargo_bazel_bootstrap/BUILD.bazel new file mode 100644 index 0000000000..e69de29bb2 diff --git a/bzlmod/private/cargo_bazel_bootstrap/cargo_bazel_bootstrap.bzl b/bzlmod/private/cargo_bazel_bootstrap/cargo_bazel_bootstrap.bzl new file mode 100644 index 0000000000..2696b8f13a --- /dev/null +++ b/bzlmod/private/cargo_bazel_bootstrap/cargo_bazel_bootstrap.bzl @@ -0,0 +1,45 @@ +load("//cargo/private:cargo_utils.bzl", "get_rust_tools") +load("//crate_universe:deps_bootstrap.bzl", _cargo_bazel_bootstrap_repo_rule = "cargo_bazel_bootstrap") +load("//rust:defs.bzl", "rust_common") + +def _cargo_bazel_bootstrap_impl(module_ctx): + _cargo_bazel_bootstrap_repo_rule( + rust_toolchain_cargo_template = "@rust_host_tools//:bin/{tool}", + rust_toolchain_rustc_template = "@rust_host_tools//:bin/{tool}", + ) + +cargo_bazel_bootstrap_cargo_bazel_bootstrap = tag_class(attrs = {}) +cargo_bazel_bootstrap = module_extension( + implementation = _cargo_bazel_bootstrap_impl, + tag_classes = dict( + cargo_bazel_bootstrap = cargo_bazel_bootstrap_cargo_bazel_bootstrap, + ), +) + +def get_cargo_bazel_runner(module_ctx): + cargo_path = str(module_ctx.path(Label("@rust_host_tools//:bin/cargo"))) + rustc_path = str(module_ctx.path(Label("@rust_host_tools//:bin/rustc"))) + cargo_bazel = module_ctx.path(Label("@cargo_bazel_bootstrap//:cargo-bazel")) + + def run(args, env = {}, timeout = 600): + final_args = [cargo_bazel] + final_args.extend(args) + final_args.extend([ + "--cargo", + cargo_path, + "--rustc", + rustc_path, + ]) + result = module_ctx.execute( + final_args, + environment = dict(CARGO = cargo_path, RUSTC = rustc_path, **env), + timeout = timeout, + ) + if result.return_code != 0: + if result.stdout: + print("Stdout:", result.stdout) + pretty_args = " ".join([str(arg) for arg in final_args]) + fail("%s returned with exit code %d:\n%s" % (pretty_args, result.return_code, result.stderr)) + return result + + return run diff --git a/rust/extensions.bzl b/rust/extensions.bzl index 675589e9ac..a2674af153 100644 --- a/rust/extensions.bzl +++ b/rust/extensions.bzl @@ -1,5 +1,7 @@ "Module extensions for using rules_rust with bzlmod" +load("//bzlmod/private/cargo_bazel_bootstrap:cargo_bazel_bootstrap.bzl", _cargo_bazel_bootstrap = "cargo_bazel_bootstrap") load("//bzlmod/private/toolchains:toolchains.bzl", _toolchains = "toolchains") -toolchains = _toolchains \ No newline at end of file +cargo_bazel_bootstrap = _cargo_bazel_bootstrap +toolchains = _toolchains From a2f9d20a794434d5d6b76b504a9f6a2320eb9d73 Mon Sep 17 00:00:00 2001 From: Matt Stark Date: Mon, 20 Feb 2023 16:39:49 +1100 Subject: [PATCH 11/12] Create crate module extension. --- bzlmod/private/crate/BUILD.bazel | 0 bzlmod/private/crate/crate.bzl | 176 +++++++++++ bzlmod/private/crate/tag_classes/BUILD.bazel | 0 .../private/crate/tag_classes/annotation.bzl | 116 +++++++ .../private/crate/tag_classes/from_cargo.bzl | 17 + bzlmod/private/crate/tag_classes/spec.bzl | 28 ++ bzlmod/private/symlink_repo.bzl | 10 + crate_universe/deps_bootstrap.bzl | 3 +- crate_universe/private/crate.bzl | 2 + crate_universe/private/crates_vendor.bzl | 291 ++++++++++-------- examples/bzlmod/crate_universe/.bazelrc | 1 + examples/bzlmod/crate_universe/.gitignore | 1 + examples/bzlmod/crate_universe/BUILD.bazel | 14 + examples/bzlmod/crate_universe/Cargo.lock | 143 +++++++++ examples/bzlmod/crate_universe/Cargo.toml | 8 + examples/bzlmod/crate_universe/MODULE.bazel | 37 +++ examples/bzlmod/crate_universe/WORKSPACE | 0 examples/bzlmod/crate_universe/src/main.rs | 22 ++ rust/extensions.bzl | 2 + 19 files changed, 742 insertions(+), 129 deletions(-) create mode 100644 bzlmod/private/crate/BUILD.bazel create mode 100644 bzlmod/private/crate/crate.bzl create mode 100644 bzlmod/private/crate/tag_classes/BUILD.bazel create mode 100644 bzlmod/private/crate/tag_classes/annotation.bzl create mode 100644 bzlmod/private/crate/tag_classes/from_cargo.bzl create mode 100644 bzlmod/private/crate/tag_classes/spec.bzl create mode 100644 bzlmod/private/symlink_repo.bzl create mode 100644 examples/bzlmod/crate_universe/.bazelrc create mode 100644 examples/bzlmod/crate_universe/.gitignore create mode 100644 examples/bzlmod/crate_universe/BUILD.bazel create mode 100644 examples/bzlmod/crate_universe/Cargo.lock create mode 100644 examples/bzlmod/crate_universe/Cargo.toml create mode 100644 examples/bzlmod/crate_universe/MODULE.bazel create mode 100644 examples/bzlmod/crate_universe/WORKSPACE create mode 100644 examples/bzlmod/crate_universe/src/main.rs diff --git a/bzlmod/private/crate/BUILD.bazel b/bzlmod/private/crate/BUILD.bazel new file mode 100644 index 0000000000..e69de29bb2 diff --git a/bzlmod/private/crate/crate.bzl b/bzlmod/private/crate/crate.bzl new file mode 100644 index 0000000000..e0442334cc --- /dev/null +++ b/bzlmod/private/crate/crate.bzl @@ -0,0 +1,176 @@ +load("//cargo/private:cargo_utils.bzl", _rust_get_rust_tools = "get_rust_tools") +load("//crate_universe/private:generate_utils.bzl", "render_config") +load("//crate_universe/private:common_utils.bzl", "get_rust_tools") +load("//crate_universe/private:crates_repository.bzl", "crates_repository") +load("//rust:repositories.bzl", "rust_register_toolchains") +load("//rust/platform:triple.bzl", "get_host_triple") +load("//crate_universe/private:crates_vendor.bzl", "CRATES_VENDOR_ATTRS", "write_config_file", "write_splicing_manifest", _crates_vendor_repo_rule = "crates_vendor") +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") +load("@bazel_tools//tools/build_defs/repo:git.bzl", "new_git_repository") +load("//bzlmod/private:symlink_repo.bzl", "symlink_repo") +load("//bzlmod/private/cargo_bazel_bootstrap:cargo_bazel_bootstrap.bzl", "get_cargo_bazel_runner") +load("//bzlmod/private/crate/tag_classes:annotation.bzl", "annotation_tags_to_json", annotation_tag = "annotation") +load("//bzlmod/private/crate/tag_classes:from_cargo.bzl", from_cargo_tag = "from_cargo") +load("//bzlmod/private/crate/tag_classes:spec.bzl", "spec_tags_to_json", spec_tag = "spec") + +def _crate_impl(module_ctx): + cargo_bazel = get_cargo_bazel_runner(module_ctx) + + for mod in module_ctx.modules: + mod_path = module_ctx.path(mod.name) + + annotations = annotation_tags_to_json(mod.tags.annotation) + packages = spec_tags_to_json(mod.tags.spec) + + # At the moment, this is rather inefficient in the case of many tags + # depending on the same crate, as we have to build each package for each + # cargo lockfile that depends on it (we generate the same package once + # per cargo lockfile). + # This is a non-issue if using cargo workspaces, which I'd generally + # recommend anyway, but in the long term we should probably try and + # share repos if they use the same configuration. + for cfg in mod.tags.from_cargo: + tag_path = mod_path + repo_name = mod.name + "_crates" + if cfg.suffix: + tag_path = mod_path.get_child(cfg.suffix) + repo_name += "_" + cfg.suffix + + cargo_lockfile = module_ctx.path(cfg.cargo_lockfile) + + def write_data_file(ctx, name, data): + path = tag_path.get_child(name) + module_ctx.file(path, content = data, executable = False) + return path + + rendering_config = json.decode(render_config()) + rendering_config["regen_command"] = "Run 'cargo update [--workspace]'" + config_file = write_config_file( + module_ctx, + mode = "remote", + annotations = annotations, + generate_build_scripts = cfg.generate_build_scripts, + supported_platform_triples = cfg.supported_platform_triples, + repository_name = repo_name, + output_pkg = repo_name, + workspace_name = repo_name, + write_data_file = write_data_file, + generate_binaries = cfg.generate_binaries, + rendering_config = rendering_config, + ) + + manifests = {module_ctx.path(m): m for m in cfg.manifests} + splicing_manifest = write_splicing_manifest( + module_ctx, + packages = packages, + splicing_config = "", + cargo_config = cfg.cargo_config, + manifests = {str(p): str(l) for p, l in manifests.items()}, + write_data_file = write_data_file, + manifest_to_path = module_ctx.path, + ) + + splicing_output_dir = tag_path.get_child("splicing-output") + cargo_bazel([ + "splice", + "--output-dir", + splicing_output_dir, + "--config", + config_file, + "--splicing-manifest", + splicing_manifest, + "--cargo-lockfile", + cargo_lockfile, + ]) + + # Create a lockfile, since we need to parse it to generate spoke + # repos. + lockfile_path = tag_path.get_child("lockfile.json") + module_ctx.file(lockfile_path, "") + + # TODO: Thanks to the namespacing of bzlmod, although we generate + # defs.bzl, it isn't useful (it generates deps like + # "_crates__env_logger-0.9.3//:env_logger", which + # don't work because that repository isn't visible from main). + # To solve this, we need to generate the alias + # @crates//:env_logger-0.9.3 as well as the @crates//:env_logger + # that it already generates, and point defs.bzl there instead. + cargo_bazel([ + "generate", + "--cargo-lockfile", + cargo_lockfile, + "--config", + config_file, + "--splicing-manifest", + splicing_manifest, + "--repository-dir", + tag_path, + "--metadata", + splicing_output_dir.get_child("metadata.json"), + "--repin", + "--lockfile", + lockfile_path, + ]) + + crates_dir = tag_path.get_child(repo_name) + symlink_repo( + name = repo_name, + files = { + "BUILD.bazel": str(crates_dir.get_child("BUILD.bazel")), + }, + ) + + contents = json.decode(module_ctx.read(lockfile_path)) + + for crate in contents["crates"].values(): + repo = crate["repository"] + if repo == None: + continue + name = crate["name"] + version = crate["version"] + # "+" isn't valid in a repo name. + crate_repo_name = "%s__%s-%s" % (repo_name, name, version.replace("+", "-")) + + patch_tool = repo.get("patch_tool", None) + patches = repo.get("patches", None) + patch_args = repo.get("patch_args", None) + + build_file_content = module_ctx.read(crates_dir.get_child("BUILD.%s-%s.bazel" % (name, version))) + if "Http" in repo: + # Replicates repo_http.j2 + http_archive( + name = crate_repo_name, + patch_args = patch_args, + patch_tool = patch_tool, + patches = patches, + sha256 = repo["Http"]["sha256"], + type = "tar.gz", + urls = [repo["Http"]["url"]], + strip_prefix = "%s-%s" % (crate["name"], crate["version"]), + build_file_content = build_file_content, + ) + elif "Git" in repo: + # Replicates repo_git.j2 + new_git_repository( + name = crate_repo_name, + init_submodules = True, + patch_args = patch_args, + patch_tool = patch_tool, + patches = patches, + shallow_since = repo.get("shallow_since", None), + remote = repo["Git"]["remote"], + build_file_content = build_file_content, + strip_prefix = repo.get("strip_prefix", None), + **repo["commitish"], + ) + else: + fail("Invalid repo: expected Http or Git to exist for crate %s-%s, got %s" % (name, version, repo)) + +crate = module_extension( + implementation = _crate_impl, + tag_classes = dict( + annotation = annotation_tag, + from_cargo = from_cargo_tag, + spec = spec_tag, + ), +) diff --git a/bzlmod/private/crate/tag_classes/BUILD.bazel b/bzlmod/private/crate/tag_classes/BUILD.bazel new file mode 100644 index 0000000000..e69de29bb2 diff --git a/bzlmod/private/crate/tag_classes/annotation.bzl b/bzlmod/private/crate/tag_classes/annotation.bzl new file mode 100644 index 0000000000..fa27f25147 --- /dev/null +++ b/bzlmod/private/crate/tag_classes/annotation.bzl @@ -0,0 +1,116 @@ +load("//crate_universe:defs.bzl", "crate") + +_ANNOTATION_ATTRS = dict( + crate = attr.string( + mandatory = True, + doc = "The crate to apply the annotation to. The wildcard `*` matches any version, including prerelease versions.", + ), + version = attr.string( + default = "*", + doc = "The version or semver-conditions to match with a crate.", + ), + additive_build_file_content = attr.string( + doc = "Extra contents to write to the bottom of generated BUILD files.", + ), + additive_build_file = attr.label( + allow_single_file = True, + doc = "A file containing extra contents to write to the bottom of generated BUILD files", + ), + build_script_data = attr.label_list( + doc = "A list of labels to add to a crate's `cargo_build_script::data` attribute.", + ), + build_script_tools = attr.label_list( + doc = "A list of labels to add to a crate's `cargo_build_script::tools` attribute.", + ), + build_script_data_glob = attr.string_list( + doc = "A list of glob patterns to add to a crate's `cargo_build_script::data` attribute", + ), + build_script_deps = attr.label_list( + doc = "A list of labels to add to a crate's `cargo_build_script::deps` attribute.", + ), + build_script_env = attr.string_dict( + doc = "Additional environment variables to set on a crate's `cargo_build_script::env` attribute.", + ), + build_script_proc_macro_deps = attr.label_list( + doc = "A list of labels to add to a crate's `cargo_build_script::proc_macro_deps` attribute.", + ), + build_script_rustc_env = attr.string_dict( + doc = "Additional environment variables to set on a crate's `cargo_build_script::env` attribute.", + ), + build_script_toolchains = attr.label_list( + doc = "A list of labels to set on a crates's `cargo_build_script::toolchains` attribute.", + ), + compile_data = attr.label_list( + doc = "A list of labels to add to a crate's `rust_library::compile_data` attribute.", + ), + compile_data_glob = attr.string_list( + doc = "A list of glob patterns to add to a crate's `rust_library::compile_data` attribute", + ), + crate_features = attr.string_list( + doc = "A list of strings to add to a crate's `rust_library::crate_features` attribute.", + ), + data = attr.label_list( + doc = "A list of labels to add to a crate's `rust_library::data` attribute.", + ), + data_glob = attr.string_list( + doc = "A list of glob patterns to add to a crate's `rust_library::data` attribute.", + ), + deps = attr.label_list( + doc = "A list of labels to add to a crate's `rust_library::deps` attribute.", + ), + disable_pipelining = attr.bool( + doc = "If True, disables pipelining for library targets for this crate.", + ), + gen_all_binaries = attr.bool( + doc = "If true, generates all binaries for a crate." + ), + gen_binaries = attr.string_list( + doc = "Thu subset of the crate's bins that should get `rust_binary` targets produced." + ), + gen_build_script = attr.bool( + doc = "An authorative flag to determine whether or not to produce `cargo_build_script` targets for the current crate.", + ), + patch_args = attr.string_list( + doc = "The `patch_args` attribute of a Bazel repository rule. See [http_archive.patch_args](https://docs.bazel.build/versions/main/repo/http.html#http_archive-patch_args)", + ), + patch_tool = attr.string( + doc = "The `patch_tool` attribute of a Bazel repository rule. See [http_archive.patch_tool](https://docs.bazel.build/versions/main/repo/http.html#http_archive-patch_tool)", + ), + patches = attr.label_list( + doc = "The `patches` attribute of a Bazel repository rule. See [http_archive.patches](https://docs.bazel.build/versions/main/repo/http.html#http_archive-patches)", + ), + proc_macro_deps = attr.label_list( + doc = "A list of labels to add to a crate's `rust_library::proc_macro_deps` attribute.", + ), + rustc_env = attr.string_dict( + doc = "Additional variables to set on a crate's `rust_library::rustc_env` attribute.", + ), + rustc_env_files = attr.label_list( + doc = "A list of labels to set on a crate's `rust_library::rustc_env_files` attribute.", + ), + rustc_flags = attr.string_list( + doc = "A list of strings to set on a crate's `rust_library::rustc_flags` attribute.", + ), + shallow_since = attr.string( + doc = "An optional timestamp used for crates originating from a git repository instead of a crate registry. This flag optimizes fetching the source code.", + ), + +) + +annotation = tag_class( + doc = "A collection of extra attributes and settings for a particular crate", + attrs = _ANNOTATION_ATTRS, +) + +def annotation_tags_to_json(tags): + annotations = {} + for tag in tags: + if tag.crate not in annotations: + annotations[tag.crate] = [] + + kwargs = {k: getattr(tag, k) for k in _ANNOTATION_ATTRS} + kwargs.pop("crate") + if kwargs.pop("gen_all_binaries"): + kwargs["gen_binaries"] = True + annotations[tag.crate].append(crate.annotation(**kwargs)) + return annotations diff --git a/bzlmod/private/crate/tag_classes/from_cargo.bzl b/bzlmod/private/crate/tag_classes/from_cargo.bzl new file mode 100644 index 0000000000..4b1867cdf1 --- /dev/null +++ b/bzlmod/private/crate/tag_classes/from_cargo.bzl @@ -0,0 +1,17 @@ +load("//crate_universe/private:crates_vendor.bzl", "CRATES_VENDOR_ATTRS", "write_config_file", "write_splicing_manifest", _crates_vendor_repo_rule = "crates_vendor") + +from_cargo = tag_class( + doc = "Generates a repo _crates", + attrs = dict( + suffix = attr.string( + doc = "If provided, instead generates a repo _crates_. " + + "This can help avoid conflicts if you declare multiple from_cargo in a single module." + ), + cargo_lockfile = CRATES_VENDOR_ATTRS["cargo_lockfile"], + manifests = CRATES_VENDOR_ATTRS["manifests"], + cargo_config = CRATES_VENDOR_ATTRS["cargo_config"], + generate_binaries = CRATES_VENDOR_ATTRS["generate_binaries"], + generate_build_scripts = CRATES_VENDOR_ATTRS["generate_build_scripts"], + supported_platform_triples = CRATES_VENDOR_ATTRS["supported_platform_triples"], + ) +) diff --git a/bzlmod/private/crate/tag_classes/spec.bzl b/bzlmod/private/crate/tag_classes/spec.bzl new file mode 100644 index 0000000000..b208e06ccb --- /dev/null +++ b/bzlmod/private/crate/tag_classes/spec.bzl @@ -0,0 +1,28 @@ +load("//crate_universe:defs.bzl", "crate") + +_SPEC_ATTRS = dict( + name = attr.string(mandatory = True, doc = "The name of the crate"), + package = attr.string(doc = "The explicit name of the package (used when attempting to alias a crate)."), + version = attr.string(doc = "The exact version of the crate. Cannot be used with `git`."), + default_features = attr.bool(doc = "Maps to the `default-features` flag."), + features = attr.string_list(doc = "A list of features to use for the crate"), + git = attr.string(doc = "The Git url to use for the crate. Cannot be used with `version`."), + rev = attr.string(doc = "The git revision of the remote crate. Tied with the `git` param."), +) + +spec = tag_class( + doc = """A constructor for a crate dependency. + + See [specifying dependencies][sd] in the Cargo book for more details. + + [sd]: https://doc.rust-lang.org/cargo/reference/specifying-dependencies.html +""", + attrs = _SPEC_ATTRS, +) + +def _spec_tag_to_json(tag): + kwargs = {k: getattr(tag, k) for k in _SPEC_ATTRS if k != "name"} + return crate.spec(**kwargs) + +def spec_tags_to_json(tags): + return {tag.name: _spec_tag_to_json(tag) for tag in tags} diff --git a/bzlmod/private/symlink_repo.bzl b/bzlmod/private/symlink_repo.bzl new file mode 100644 index 0000000000..93614d2bf9 --- /dev/null +++ b/bzlmod/private/symlink_repo.bzl @@ -0,0 +1,10 @@ +def _symlink_repo_impl(repo_ctx): + for path, target in repo_ctx.attr.files.items(): + repo_ctx.symlink(target, path) + +symlink_repo = repository_rule( + implementation = _symlink_repo_impl, + attrs = dict( + files = attr.string_dict(mandatory = True), + ), +) diff --git a/crate_universe/deps_bootstrap.bzl b/crate_universe/deps_bootstrap.bzl index 28f6d14b0d..1c52c8f6ce 100644 --- a/crate_universe/deps_bootstrap.bzl +++ b/crate_universe/deps_bootstrap.bzl @@ -6,7 +6,7 @@ load("//crate_universe/private:srcs.bzl", "CARGO_BAZEL_SRCS") # buildifier: disable=bzl-visibility load("//rust/private:common.bzl", "rust_common") -def cargo_bazel_bootstrap(name = "cargo_bazel_bootstrap", rust_version = rust_common.default_version): +def cargo_bazel_bootstrap(name = "cargo_bazel_bootstrap", rust_version = rust_common.default_version, **kwargs): """An optional repository which bootstraps `cargo-bazel` for use with `crates_repository` Args: @@ -22,4 +22,5 @@ def cargo_bazel_bootstrap(name = "cargo_bazel_bootstrap", rust_version = rust_co version = rust_version, # The increased timeout helps avoid flakes in CI timeout = 900, + **kwargs, ) diff --git a/crate_universe/private/crate.bzl b/crate_universe/private/crate.bzl index ef651836eb..6f939f6882 100644 --- a/crate_universe/private/crate.bzl +++ b/crate_universe/private/crate.bzl @@ -15,6 +15,7 @@ def _workspace_member(version, sha256 = None): sha256 = sha256, )) +# If you change this, also change //bzlmod/private/crate/tag_classes/spec.bzl def _spec( package = None, version = None, @@ -66,6 +67,7 @@ def _assert_absolute(label): label_str, )) +# If you change this, also change //bzlmod/private/crate/tag_classes/annotations.bzl def _annotation( version = "*", additive_build_file = None, diff --git a/crate_universe/private/crates_vendor.bzl b/crate_universe/private/crates_vendor.bzl index 3cd6ed533b..282b7eb691 100644 --- a/crate_universe/private/crates_vendor.bzl +++ b/crate_universe/private/crates_vendor.bzl @@ -1,7 +1,7 @@ """Rules for vendoring Bazel targets into existing workspaces""" load("//crate_universe/private:generate_utils.bzl", "compile_config", "render_config") -load("//crate_universe/private:splicing_utils.bzl", "kebab_case_keys", "splicing_config") +load("//crate_universe/private:splicing_utils.bzl", "kebab_case_keys", generate_splicing_config="splicing_config") load("//crate_universe/private:urls.bzl", "CARGO_BAZEL_LABEL") load("//rust/platform:triple_mappings.bzl", "SUPPORTED_PLATFORM_TRIPLES") @@ -93,24 +93,41 @@ def _prepare_manifest_path(target): return "${build_workspace_directory}/" + manifest.short_path def _write_splicing_manifest(ctx): + # Manifests are required to be single files + manifests = {_prepare_manifest_path(m): str(m.label) for m in ctx.attr.manifests} + + manifest = write_splicing_manifest( + ctx, + write_data_file = _write_data_file, + packages=ctx.attr.packages, + splicing_config=ctx.attr.splicing_config, + cargo_config=ctx.attr.cargo_config, + manifests = manifests, + manifest_to_path = _prepare_manifest_path, + ) + + is_windows = _is_windows(ctx) + + args = ["--splicing-manifest", _runfiles_path(manifest.short_path, is_windows)] + runfiles = [manifest] + ctx.files.manifests + ([ctx.file.cargo_config] if ctx.attr.cargo_config else []) + return args, runfiles + +def write_splicing_manifest(ctx, write_data_file, packages, splicing_config, cargo_config, manifests, manifest_to_path): # Deserialize information about direct packges direct_packages_info = { # Ensure the data is using kebab-case as that's what `cargo_toml::DependencyDetail` expects. pkg: kebab_case_keys(dict(json.decode(data))) - for (pkg, data) in ctx.attr.packages.items() + for (pkg, data) in packages.items() } - # Manifests are required to be single files - manifests = {_prepare_manifest_path(m): str(m.label) for m in ctx.attr.manifests} - - config = json.decode(ctx.attr.splicing_config or splicing_config()) + config = json.decode(splicing_config or generate_splicing_config()) splicing_manifest_content = { - "cargo_config": _prepare_manifest_path(ctx.attr.cargo_config) if ctx.attr.cargo_config else None, + "cargo_config": manifest_to_path(cargo_config) if cargo_config else None, "direct_packages": direct_packages_info, "manifests": manifests, } - manifest = _write_data_file( + return write_data_file( ctx = ctx, name = "cargo-bazel-splicing-manifest.json", data = json.encode_indent( @@ -119,27 +136,47 @@ def _write_splicing_manifest(ctx): ), ) - is_windows = _is_windows(ctx) - - args = ["--splicing-manifest", _runfiles_path(manifest, is_windows)] - runfiles = [manifest] + ctx.files.manifests + ([ctx.file.cargo_config] if ctx.attr.cargo_config else []) - return args, runfiles - def _write_config_file(ctx): - default_render_config = dict(json.decode(render_config())) + workspace_name = ctx.workspace_name + if workspace_name == "__main__": + workspace_name = "" - if ctx.attr.render_config: - rendering_config = dict(json.decode(ctx.attr.render_config)) - else: - rendering_config = default_render_config + config = write_config_file( + ctx, + regen_command = "bazel run {}".format(ctx.label), + mode = ctx.attr.mode, + annotations = ctx.attr.annotations, + generate_binaries = ctx.attr.generate_binaries, + generate_build_scripts = ctx.attr.generate_build_scripts, + supported_platform_triples = ctx.attr.supported_platform_triples, + repository_name = ctx.attr.repository_name, + output_pkg = _get_output_package(ctx), + workspace_name = workspace_name, + write_data_file = _write_data_file, + rendering_config = dict(json.decode(ctx.attr.render_config)) if ctx.attr.render_config else None, + ) - output_pkg = _get_output_package(ctx) + is_windows = _is_windows(ctx) + args = ["--config", _runfiles_path(config, is_windows)] + runfiles = [config] + ctx.files.manifests + return args, runfiles - workspace_name = ctx.workspace_name - if ctx.workspace_name == "__main__": - workspace_name = "" - if ctx.attr.mode == "local": +def write_config_file( + ctx, + mode, + annotations, + generate_binaries, + generate_build_scripts, + supported_platform_triples, + repository_name, + output_pkg, + workspace_name, + write_data_file, + rendering_config): + default_render_config = json.decode(render_config()) + + if mode == "local": build_file_base_template = "@{}//{}/{{name}}-{{version}}:BUILD.bazel" crate_label_template = "//{}/{{name}}-{{version}}:{{target}}".format( output_pkg, @@ -158,7 +195,7 @@ def _write_config_file(ctx): workspace_name, output_pkg, ), - "vendor_mode": ctx.attr.mode, + "vendor_mode": mode, } for key in updates: @@ -175,16 +212,16 @@ def _write_config_file(ctx): rendering_config.update({"regen_command": "bazel run {}".format(ctx.label)}) config_data = compile_config( - crate_annotations = ctx.attr.annotations, - generate_binaries = ctx.attr.generate_binaries, - generate_build_scripts = ctx.attr.generate_build_scripts, + crate_annotations = annotations, + generate_binaries = generate_binaries, + generate_build_scripts = generate_build_scripts, cargo_config = None, render_config = rendering_config, - supported_platform_triples = ctx.attr.supported_platform_triples, - repository_name = ctx.attr.repository_name or ctx.label.name, + supported_platform_triples = supported_platform_triples, + repository_name = repository_name or ctx.label.name, ) - config = _write_data_file( + return write_data_file( ctx = ctx, name = "cargo-bazel-config.json", data = json.encode_indent( @@ -193,11 +230,6 @@ def _write_config_file(ctx): ), ) - is_windows = _is_windows(ctx) - args = ["--config", _runfiles_path(config, is_windows)] - runfiles = [config] + ctx.files.manifests - return args, runfiles - def _crates_vendor_impl(ctx): toolchain = ctx.toolchains[Label("@rules_rust//rust:toolchain_type")] is_windows = _is_windows(ctx) @@ -282,6 +314,99 @@ def _crates_vendor_impl(ctx): executable = runner, ) +CRATES_VENDOR_ATTRS = { + "annotations": attr.string_list_dict( + doc = "Extra settings to apply to crates. See [crate.annotation](#crateannotation).", + ), + "bazel": attr.label( + doc = "The path to a bazel binary used to locate the output_base for the current workspace.", + cfg = "exec", + executable = True, + allow_files = True, + ), + "buildifier": attr.label( + doc = "The path to a [buildifier](https://github.com/bazelbuild/buildtools/blob/5.0.1/buildifier/README.md) binary used to format generated BUILD files.", + cfg = "exec", + executable = True, + allow_files = True, + default = Label("//crate_universe/private/vendor:buildifier"), + ), + "cargo_bazel": attr.label( + doc = ( + "The cargo-bazel binary to use for vendoring. If this attribute is not set, then a " + + "`{}` action env will be used.".format(CARGO_BAZEL_GENERATOR_PATH) + ), + cfg = "exec", + executable = True, + allow_files = True, + default = CARGO_BAZEL_LABEL, + ), + "cargo_config": attr.label( + doc = "A [Cargo configuration](https://doc.rust-lang.org/cargo/reference/config.html) file.", + allow_single_file = True, + ), + "cargo_lockfile": attr.label( + doc = "The path to an existing `Cargo.lock` file", + allow_single_file = True, + ), + "generate_binaries": attr.bool( + doc = ( + "Whether to generate `rust_binary` targets for all the binary crates in every package. " + + "By default only the `rust_library` targets are generated." + ), + default = False, + ), + "generate_build_scripts": attr.bool( + doc = ( + "Whether or not to generate " + + "[cargo build scripts](https://doc.rust-lang.org/cargo/reference/build-scripts.html) by default." + ), + default = True, + ), + "manifests": attr.label_list( + doc = "A list of Cargo manifests (`Cargo.toml` files).", + allow_files = ["Cargo.toml"], + ), + "mode": attr.string( + doc = ( + "Flags determining how crates should be vendored. `local` is where crate source and BUILD files are " + + "written to the repository. `remote` is where only BUILD files are written and repository rules " + + "used to fetch source code." + ), + values = [ + "local", + "remote", + ], + default = "remote", + ), + "packages": attr.string_dict( + doc = "A set of crates (packages) specifications to depend on. See [crate.spec](#crate.spec).", + ), + "render_config": attr.string( + doc = ( + "The configuration flags to use for rendering. Use `//crate_universe:defs.bzl\\%render_config` to " + + "generate the value for this field. If unset, the defaults defined there will be used." + ), + ), + "repository_name": attr.string( + doc = "The name of the repository to generate for `remote` vendor modes. If unset, the label name will be used", + ), + "splicing_config": attr.string( + doc = ( + "The configuration flags to use for splicing Cargo maniests. Use `//crate_universe:defs.bzl\\%rsplicing_config` to " + + "generate the value for this field. If unset, the defaults defined there will be used." + ), + ), + "supported_platform_triples": attr.string_list( + doc = "A set of all platform triples to consider when generating dependencies.", + default = SUPPORTED_PLATFORM_TRIPLES, + ), + "vendor_path": attr.string( + doc = "The path to a directory to write files into. Absolute paths will be treated as relative to the workspace root", + default = "crates", + ), +} + crates_vendor = rule( implementation = _crates_vendor_impl, doc = """\ @@ -356,102 +481,12 @@ call against the generated workspace. The following table describes how to contr | `package_name@1.2.3` | `cargo upgrade --package package_name --precise 1.2.3` | """, - attrs = { - "annotations": attr.string_list_dict( - doc = "Extra settings to apply to crates. See [crate.annotation](#crateannotation).", - ), - "bazel": attr.label( - doc = "The path to a bazel binary used to locate the output_base for the current workspace.", - cfg = "exec", - executable = True, - allow_files = True, - ), - "buildifier": attr.label( - doc = "The path to a [buildifier](https://github.com/bazelbuild/buildtools/blob/5.0.1/buildifier/README.md) binary used to format generated BUILD files.", - cfg = "exec", - executable = True, - allow_files = True, - default = Label("//crate_universe/private/vendor:buildifier"), - ), - "cargo_bazel": attr.label( - doc = ( - "The cargo-bazel binary to use for vendoring. If this attribute is not set, then a " + - "`{}` action env will be used.".format(CARGO_BAZEL_GENERATOR_PATH) - ), - cfg = "exec", - executable = True, - allow_files = True, - default = CARGO_BAZEL_LABEL, - ), - "cargo_config": attr.label( - doc = "A [Cargo configuration](https://doc.rust-lang.org/cargo/reference/config.html) file.", - allow_single_file = True, - ), - "cargo_lockfile": attr.label( - doc = "The path to an existing `Cargo.lock` file", - allow_single_file = True, - ), - "generate_binaries": attr.bool( - doc = ( - "Whether to generate `rust_binary` targets for all the binary crates in every package. " + - "By default only the `rust_library` targets are generated." - ), - default = False, - ), - "generate_build_scripts": attr.bool( - doc = ( - "Whether or not to generate " + - "[cargo build scripts](https://doc.rust-lang.org/cargo/reference/build-scripts.html) by default." - ), - default = True, - ), - "manifests": attr.label_list( - doc = "A list of Cargo manifests (`Cargo.toml` files).", - allow_files = ["Cargo.toml"], - ), - "mode": attr.string( - doc = ( - "Flags determining how crates should be vendored. `local` is where crate source and BUILD files are " + - "written to the repository. `remote` is where only BUILD files are written and repository rules " + - "used to fetch source code." - ), - values = [ - "local", - "remote", - ], - default = "remote", - ), - "packages": attr.string_dict( - doc = "A set of crates (packages) specifications to depend on. See [crate.spec](#crate.spec).", - ), - "render_config": attr.string( - doc = ( - "The configuration flags to use for rendering. Use `//crate_universe:defs.bzl\\%render_config` to " + - "generate the value for this field. If unset, the defaults defined there will be used." - ), - ), - "repository_name": attr.string( - doc = "The name of the repository to generate for `remote` vendor modes. If unset, the label name will be used", - ), - "splicing_config": attr.string( - doc = ( - "The configuration flags to use for splicing Cargo maniests. Use `//crate_universe:defs.bzl\\%rsplicing_config` to " + - "generate the value for this field. If unset, the defaults defined there will be used." - ), - ), - "supported_platform_triples": attr.string_list( - doc = "A set of all platform triples to consider when generating dependencies.", - default = SUPPORTED_PLATFORM_TRIPLES, - ), - "vendor_path": attr.string( - doc = "The path to a directory to write files into. Absolute paths will be treated as relative to the workspace root", - default = "crates", - ), - }, + attrs = CRATES_VENDOR_ATTRS, executable = True, toolchains = ["@rules_rust//rust:toolchain_type"], ) + def _crates_vendor_remote_repository_impl(repository_ctx): build_file = repository_ctx.path(repository_ctx.attr.build_file) defs_module = repository_ctx.path(repository_ctx.attr.defs_module) diff --git a/examples/bzlmod/crate_universe/.bazelrc b/examples/bzlmod/crate_universe/.bazelrc new file mode 100644 index 0000000000..e2ece0c386 --- /dev/null +++ b/examples/bzlmod/crate_universe/.bazelrc @@ -0,0 +1 @@ +build --experimental_enable_bzlmod diff --git a/examples/bzlmod/crate_universe/.gitignore b/examples/bzlmod/crate_universe/.gitignore new file mode 100644 index 0000000000..a6ef824c1f --- /dev/null +++ b/examples/bzlmod/crate_universe/.gitignore @@ -0,0 +1 @@ +/bazel-* diff --git a/examples/bzlmod/crate_universe/BUILD.bazel b/examples/bzlmod/crate_universe/BUILD.bazel new file mode 100644 index 0000000000..03b3ffffc2 --- /dev/null +++ b/examples/bzlmod/crate_universe/BUILD.bazel @@ -0,0 +1,14 @@ +load("@rules_rust//rust:defs.bzl", "rust_binary") + +rust_binary( + name = "hello_log", + srcs = ["src/main.rs"], + # At the moment, defs.bzl isn't generated, so we have no aliases, + # all_crate_deps, etc. + # See todo in //bzlmod/private/crate/crate.bzl for details. + # So for now users need to explicitly specify which crates they depend on. + deps = [ + "@crates//:env_logger", + "@crates//:log", + ] +) diff --git a/examples/bzlmod/crate_universe/Cargo.lock b/examples/bzlmod/crate_universe/Cargo.lock new file mode 100644 index 0000000000..efb8305ad2 --- /dev/null +++ b/examples/bzlmod/crate_universe/Cargo.lock @@ -0,0 +1,143 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "aho-corasick" +version = "0.7.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc936419f96fa211c1b9166887b38e5e40b19958e5b895be7c1f93adec7071ac" +dependencies = [ + "memchr", +] + +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi", + "libc", + "winapi", +] + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "demo" +version = "0.1.0" +dependencies = [ + "env_logger", + "log", +] + +[[package]] +name = "env_logger" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a12e6657c4c97ebab115a42dcee77225f7f482cdd841cf7088c657a42e9e00e7" +dependencies = [ + "atty", + "humantime", + "log", + "regex", + "termcolor", +] + +[[package]] +name = "hermit-abi" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +dependencies = [ + "libc", +] + +[[package]] +name = "humantime" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" + +[[package]] +name = "libc" +version = "0.2.139" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "201de327520df007757c1f0adce6e827fe8562fbc28bfd9c15571c66ca1f5f79" + +[[package]] +name = "log" +version = "0.4.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "memchr" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" + +[[package]] +name = "regex" +version = "1.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48aaa5748ba571fb95cd2c85c09f629215d3a6ece942baa100950af03a34f733" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.6.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "456c603be3e8d448b072f410900c09faf164fbce2d480456f50eea6e25f9c848" + +[[package]] +name = "termcolor" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be55cf8942feac5c765c2c993422806843c9a9a45d4d5c407ad6dd2ea95eb9b6" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +dependencies = [ + "winapi", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" diff --git a/examples/bzlmod/crate_universe/Cargo.toml b/examples/bzlmod/crate_universe/Cargo.toml new file mode 100644 index 0000000000..116f603c90 --- /dev/null +++ b/examples/bzlmod/crate_universe/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "demo" +version = "0.1.0" +edition = "2021" + +[dependencies] +env_logger = "0.9.2" +log = "0.4.17" \ No newline at end of file diff --git a/examples/bzlmod/crate_universe/MODULE.bazel b/examples/bzlmod/crate_universe/MODULE.bazel new file mode 100644 index 0000000000..77e63babca --- /dev/null +++ b/examples/bzlmod/crate_universe/MODULE.bazel @@ -0,0 +1,37 @@ +module( + name = "crates_repository", + version = "1.0" +) + +bazel_dep(name = "rules_rust", version = "0.17.0") + + +local_path_override(module_name = "rules_rust", path = "../../..") + +rust_toolchains = use_extension("@rules_rust//rust:extensions.bzl", "toolchains") +rust_toolchains.toolchain(edition = "2021") +use_repo( + rust_toolchains, + # Rename it from _rust_toolchains. + rust_toolchains = "crates_repository_rust_toolchains", +) +register_toolchains("@rust_toolchains//:all") + +crate = use_extension("@rules_rust//rust:extensions.bzl", "crate") +crate.from_cargo( + manifests = ["//:Cargo.toml"], + cargo_lockfile = "//:Cargo.lock", +) + +# Add annotations to crates like so: +crate.annotation( + crate = "env_logger", + version = "*", + rustc_env = dict(additive_build_file_content = "") +) + +use_repo( + crate, + # Rename it from _crates to crates + crates = "crates_repository_crates", +) \ No newline at end of file diff --git a/examples/bzlmod/crate_universe/WORKSPACE b/examples/bzlmod/crate_universe/WORKSPACE new file mode 100644 index 0000000000..e69de29bb2 diff --git a/examples/bzlmod/crate_universe/src/main.rs b/examples/bzlmod/crate_universe/src/main.rs new file mode 100644 index 0000000000..14e1f53e94 --- /dev/null +++ b/examples/bzlmod/crate_universe/src/main.rs @@ -0,0 +1,22 @@ +// Copyright 2023 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +fn main() { + env_logger::init(); + if log::log_enabled!(log::Level::Info) { + println!("Info enabled"); + } else { + println!("Info disabled"); + } +} diff --git a/rust/extensions.bzl b/rust/extensions.bzl index a2674af153..1c199d3720 100644 --- a/rust/extensions.bzl +++ b/rust/extensions.bzl @@ -1,7 +1,9 @@ "Module extensions for using rules_rust with bzlmod" load("//bzlmod/private/cargo_bazel_bootstrap:cargo_bazel_bootstrap.bzl", _cargo_bazel_bootstrap = "cargo_bazel_bootstrap") +load("//bzlmod/private/crate:crate.bzl", _crate = "crate") load("//bzlmod/private/toolchains:toolchains.bzl", _toolchains = "toolchains") cargo_bazel_bootstrap = _cargo_bazel_bootstrap +crate = _crate toolchains = _toolchains From da61bfee18d6a330cd0d9743b07cee7971d5ecc2 Mon Sep 17 00:00:00 2001 From: Matt Stark Date: Mon, 20 Feb 2023 16:43:19 +1100 Subject: [PATCH 12/12] Add bzlmod crate universe to CI. --- .bazelci/presubmit.yml | 10 ++++++++-- MODULE.bazel | 2 +- bzlmod/private/crate/crate.bzl | 8 ++++---- bzlmod/private/generate_repo.bzl | 10 ++++++++++ bzlmod/private/symlink_repo.bzl | 10 ---------- 5 files changed, 23 insertions(+), 17 deletions(-) create mode 100644 bzlmod/private/generate_repo.bzl delete mode 100644 bzlmod/private/symlink_repo.bzl diff --git a/.bazelci/presubmit.yml b/.bazelci/presubmit.yml index 6892852f4b..c5e87dcb79 100644 --- a/.bazelci/presubmit.yml +++ b/.bazelci/presubmit.yml @@ -342,12 +342,18 @@ tasks: - "//..." build_flags: *aspects_flags soft_fail: yes - ubuntu2004_examples_bzlmod: - name: Bzlmod Examples + ubuntu2004_examples_bzlmod_hello_world: + name: Bzlmod hello world example platform: ubuntu2004 working_directory: examples/bzlmod/hello_world build_targets: - "//..." + ubuntu2004_examples_bzlmod_crate_universe: + name: Bzlmod crate universe examples + platform: ubuntu2004 + working_directory: examples/bzlmod/crate_universe + build_targets: + - "//..." rbe_ubuntu1604_examples: name: Examples platform: rbe_ubuntu1604 diff --git a/MODULE.bazel b/MODULE.bazel index 7c691ac214..307defbac1 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -9,7 +9,7 @@ bazel_dep(name = "bazel_skylib", version = "1.2.0") bazel_dep( name = "apple_support", repo_name = "build_bazel_apple_support", - version = "1.4.0" + version = "1.3.2" ) non_bzlmod_deps = use_extension("//bzlmod/private/non_bzlmod_deps:non_bzlmod_deps.bzl", "non_bzlmod_deps") diff --git a/bzlmod/private/crate/crate.bzl b/bzlmod/private/crate/crate.bzl index e0442334cc..f9cb45a5cb 100644 --- a/bzlmod/private/crate/crate.bzl +++ b/bzlmod/private/crate/crate.bzl @@ -7,7 +7,7 @@ load("//rust/platform:triple.bzl", "get_host_triple") load("//crate_universe/private:crates_vendor.bzl", "CRATES_VENDOR_ATTRS", "write_config_file", "write_splicing_manifest", _crates_vendor_repo_rule = "crates_vendor") load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") load("@bazel_tools//tools/build_defs/repo:git.bzl", "new_git_repository") -load("//bzlmod/private:symlink_repo.bzl", "symlink_repo") +load("//bzlmod/private:generate_repo.bzl", "generate_repo") load("//bzlmod/private/cargo_bazel_bootstrap:cargo_bazel_bootstrap.bzl", "get_cargo_bazel_runner") load("//bzlmod/private/crate/tag_classes:annotation.bzl", "annotation_tags_to_json", annotation_tag = "annotation") load("//bzlmod/private/crate/tag_classes:from_cargo.bzl", from_cargo_tag = "from_cargo") @@ -113,10 +113,10 @@ def _crate_impl(module_ctx): ]) crates_dir = tag_path.get_child(repo_name) - symlink_repo( + generate_repo( name = repo_name, - files = { - "BUILD.bazel": str(crates_dir.get_child("BUILD.bazel")), + contents = { + "BUILD.bazel": module_ctx.read(crates_dir.get_child("BUILD.bazel")), }, ) diff --git a/bzlmod/private/generate_repo.bzl b/bzlmod/private/generate_repo.bzl new file mode 100644 index 0000000000..c3c2c90e37 --- /dev/null +++ b/bzlmod/private/generate_repo.bzl @@ -0,0 +1,10 @@ +def _generate_repo_impl(repo_ctx): + for path, contents in repo_ctx.attr.contents.items(): + repo_ctx.file(path, contents) + +generate_repo = repository_rule( + implementation = _generate_repo_impl, + attrs = dict( + contents = attr.string_dict(mandatory = True), + ), +) diff --git a/bzlmod/private/symlink_repo.bzl b/bzlmod/private/symlink_repo.bzl deleted file mode 100644 index 93614d2bf9..0000000000 --- a/bzlmod/private/symlink_repo.bzl +++ /dev/null @@ -1,10 +0,0 @@ -def _symlink_repo_impl(repo_ctx): - for path, target in repo_ctx.attr.files.items(): - repo_ctx.symlink(target, path) - -symlink_repo = repository_rule( - implementation = _symlink_repo_impl, - attrs = dict( - files = attr.string_dict(mandatory = True), - ), -)