Crate universe

.bazelci/presubmit.yml

@@ -342,6 +342,18 @@ tasks:
       - "//..."
     build_flags: *aspects_flags
     soft_fail: yes
+  ubuntu2004_examples_bzlmod_hello_world:
+    name: Bzlmod hello world example
+    platform: ubuntu2004
+    working_directory: examples/bzlmod/hello_world
+    build_targets:
+      - "//..."
+  ubuntu2004_examples_bzlmod_crate_universe:
+    name: Bzlmod crate universe examples
+    platform: ubuntu2004
+    working_directory: examples/bzlmod/crate_universe
+    build_targets:
+      - "//..."
   rbe_ubuntu1604_examples:
     name: Examples
     platform: rbe_ubuntu1604

MODULE.bazel

@@ -0,0 +1,35 @@
+module(
+    name = "rules_rust",
+    version = "0.19.0",
+)
+
+bazel_dep(name = "platforms", version = "0.0.5")
+bazel_dep(name = "rules_cc", version = "0.0.1")
+bazel_dep(name = "bazel_skylib", version = "1.2.0")
+bazel_dep(
+    name = "apple_support",
+    repo_name = "build_bazel_apple_support",
+    version = "1.3.2"
+)
+
+non_bzlmod_deps = use_extension("//bzlmod/private/non_bzlmod_deps:non_bzlmod_deps.bzl", "non_bzlmod_deps")
+non_bzlmod_deps.non_bzlmod_deps()
+use_repo(
+    non_bzlmod_deps,
+    "rules_rust_tinyjson"
+)
+
+# Set up a default toolchain, so we can use it to bootstrap cargo bazel.
+toolchains = use_extension("//rust:extensions.bzl", "toolchains")
+toolchains.host_tools()
+use_repo(
+    toolchains,
+    rust_host_tools = "rules_rust_rust_host_tools",
+)
+
+cargo_bazel_bootstrap = use_extension("//rust:extensions.bzl", "cargo_bazel_bootstrap")
+cargo_bazel_bootstrap.cargo_bazel_bootstrap()
+use_repo(
+    cargo_bazel_bootstrap,
+    "cargo_bazel_bootstrap"
+)

bzlmod/private/cargo_bazel_bootstrap/cargo_bazel_bootstrap.bzl

@@ -0,0 +1,45 @@
+load("//cargo/private:cargo_utils.bzl", "get_rust_tools")
+load("//crate_universe:deps_bootstrap.bzl", _cargo_bazel_bootstrap_repo_rule = "cargo_bazel_bootstrap")
+load("//rust:defs.bzl", "rust_common")
+
+def _cargo_bazel_bootstrap_impl(module_ctx):
+    _cargo_bazel_bootstrap_repo_rule(
+        rust_toolchain_cargo_template = "@rust_host_tools//:bin/{tool}",
+        rust_toolchain_rustc_template = "@rust_host_tools//:bin/{tool}",
+    )
+
+cargo_bazel_bootstrap_cargo_bazel_bootstrap = tag_class(attrs = {})
+cargo_bazel_bootstrap = module_extension(
+    implementation = _cargo_bazel_bootstrap_impl,
+    tag_classes = dict(
+        cargo_bazel_bootstrap = cargo_bazel_bootstrap_cargo_bazel_bootstrap,
+    ),
+)
+
+def get_cargo_bazel_runner(module_ctx):
+    cargo_path = str(module_ctx.path(Label("@rust_host_tools//:bin/cargo")))
+    rustc_path = str(module_ctx.path(Label("@rust_host_tools//:bin/rustc")))
+    cargo_bazel = module_ctx.path(Label("@cargo_bazel_bootstrap//:cargo-bazel"))
+
+    def run(args, env = {}, timeout = 600):
+        final_args = [cargo_bazel]
+        final_args.extend(args)
+        final_args.extend([
+            "--cargo",
+            cargo_path,
+            "--rustc",
+            rustc_path,
+        ])
+        result = module_ctx.execute(
+            final_args,
+            environment = dict(CARGO = cargo_path, RUSTC = rustc_path, **env),
+            timeout = timeout,
+        )
+        if result.return_code != 0:
+            if result.stdout:
+                print("Stdout:", result.stdout)
+            pretty_args = " ".join([str(arg) for arg in final_args])
+            fail("%s returned with exit code %d:\n%s" % (pretty_args, result.return_code, result.stderr))
+        return result
+
+    return run

bzlmod/private/crate/crate.bzl

@@ -0,0 +1,176 @@
+load("//cargo/private:cargo_utils.bzl", _rust_get_rust_tools = "get_rust_tools")
+load("//crate_universe/private:generate_utils.bzl", "render_config")
+load("//crate_universe/private:common_utils.bzl", "get_rust_tools")
+load("//crate_universe/private:crates_repository.bzl", "crates_repository")
+load("//rust:repositories.bzl", "rust_register_toolchains")
+load("//rust/platform:triple.bzl", "get_host_triple")
+load("//crate_universe/private:crates_vendor.bzl", "CRATES_VENDOR_ATTRS", "write_config_file", "write_splicing_manifest", _crates_vendor_repo_rule = "crates_vendor")
+load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
+load("@bazel_tools//tools/build_defs/repo:git.bzl", "new_git_repository")
+load("//bzlmod/private:generate_repo.bzl", "generate_repo")
+load("//bzlmod/private/cargo_bazel_bootstrap:cargo_bazel_bootstrap.bzl", "get_cargo_bazel_runner")
+load("//bzlmod/private/crate/tag_classes:annotation.bzl", "annotation_tags_to_json", annotation_tag = "annotation")
+load("//bzlmod/private/crate/tag_classes:from_cargo.bzl", from_cargo_tag = "from_cargo")
+load("//bzlmod/private/crate/tag_classes:spec.bzl", "spec_tags_to_json", spec_tag = "spec")
+
+def _crate_impl(module_ctx):
+    cargo_bazel = get_cargo_bazel_runner(module_ctx)
+
+    for mod in module_ctx.modules:
+        mod_path = module_ctx.path(mod.name)
+
+        annotations = annotation_tags_to_json(mod.tags.annotation)
+        packages = spec_tags_to_json(mod.tags.spec)
+
+        # At the moment, this is rather inefficient in the case of many tags
+        # depending on the same crate, as we have to build each package for each
+        # cargo lockfile that depends on it (we generate the same package once
+        # per cargo lockfile).
+        # This is a non-issue if using cargo workspaces, which I'd generally
+        # recommend anyway, but in the long term we should probably try and
+        # share repos if they use the same configuration.
+        for cfg in mod.tags.from_cargo:
+            tag_path = mod_path
+            repo_name = mod.name + "_crates"
+            if cfg.suffix:
+                tag_path = mod_path.get_child(cfg.suffix)
+                repo_name += "_" + cfg.suffix
+
+            cargo_lockfile = module_ctx.path(cfg.cargo_lockfile)
+
+            def write_data_file(ctx, name, data):
+                path = tag_path.get_child(name)
+                module_ctx.file(path, content = data, executable = False)
+                return path
+
+            rendering_config = json.decode(render_config())
+            rendering_config["regen_command"] = "Run 'cargo update [--workspace]'"
+            config_file = write_config_file(
+                module_ctx,
+                mode = "remote",
+                annotations = annotations,
+                generate_build_scripts = cfg.generate_build_scripts,
+                supported_platform_triples = cfg.supported_platform_triples,
+                repository_name = repo_name,
+                output_pkg = repo_name,
+                workspace_name = repo_name,
+                write_data_file = write_data_file,
+                generate_binaries = cfg.generate_binaries,
+                rendering_config = rendering_config,
+            )
+
+            manifests = {module_ctx.path(m): m for m in cfg.manifests}
+            splicing_manifest = write_splicing_manifest(
+                module_ctx,
+                packages = packages,
+                splicing_config = "",
+                cargo_config = cfg.cargo_config,
+                manifests = {str(p): str(l) for p, l in manifests.items()},
+                write_data_file = write_data_file,
+                manifest_to_path = module_ctx.path,
+            )
+
+            splicing_output_dir = tag_path.get_child("splicing-output")
+            cargo_bazel([
+                "splice",
+                "--output-dir",
+                splicing_output_dir,
+                "--config",
+                config_file,
+                "--splicing-manifest",
+                splicing_manifest,
+                "--cargo-lockfile",
+                cargo_lockfile,
+            ])
+
+            # Create a lockfile, since we need to parse it to generate spoke
+            # repos.
+            lockfile_path = tag_path.get_child("lockfile.json")
+            module_ctx.file(lockfile_path, "")
+
+            # TODO: Thanks to the namespacing of bzlmod, although we generate
+            # defs.bzl, it isn't useful (it generates deps like
+            # "<module_name>_crates__env_logger-0.9.3//:env_logger", which
+            # don't work because that repository isn't visible from main).
+            # To solve this, we need to generate the alias
+            # @crates//:env_logger-0.9.3 as well as the @crates//:env_logger
+            # that it already generates, and point defs.bzl there instead.
+            cargo_bazel([
+                "generate",
+                "--cargo-lockfile",
+                cargo_lockfile,
+                "--config",
+                config_file,
+                "--splicing-manifest",
+                splicing_manifest,
+                "--repository-dir",
+                tag_path,
+                "--metadata",
+                splicing_output_dir.get_child("metadata.json"),
+                "--repin",
+                "--lockfile",
+                lockfile_path,
+            ])
+
+            crates_dir = tag_path.get_child(repo_name)
+            generate_repo(
+                name = repo_name,
+                contents = {
+                    "BUILD.bazel": module_ctx.read(crates_dir.get_child("BUILD.bazel")),
+                },
+            )
+
+            contents = json.decode(module_ctx.read(lockfile_path))
+
+            for crate in contents["crates"].values():
+                repo = crate["repository"]
+                if repo == None:
+                    continue
+                name = crate["name"]
+                version = crate["version"]
+                # "+" isn't valid in a repo name.
+                crate_repo_name = "%s__%s-%s" % (repo_name, name, version.replace("+", "-"))
+
+                patch_tool = repo.get("patch_tool", None)
+                patches = repo.get("patches", None)
+                patch_args = repo.get("patch_args", None)
+
+                build_file_content = module_ctx.read(crates_dir.get_child("BUILD.%s-%s.bazel" % (name, version)))
+                if "Http" in repo:
+                    # Replicates repo_http.j2
+                    http_archive(
+                        name = crate_repo_name,
+                        patch_args = patch_args,
+                        patch_tool = patch_tool,
+                        patches = patches,
+                        sha256 = repo["Http"]["sha256"],
+                        type = "tar.gz",
+                        urls = [repo["Http"]["url"]],
+                        strip_prefix = "%s-%s" % (crate["name"], crate["version"]),
+                        build_file_content = build_file_content,
+                    )
+                elif "Git" in repo:
+                    # Replicates repo_git.j2
+                    new_git_repository(
+                        name = crate_repo_name,
+                        init_submodules = True,
+                        patch_args = patch_args,
+                        patch_tool = patch_tool,
+                        patches = patches,
+                        shallow_since = repo.get("shallow_since", None),
+                        remote = repo["Git"]["remote"],
+                        build_file_content = build_file_content,
+                        strip_prefix = repo.get("strip_prefix", None),
+                        **repo["commitish"],
+                    )
+                else:
+                    fail("Invalid repo: expected Http or Git to exist for crate %s-%s, got %s" % (name, version, repo))
+
+crate = module_extension(
+    implementation = _crate_impl,
+    tag_classes = dict(
+        annotation = annotation_tag,
+        from_cargo = from_cargo_tag,
+        spec = spec_tag,
+    ),
+)

bzlmod/private/crate/tag_classes/annotation.bzl

@@ -0,0 +1,116 @@
+load("//crate_universe:defs.bzl", "crate")
+
+_ANNOTATION_ATTRS = dict(
+    crate = attr.string(
+        mandatory = True,
+        doc = "The crate to apply the annotation to. The wildcard `*` matches any version, including prerelease versions.",
+    ),
+    version = attr.string(
+        default = "*",
+        doc = "The version or semver-conditions to match with a crate.",
+    ),
+    additive_build_file_content = attr.string(
+        doc = "Extra contents to write to the bottom of generated BUILD files.",
+    ),
+    additive_build_file = attr.label(
+        allow_single_file = True,
+        doc = "A file containing extra contents to write to the bottom of generated BUILD files",
+    ),
+    build_script_data = attr.label_list(
+        doc = "A list of labels to add to a crate's `cargo_build_script::data` attribute.",
+    ),
+    build_script_tools = attr.label_list(
+        doc = "A list of labels to add to a crate's `cargo_build_script::tools` attribute.",
+    ),
+    build_script_data_glob = attr.string_list(
+        doc = "A list of glob patterns to add to a crate's `cargo_build_script::data` attribute",
+    ),
+    build_script_deps = attr.label_list(
+        doc = "A list of labels to add to a crate's `cargo_build_script::deps` attribute.",
+    ),
+    build_script_env = attr.string_dict(
+        doc = "Additional environment variables to set on a crate's `cargo_build_script::env` attribute.",
+    ),
+    build_script_proc_macro_deps = attr.label_list(
+        doc = "A list of labels to add to a crate's `cargo_build_script::proc_macro_deps` attribute.",
+    ),
+    build_script_rustc_env = attr.string_dict(
+        doc = "Additional environment variables to set on a crate's `cargo_build_script::env` attribute.",
+    ),
+    build_script_toolchains = attr.label_list(
+        doc = "A list of labels to set on a crates's `cargo_build_script::toolchains` attribute.",
+    ),
+    compile_data = attr.label_list(
+        doc = "A list of labels to add to a crate's `rust_library::compile_data` attribute.",
+    ),
+    compile_data_glob = attr.string_list(
+        doc = "A list of glob patterns to add to a crate's `rust_library::compile_data` attribute",
+    ),
+    crate_features = attr.string_list(
+        doc = "A list of strings to add to a crate's `rust_library::crate_features` attribute.",
+    ),
+    data = attr.label_list(
+        doc = "A list of labels to add to a crate's `rust_library::data` attribute.",
+    ),
+    data_glob = attr.string_list(
+        doc = "A list of glob patterns to add to a crate's `rust_library::data` attribute.",
+    ),
+    deps = attr.label_list(
+        doc = "A list of labels to add to a crate's `rust_library::deps` attribute.",
+    ),
+    disable_pipelining = attr.bool(
+        doc = "If True, disables pipelining for library targets for this crate.",
+    ),
+    gen_all_binaries = attr.bool(
+        doc = "If true, generates all binaries for a crate."
+    ),
+    gen_binaries = attr.string_list(
+        doc = "Thu subset of the crate's bins that should get `rust_binary` targets produced."
+    ),
+    gen_build_script = attr.bool(
+        doc = "An authorative flag to determine whether or not to produce `cargo_build_script` targets for the current crate.",
+    ),
+    patch_args = attr.string_list(
+        doc = "The `patch_args` attribute of a Bazel repository rule. See [http_archive.patch_args](https://docs.bazel.build/versions/main/repo/http.html#http_archive-patch_args)",
+    ),
+    patch_tool = attr.string(
+        doc = "The `patch_tool` attribute of a Bazel repository rule. See [http_archive.patch_tool](https://docs.bazel.build/versions/main/repo/http.html#http_archive-patch_tool)",
+    ),
+    patches = attr.label_list(
+        doc = "The `patches` attribute of a Bazel repository rule. See [http_archive.patches](https://docs.bazel.build/versions/main/repo/http.html#http_archive-patches)",
+    ),
+    proc_macro_deps = attr.label_list(
+        doc = "A list of labels to add to a crate's `rust_library::proc_macro_deps` attribute.",
+    ),
+    rustc_env = attr.string_dict(
+        doc = "Additional variables to set on a crate's `rust_library::rustc_env` attribute.",
+    ),
+    rustc_env_files = attr.label_list(
+        doc = "A list of labels to set on a crate's `rust_library::rustc_env_files` attribute.",
+    ),
+    rustc_flags = attr.string_list(
+        doc = "A list of strings to set on a crate's `rust_library::rustc_flags` attribute.",
+    ),
+    shallow_since = attr.string(
+        doc = "An optional timestamp used for crates originating from a git repository instead of a crate registry. This flag optimizes fetching the source code.",
+    ),
+
+)
+
+annotation = tag_class(
+    doc = "A collection of extra attributes and settings for a particular crate",
+    attrs = _ANNOTATION_ATTRS,
+)
+
+def annotation_tags_to_json(tags):
+    annotations = {}
+    for tag in tags:
+        if tag.crate not in annotations:
+            annotations[tag.crate] = []
+
+        kwargs = {k: getattr(tag, k) for k in _ANNOTATION_ATTRS}
+        kwargs.pop("crate")
+        if kwargs.pop("gen_all_binaries"):
+            kwargs["gen_binaries"] = True
+        annotations[tag.crate].append(crate.annotation(**kwargs))
+    return annotations