Add subcommands and busybox entry points for LLVM tools

This adds support for most of the remaining LLVM command line tools
using a generic, generated wrapper. The subcommand interface for these
is (much) less interesting than our other subcommands, but it gives us
a uniform and consistent layer.

Note that I structured these as nested sub-sub-commands below an `llvm`
subcommand because of an expectation that we will want to add more, and
ones that don't use this generic layer. Some concrete future work:

- Add the `opt` and `llc` tools as subcommands for easier debugging and
  experimentation with LLVM IR output from Carbon's toolchain.
- Potentially sink `lld` below the `llvm` layer given that it has
  significantly less user visibility than commands like `clang`.

Unfortunately, the current driver subcommand APIs make nested
subcommands awkward. I've added a somewhat rough hack here to let the
LLVM tools go in, but there are some TODOs that I want to address in
a follow-up that works to adjust the structure of this code to be more
conducive to nesting like this.

scripts/fix_cc_deps.py

@@ -68,7 +68,9 @@ class RuleChoice(NamedTuple):
 }
 
 IGNORE_SOURCE_FILE_REGEX = re.compile(
-    r"^(third_party/clangd.*|common/version.*\.cpp|.*_autogen_manifest\.cpp)$"
+    r"^(third_party/clangd.*|common/version.*\.cpp"
+    r"|.*_autogen_manifest\.cpp"
+    r"|toolchain/base/llvm_tools.def)$"
 )
 
 

toolchain/base/BUILD

@@ -3,6 +3,7 @@
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
 load("@rules_cc//cc:defs.bzl", "cc_library", "cc_test")
+load("llvm_tools.bzl", "LLVM_TOOLS", "generate_llvm_tools_def")
 
 package(default_visibility = ["//visibility:public"])
 
@@ -120,6 +121,23 @@ cc_test(
     ],
 )
 
+generate_llvm_tools_def(
+    name = "llvm_tools_def",
+    out = "llvm_tools.def",
+)
+
+cc_library(
+    name = "llvm_tools",
+    srcs = ["llvm_tools.cpp"],
+    hdrs = ["llvm_tools.h"],
+    deps = [
+        ":llvm_tools_def",
+        "//common:command_line",
+        "//common:enum_base",
+        "@llvm-project//llvm:Support",
+    ] + [info.lib for info in LLVM_TOOLS.values()],
+)
+
 cc_library(
     name = "shared_value_stores",
     hdrs = ["shared_value_stores.h"],

toolchain/base/llvm_tools.bzl

@@ -0,0 +1,127 @@
+# Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+# Exceptions. See /LICENSE for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+"""Provides variables and rules to automate working with LLVM's CLI tools."""
+
+load("@rules_cc//cc:cc_library.bzl", "cc_library")
+
+LLVM_TOOLS = {
+    "ar": struct(bin_name = "llvm-ar", lib = "@llvm-project//llvm:llvm-ar-lib"),
+    "cgdata": struct(bin_name = "llvm-cgdata", lib = "@llvm-project//llvm:llvm-cgdata-lib"),
+    "cxxfilt": struct(bin_name = "llvm-cxxfilt", lib = "@llvm-project//llvm:llvm-cxxfilt-lib"),
+    "debuginfod-find": struct(bin_name = "llvm-debuginfod-find", lib = "@llvm-project//llvm:llvm-debuginfod-find-lib"),
+    "dsymutil": struct(bin_name = "dsymutil", lib = "@llvm-project//llvm:dsymutil-lib"),
+    "dwp": struct(bin_name = "llvm-dwp", lib = "@llvm-project//llvm:llvm-dwp-lib"),
+    "gsymutil": struct(bin_name = "llvm-gsymutil", lib = "@llvm-project//llvm:llvm-gsymutil-lib"),
+    "ifs": struct(bin_name = "llvm-ifs", lib = "@llvm-project//llvm:llvm-ifs-lib"),
+    "libtool-darwin": struct(bin_name = "llvm-libtool-darwin", lib = "@llvm-project//llvm:llvm-libtool-darwin-lib"),
+    "lipo": struct(bin_name = "llvm-lipo", lib = "@llvm-project//llvm:llvm-lipo-lib"),
+    "ml": struct(bin_name = "llvm-ml", lib = "@llvm-project//llvm:llvm-ml-lib"),
+    "mt": struct(bin_name = "llvm-mt", lib = "@llvm-project//llvm:llvm-mt-lib"),
+    "nm": struct(bin_name = "llvm-nm", lib = "@llvm-project//llvm:llvm-nm-lib"),
+    "objcopy": struct(bin_name = "llvm-objcopy", lib = "@llvm-project//llvm:llvm-objcopy-lib"),
+    "objdump": struct(bin_name = "llvm-objdump", lib = "@llvm-project//llvm:llvm-objdump-lib"),
+    "profdata": struct(bin_name = "llvm-profdata", lib = "@llvm-project//llvm:llvm-profdata-lib"),
+    "rc": struct(bin_name = "llvm-rc", lib = "@llvm-project//llvm:llvm-rc-lib"),
+    "readobj": struct(bin_name = "llvm-readobj", lib = "@llvm-project//llvm:llvm-readobj-lib"),
+    "sancov": struct(bin_name = "sancov", lib = "@llvm-project//llvm:sancov-lib"),
+    "size": struct(bin_name = "llvm-size", lib = "@llvm-project//llvm:llvm-size-lib"),
+    "symbolizer": struct(bin_name = "llvm-symbolizer", lib = "@llvm-project//llvm:llvm-symbolizer-lib"),
+}
+
+LLVM_TOOL_ALIASES = {
+    "ar": ["ranlib", "lib", "dlltool"],
+    #"cxxfilt": ["c++filt"],
+    "objcopy": ["bitcode-strip", "install-name-tool", "strip"],
+    "objdump": ["otool"],
+    "rc": ["windres"],
+    "readobj": ["readelf"],
+    "symbolizer": ["addr2line"],
+}
+
+_DEF_FILE_TEMPLATE = """
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// This is a generated X-macro header for defining LLVM tools.
+//
+// See toolchain/driver/llvm_tools.bzl for more details.
+
+#ifndef CARBON_LLVM_TOOL
+#define CARBON_LLVM_TOOL(Id, Name, BinName, MainFn)
+#endif
+
+#ifndef CARBON_LLVM_MAIN_TOOL
+#define CARBON_LLVM_MAIN_TOOL(Id, Name, BinName, MainFn) \\
+  CARBON_LLVM_TOOL(Id, Name, BinName, MainFn)
+#endif
+
+#ifndef CARBON_LLVM_ALIAS_TOOL
+#define CARBON_LLVM_ALIAS_TOOL(Id, Name, BinName, MainFn) \\
+  CARBON_LLVM_TOOL(Id, Name, BinName, MainFn)
+#endif
+
+{}
+
+#undef CARBON_LLVM_TOOL
+#undef CARBON_LLVM_MAIN_TOOL
+#undef CARBON_LLVM_ALIAS_TOOL
+"""
+
+_DEF_MACRO_TEMPLATE = """
+CARBON_LLVM_{kind}TOOL({id}, "{name}", "{bin_name}", {main_fn})
+""".strip()
+
+def _build_def_macro(kind, name, bin_name, main_info):
+    id = "".join([w.title() for w in name.split("-")])
+    main_fn = main_info.bin_name.replace("-", "_") + "_main"
+    return _DEF_MACRO_TEMPLATE.format(
+        kind = kind,
+        id = id,
+        name = name,
+        bin_name = bin_name,
+        main_fn = main_fn,
+    )
+
+def _generate_llvm_tools_def_rule(ctx):
+    def_lines = []
+
+    for name, tool_info in LLVM_TOOLS.items():
+        def_lines.append(_build_def_macro("MAIN_", name, tool_info.bin_name, tool_info))
+
+    for target, aliases in LLVM_TOOL_ALIASES.items():
+        tool_info = LLVM_TOOLS[target]
+        for alias in aliases:
+            bin_name = "llvm-" + alias
+            def_lines.append(_build_def_macro("ALIAS_", alias, bin_name, tool_info))
+
+    def_file = ctx.actions.declare_file(ctx.label.name)
+    ctx.actions.write(def_file, _DEF_FILE_TEMPLATE.format("\n".join(def_lines)))
+    return [DefaultInfo(files = depset([def_file]))]
+
+generate_llvm_tools_def_rule = rule(
+    implementation = _generate_llvm_tools_def_rule,
+    attrs = {},
+)
+
+def generate_llvm_tools_def(name, out, **kwargs):
+    """Generates the LLVM tools `.def` file.
+
+    This first generates the `.def` file into the `out` filename, and then
+    synthesizes a `cc_library` rule exporting that file in its `textual_hdrs`.
+
+    The `cc_library` rule name is the provided `name` and should be depended on
+    by code that includes the generated file. The `kwargs` are expanded into the
+    `cc_library` in case other attributes need to be configured there.
+
+    The two-step process is necessary to avoid trying to compile or otherwise
+    process the generated file as something other than a textual header.
+    """
+    generate_llvm_tools_def_rule(name = out)
+    cc_library(
+        name = name,
+        textual_hdrs = [out],
+        **kwargs
+    )

toolchain/base/llvm_tools.cpp

@@ -0,0 +1,41 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include "toolchain/base/llvm_tools.h"
+
+#include "common/command_line.h"
+
+// NOLINTBEGIN(readability-identifier-naming): External library name.
+#define CARBON_LLVM_MAIN_TOOL(Identifier, Name, BinName, MainFn)              \
+  extern auto MainFn(int argc, char** argv, const llvm::ToolContext& context) \
+      -> int;
+#include "toolchain/base/llvm_tools.def"
+// NOLINTEND(readability-identifier-naming): External library name.
+
+namespace Carbon {
+
+CARBON_DEFINE_ENUM_CLASS_NAMES(LLVMTool) = {
+#define CARBON_LLVM_TOOL(Identifier, Name, BinName, MainFn) Name,
+#include "toolchain/base/llvm_tools.def"
+};
+
+constexpr llvm::StringLiteral LLVMTool::BinNames[] = {
+#define CARBON_LLVM_TOOL(Identifier, Name, BinName, MainFn) BinName,
+#include "toolchain/base/llvm_tools.def"
+};
+
+constexpr LLVMTool::MainFnT* LLVMTool::MainFns[] = {
+#define CARBON_LLVM_TOOL(Identifier, Name, BinName, MainFn) &::MainFn,
+#include "toolchain/base/llvm_tools.def"
+};
+
+constexpr CommandLine::CommandInfo LLVMTool::SubcommandInfos[] = {
+#define CARBON_LLVM_TOOL(Identifier, Name, BinName, MainFn) \
+  {.name = Name,                                            \
+   .help = "Runs the LLVM " Name                            \
+           " command line tool with the provided arguments."},
+#include "toolchain/base/llvm_tools.def"
+};
+
+}  // namespace Carbon

toolchain/base/llvm_tools.h

@@ -0,0 +1,67 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#ifndef CARBON_TOOLCHAIN_BASE_LLVM_TOOLS_H_
+#define CARBON_TOOLCHAIN_BASE_LLVM_TOOLS_H_
+
+#include <cstdint>
+
+#include "common/command_line.h"
+#include "common/enum_base.h"
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/LLVMDriver.h"
+
+namespace Carbon {
+
+CARBON_DEFINE_RAW_ENUM_CLASS(LLVMTool, uint8_t) {
+#define CARBON_LLVM_TOOL(Identifier, Name, BinName, MainFn) \
+  CARBON_RAW_ENUM_ENUMERATOR(Identifier)
+#include "toolchain/base/llvm_tools.def"
+};
+
+class LLVMTool : public CARBON_ENUM_BASE(LLVMTool) {
+ public:
+#define CARBON_LLVM_TOOL(Identifier, Name, BinName, MainFn) \
+  CARBON_ENUM_CONSTANT_DECL(Identifier)
+#include "toolchain/base/llvm_tools.def"
+
+  static const llvm::ArrayRef<LLVMTool> Tools;
+
+  using MainFnT = auto(int argc, char** argv, const llvm::ToolContext& context)
+      -> int;
+
+  using EnumBase::EnumBase;
+
+  auto bin_name() const -> llvm::StringLiteral { return BinNames[AsInt()]; }
+
+  auto main_fn() const -> MainFnT* { return MainFns[AsInt()]; }
+
+  auto subcommand_info() const -> const CommandLine::CommandInfo& {
+    return SubcommandInfos[AsInt()];
+  }
+
+ private:
+  static const LLVMTool ToolsStorage[];
+
+  static const llvm::StringLiteral BinNames[];
+  static MainFnT* const MainFns[];
+
+  static const CommandLine::CommandInfo SubcommandInfos[];
+};
+
+#define CARBON_LLVM_TOOL(Identifier, Name, BinName, MainFn) \
+  CARBON_ENUM_CONSTANT_DEFINITION(LLVMTool, Identifier)
+#include "toolchain/base/llvm_tools.def"
+
+constexpr LLVMTool LLVMTool::ToolsStorage[] = {
+#define CARBON_LLVM_TOOL(Identifier, Name, BinName, MainFn) \
+  LLVMTool::Identifier,
+#include "toolchain/base/llvm_tools.def"
+};
+constexpr llvm::ArrayRef<LLVMTool> LLVMTool::Tools = ToolsStorage;
+
+}  // namespace Carbon
+
+#endif  // CARBON_TOOLCHAIN_BASE_LLVM_TOOLS_H_

toolchain/driver/BUILD

@@ -104,6 +104,8 @@ cc_library(
         "link_subcommand.h",
         "lld_subcommand.cpp",
         "lld_subcommand.h",
+        "llvm_subcommand.cpp",
+        "llvm_subcommand.h",
     ],
     hdrs = [
         "driver.h",
@@ -116,12 +118,14 @@ cc_library(
     deps = [
         ":clang_runner",
         ":lld_runner",
+        ":llvm_runner",
         "//common:command_line",
         "//common:error",
         "//common:ostream",
         "//common:raw_string_ostream",
         "//common:version",
         "//common:vlog",
+        "//toolchain/base:llvm_tools",
         "//toolchain/base:pretty_stack_trace_function",
         "//toolchain/base:shared_value_stores",
         "//toolchain/base:timings",
@@ -223,6 +227,55 @@ cc_test(
     ],
 )
 
+config_setting(
+    name = "is_macos",
+    constraint_values = ["@platforms//os:macos"],
+)
+
+cc_library(
+    name = "llvm_runner",
+    srcs = ["llvm_runner.cpp"],
+    hdrs = ["llvm_runner.h"],
+    linkopts = select({
+        # TODO: This should be moved upstream to LLVM's tool libraries that
+        # require it.
+        ":is_macos": [
+            "-framework",
+            "CoreFoundation",
+        ],
+        "//conditions:default": [],
+    }),
+    deps = [
+        ":tool_runner_base",
+        "//common:ostream",
+        "//common:vlog",
+        "//toolchain/base:llvm_tools",
+        "//toolchain/install:install_paths",
+        "@llvm-project//lld:Common",
+        "@llvm-project//lld:ELF",
+        "@llvm-project//lld:MachO",
+        "@llvm-project//llvm:Support",
+    ],
+)
+
+cc_test(
+    name = "llvm_runner_test",
+    size = "small",
+    srcs = ["llvm_runner_test.cpp"],
+    env = cc_env(),
+    deps = [
+        ":llvm_runner",
+        "//common:all_llvm_targets",
+        "//common:ostream",
+        "//common:raw_string_ostream",
+        "//testing/base:capture_std_streams",
+        "//testing/base:global_exe_path",
+        "//testing/base:gtest_main",
+        "@googletest//:gtest",
+        "@llvm-project//llvm:Support",
+    ],
+)
+
 cc_library(
     name = "tool_runner_base",
     srcs = ["tool_runner_base.cpp"],

toolchain/driver/driver.cpp

@@ -16,6 +16,7 @@
 #include "toolchain/driver/language_server_subcommand.h"
 #include "toolchain/driver/link_subcommand.h"
 #include "toolchain/driver/lld_subcommand.h"
+#include "toolchain/driver/llvm_subcommand.h"
 
 namespace Carbon {
 
@@ -35,6 +36,7 @@ struct Options {
   LanguageServerSubcommand language_server;
   LinkSubcommand link;
   LldSubcommand lld;
+  LLVMSubcommand llvm;
 
   // On success, this is set to the subcommand to run.
   DriverSubcommand* selected_subcommand = nullptr;
@@ -92,6 +94,7 @@ applies to each message that forms a diagnostic, not just the primary message.
   language_server.AddTo(b, &selected_subcommand);
   link.AddTo(b, &selected_subcommand);
   lld.AddTo(b, &selected_subcommand);
+  llvm.AddTo(b, &selected_subcommand);
 
   b.RequiresSubcommand();
 }