Auto merge of #50336 - japaric:llvm-tools, r=Mark-Simulacrum

ship LLVM tools with the toolchain

this PR adds llvm-{nm,objcopy,objdump,size} to the rustc sysroot (right next to LLD)

this slightly increases the size of the rustc component. I measured these numbers on x86_64 Linux:

- rustc-1.27.0-dev-x86_64-unknown-linux-gnu.tar.gz 180M -> 193M (+7%)
- rustc-1.27.0-dev-x86_64-unknown-linux-gnu.tar.xz 129M -> 137M (+6%)

r? @alexcrichton
cc #49584

config.toml.example

@@ -350,6 +350,10 @@
 # rustc to execute.
 #lld = false
 
+# Indicates whether some LLVM tools, like llvm-objdump, will be made available in the
+# sysroot.
+#llvm-tools = false
+
 # Whether to deny warnings in crates
 #deny-warnings = true
 

src/bootstrap/builder.rs

@@ -453,6 +453,7 @@ impl<'a> Builder<'a> {
                 dist::Cargo,
                 dist::Rls,
                 dist::Rustfmt,
+                dist::LlvmTools,
                 dist::Extended,
                 dist::HashSign
             ),

src/bootstrap/config.rs

@@ -88,6 +88,7 @@ pub struct Config {
     pub llvm_link_jobs: Option<u32>,
 
     pub lld_enabled: bool,
+    pub llvm_tools_enabled: bool,
 
     // rust codegen options
     pub rust_optimize: bool,
@@ -309,6 +310,7 @@ struct Rust {
     codegen_backends_dir: Option<String>,
     wasm_syscall: Option<bool>,
     lld: Option<bool>,
+    llvm_tools: Option<bool>,
     deny_warnings: Option<bool>,
     backtrace_on_ice: Option<bool>,
 }
@@ -536,6 +538,7 @@ impl Config {
             }
             set(&mut config.wasm_syscall, rust.wasm_syscall);
             set(&mut config.lld_enabled, rust.lld);
+            set(&mut config.llvm_tools_enabled, rust.llvm_tools);
             config.rustc_parallel_queries = rust.experimental_parallel_queries.unwrap_or(false);
             config.rustc_default_linker = rust.default_linker.clone();
             config.musl_root = rust.musl_root.clone().map(PathBuf::from);

src/bootstrap/configure.py

@@ -335,6 +335,7 @@ def set(key, value):
     elif option.name == 'full-tools':
         set('rust.codegen-backends', ['llvm', 'emscripten'])
         set('rust.lld', True)
+        set('rust.llvm-tools', True)
         set('build.extended', True)
     elif option.name == 'option-checking':
         # this was handled above

src/bootstrap/dist.rs

@@ -26,7 +26,7 @@ use std::process::{Command, Stdio};
 
 use build_helper::output;
 
-use {Compiler, Mode};
+use {Compiler, Mode, LLVM_TOOLS};
 use channel;
 use util::{libdir, is_dylib, exe};
 use builder::{Builder, RunConfig, ShouldRun, Step};
@@ -43,6 +43,8 @@ pub fn pkgname(builder: &Builder, component: &str) -> String {
         format!("{}-{}", component, builder.rls_package_vers())
     } else if component == "rustfmt" {
         format!("{}-{}", component, builder.rustfmt_package_vers())
+    } else if component == "llvm-tools" {
+        format!("{}-{}", component, builder.llvm_tools_vers())
     } else {
         assert!(component.starts_with("rust"));
         format!("{}-{}", component, builder.rust_package_vers())
@@ -394,7 +396,7 @@ impl Step for Rustc {
         let compiler = self.compiler;
         let host = self.compiler.host;
 
-        builder.info(&format!("Dist rustc stage{} ({})", compiler.stage, compiler.host));
+        builder.info(&format!("Dist rustc stage{} ({})", compiler.stage, host));
         let name = pkgname(builder, "rustc");
         let image = tmpdir(builder).join(format!("{}-{}-image", name, host));
         let _ = fs::remove_dir_all(&image);
@@ -1738,6 +1740,7 @@ impl Step for HashSign {
         cmd.arg(builder.package_vers(&builder.release_num("cargo")));
         cmd.arg(builder.package_vers(&builder.release_num("rls")));
         cmd.arg(builder.package_vers(&builder.release_num("rustfmt")));
+        cmd.arg(builder.llvm_tools_vers());
         cmd.arg(addr);
 
         builder.create_dir(&distdir(builder));
@@ -1748,3 +1751,78 @@ impl Step for HashSign {
         assert!(status.success());
     }
 }
+
+#[derive(Clone, Debug, Eq, Hash, PartialEq)]
+pub struct LlvmTools {
+    pub stage: u32,
+    pub compiler: Compiler,
+    pub target: Interned<String>,
+}
+
+impl Step for LlvmTools {
+    type Output = Option<PathBuf>;
+    const ONLY_HOSTS: bool = true;
+
+    fn should_run(run: ShouldRun) -> ShouldRun {
+        run.path("llvm-tools")
+    }
+
+    fn make_run(run: RunConfig) {
+        run.builder.ensure(LlvmTools {
+            stage: run.builder.top_stage,
+            compiler: run.builder.compiler(run.builder.top_stage, run.target),
+            target: run.target,
+        });
+    }
+
+    fn run(self, builder: &Builder) -> Option<PathBuf> {
+        let compiler = self.compiler;
+        let host = compiler.host;
+
+        let stage = self.stage;
+        assert!(builder.config.extended);
+
+        builder.info(&format!("Dist LlvmTools stage{} ({})", stage, host));
+        let src = builder.src.join("src/llvm");
+        let name = pkgname(builder, "llvm-tools");
+
+        let tmp = tmpdir(builder);
+        let image = tmp.join("llvm-tools-image");
+        drop(fs::remove_dir_all(&image));
+        t!(fs::create_dir_all(&image.join("bin")));
+
+        // Prepare the image directory
+        for tool in LLVM_TOOLS {
+            let exe = builder
+                .llvm_out(host)
+                .join("bin")
+                .join(exe(tool, &compiler.host));
+            builder.install(&exe, &image.join("bin"), 0o755);
+        }
+
+        // Prepare the overlay
+        let overlay = tmp.join("llvm-tools-overlay");
+        drop(fs::remove_dir_all(&overlay));
+        builder.create_dir(&overlay);
+        builder.install(&src.join("README.txt"), &overlay, 0o644);
+        builder.install(&src.join("LICENSE.TXT"), &overlay, 0o644);
+
+        // Generate the installer tarball
+        let mut cmd = rust_installer(builder);
+        cmd.arg("generate")
+            .arg("--product-name=Rust")
+            .arg("--rel-manifest-dir=rustlib")
+            .arg("--success-message=llvm-tools-installed.")
+            .arg("--image-dir").arg(&image)
+            .arg("--work-dir").arg(&tmpdir(builder))
+            .arg("--output-dir").arg(&distdir(builder))
+            .arg("--non-installed-overlay").arg(&overlay)
+            .arg(format!("--package-name={}-{}", name, host))
+            .arg("--legacy-manifest-dirs=rustlib,cargo")
+            .arg("--component-name=llvm-tools");
+
+
+        builder.run(&mut cmd);
+        Some(distdir(builder).join(format!("{}-{}.tar.gz", name, host)))
+    }
+}

src/bootstrap/lib.rs

@@ -199,6 +199,14 @@ use flags::Subcommand;
 use cache::{Interned, INTERNER};
 use toolstate::ToolState;
 
+const LLVM_TOOLS: &[&str] = &[
+    "llvm-nm", // used to inspect binaries; it shows symbol names, their sizes and visibility
+    "llvm-objcopy", // used to transform ELFs into binary format which flashing tools consume
+    "llvm-objdump", // used to disassemble programs
+    "llvm-profdata", // used to inspect and merge files generated by profiles
+    "llvm-size", // prints the size of the linker sections of a program
+];
+
 /// A structure representing a Rust compiler.
 ///
 /// Each compiler has a `stage` that it is associated with and a `host` that
@@ -965,6 +973,27 @@ impl Build {
         self.package_vers(&self.release_num("rustfmt"))
     }
 
+    fn llvm_tools_vers(&self) -> String {
+        // japaric: should we use LLVM version here?
+        // let stdout = build_helper::output(
+        //     Command::new(self.llvm_out(self.config.build).join("build/bin/llvm-size"))
+        //         .arg("--version"),
+        // );
+
+        // for line in stdout.lines() {
+        //     if line.contains("LLVM version") {
+        //         if let Some(vers) = line.split_whitespace().nth(2) {
+        //             return vers.to_string();
+        //         }
+        //     }
+        // }
+
+        // panic!("The output of $LLVM_TOOL has changed; \
+        //         please fix `bootstrap::Build.llvm_tools_vers`");
+
+        self.rust_version()
+    }
+
     /// Returns the `version` string associated with this compiler for Rust
     /// itself.
     ///

src/bootstrap/native.rs

@@ -167,8 +167,22 @@ impl Step for Llvm {
         // which saves both memory during parallel links and overall disk space
         // for the tools.  We don't distribute any of those tools, so this is
         // just a local concern.  However, it doesn't work well everywhere.
-        if target.contains("linux-gnu") || target.contains("apple-darwin") {
-           cfg.define("LLVM_LINK_LLVM_DYLIB", "ON");
+        //
+        // If we are shipping llvm tools then we statically link them LLVM
+        if (target.contains("linux-gnu") || target.contains("apple-darwin")) &&
+            !builder.config.llvm_tools_enabled {
+                cfg.define("LLVM_LINK_LLVM_DYLIB", "ON");
+        }
+
+        // For distribution we want the LLVM tools to be *statically* linked to libstdc++
+        if builder.config.llvm_tools_enabled {
+            if !target.contains("windows") {
+                if target.contains("apple") {
+                    cfg.define("CMAKE_EXE_LINKER_FLAGS", "-static-libstdc++");
+                } else {
+                    cfg.define("CMAKE_EXE_LINKER_FLAGS", "-Wl,-Bsymbolic -static-libstdc++");
+                }
+            }
         }
 
         if target.contains("msvc") {