Auto merge of #50000 - michaelwoerister:cross-lang-lto, r=alexcrichton

Add some groundwork for cross-language LTO.

Implements part of #49879:
- Adds a `-Z cross-lang-lto` flag to rustc
- Makes sure that bitcode is embedded in object files if the flag is set.

This should already allow for using cross language LTO for staticlibs (where one has to invoke the linker manually anyway). However, `rustc` will not try to enable LTO for its own linker invocations yet.

r? @alexcrichton

src/bootstrap/test.rs

@@ -1001,7 +1001,7 @@ impl Step for Compiletest {
 
             // Only pass correct values for these flags for the `run-make` suite as it
             // requires that a C++ compiler was configured which isn't always the case.
-            if !builder.config.dry_run && suite == "run-make-fulldeps" {
+            if !builder.config.dry_run && mode == "run-make" {
                 let llvm_components = output(Command::new(&llvm_config).arg("--components"));
                 let llvm_cxxflags = output(Command::new(&llvm_config).arg("--cxxflags"));
                 cmd.arg("--cc").arg(builder.cc(target))
@@ -1012,15 +1012,26 @@ impl Step for Compiletest {
                 if let Some(ar) = builder.ar(target) {
                     cmd.arg("--ar").arg(ar);
                 }
+
+                // Add the llvm/bin directory to PATH since it contains lots of
+                // useful, platform-independent tools
+                let llvm_bin_path = llvm_config.parent()
+                    .expect("Expected llvm-config to be contained in directory");
+                assert!(llvm_bin_path.is_dir());
+                let old_path = env::var_os("PATH").unwrap_or_default();
+                let new_path = env::join_paths(iter::once(llvm_bin_path.to_path_buf())
+                                               .chain(env::split_paths(&old_path)))
+                               .expect("");
+                cmd.env("PATH", new_path);
             }
         }
-        if suite == "run-make-fulldeps" && !builder.config.llvm_enabled {
+        if mode == "run-make" && !builder.config.llvm_enabled {
             builder.info(
                 &format!("Ignoring run-make test suite as they generally don't work without LLVM"));
             return;
         }
 
-        if suite != "run-make-fulldeps" {
+        if mode != "run-make" {
             cmd.arg("--cc").arg("")
                .arg("--cxx").arg("")
                .arg("--cflags").arg("")

src/librustc/session/config.rs

@@ -1306,6 +1306,8 @@ options! {DebuggingOptions, DebuggingSetter, basic_debugging_options,
         "tell the linker to strip debuginfo when building without debuginfo enabled."),
     share_generics: Option<bool> = (None, parse_opt_bool, [TRACKED],
           "make the current crate share its generic instantiations"),
+    cross_lang_lto: bool = (false, parse_bool, [TRACKED],
+          "generate build artifacts that are compatible with linker-based LTO."),
 }
 
 pub fn default_lib_output() -> CrateType {

src/librustc_trans/back/write.rs

@@ -314,7 +314,8 @@ impl ModuleConfig {
         self.inline_threshold = sess.opts.cg.inline_threshold;
         self.obj_is_bitcode = sess.target.target.options.obj_is_bitcode;
         let embed_bitcode = sess.target.target.options.embed_bitcode ||
-            sess.opts.debugging_opts.embed_bitcode;
+                            sess.opts.debugging_opts.embed_bitcode ||
+                            sess.opts.debugging_opts.cross_lang_lto;
         if embed_bitcode {
             match sess.opts.optimize {
                 config::OptLevel::No |
@@ -862,13 +863,18 @@ unsafe fn embed_bitcode(cgcx: &CodegenContext,
         "rustc.embedded.module\0".as_ptr() as *const _,
     );
     llvm::LLVMSetInitializer(llglobal, llconst);
-    let section = if cgcx.opts.target_triple.triple().contains("-ios") {
+
+    let is_apple = cgcx.opts.target_triple.triple().contains("-ios") ||
+                   cgcx.opts.target_triple.triple().contains("-darwin");
+
+    let section = if is_apple {
         "__LLVM,__bitcode\0"
     } else {
         ".llvmbc\0"
     };
     llvm::LLVMSetSection(llglobal, section.as_ptr() as *const _);
     llvm::LLVMRustSetLinkage(llglobal, llvm::Linkage::PrivateLinkage);
+    llvm::LLVMSetGlobalConstant(llglobal, llvm::True);
 
     let llconst = C_bytes_in_context(llcx, &[]);
     let llglobal = llvm::LLVMAddGlobal(
@@ -877,7 +883,7 @@ unsafe fn embed_bitcode(cgcx: &CodegenContext,
         "rustc.embedded.cmdline\0".as_ptr() as *const _,
     );
     llvm::LLVMSetInitializer(llglobal, llconst);
-    let section = if cgcx.opts.target_triple.triple().contains("-ios") {
+    let section = if  is_apple {
         "__LLVM,__cmdline\0"
     } else {
         ".llvmcmd\0"

src/test/run-make/cross-lang-lto/Makefile

@@ -0,0 +1,52 @@
+
+# min-llvm-version 4.0
+
+-include ../../run-make-fulldeps/tools.mk
+
+# This test makes sure that the expected .llvmbc sections for use by
+# linker-based LTO are available in object files when compiling with
+# -Z cross-lang-lto
+
+LLVMBC_SECTION_NAME=\\.llvmbc
+
+ifeq ($(UNAME),Darwin)
+	LLVMBC_SECTION_NAME=__LLVM,__bitcode
+endif
+
+
+OBJDUMP=llvm-objdump
+SECTION_HEADERS=$(OBJDUMP) -section-headers
+
+BUILD_LIB=$(RUSTC) lib.rs -Copt-level=2 -Z cross-lang-lto -Ccodegen-units=1
+
+BUILD_EXE=$(RUSTC) main.rs -Copt-level=2 -Z cross-lang-lto -Ccodegen-units=1 --emit=obj
+
+all: staticlib staticlib-fat-lto staticlib-thin-lto rlib exe cdylib rdylib
+
+staticlib: lib.rs
+	$(BUILD_LIB) --crate-type=staticlib
+	[ "$$($(SECTION_HEADERS) $(TMPDIR)/liblib.a | grep -c $(LLVMBC_SECTION_NAME))" -ne "0" ]
+
+staticlib-fat-lto: lib.rs
+	$(BUILD_LIB) --crate-type=staticlib -o $(TMPDIR)/liblib-fat-lto.a -Clto=fat
+	[ "$$($(SECTION_HEADERS) $(TMPDIR)/liblib-fat-lto.a | grep -c $(LLVMBC_SECTION_NAME))" -ne "0" ]
+
+staticlib-thin-lto: lib.rs
+	$(BUILD_LIB) --crate-type=staticlib -o $(TMPDIR)/liblib-thin-lto.a -Clto=thin
+	[ "$$($(SECTION_HEADERS) $(TMPDIR)/liblib-thin-lto.a | grep -c $(LLVMBC_SECTION_NAME))" -ne "0" ]
+
+rlib: lib.rs
+	$(BUILD_LIB) --crate-type=rlib
+	[ "$$($(SECTION_HEADERS) $(TMPDIR)/liblib.rlib | grep -c $(LLVMBC_SECTION_NAME))" -ne "0" ]
+
+cdylib: lib.rs
+	$(BUILD_LIB) --crate-type=cdylib --emit=obj -o $(TMPDIR)/cdylib.o
+	[ "$$($(SECTION_HEADERS) $(TMPDIR)/cdylib.o | grep -c $(LLVMBC_SECTION_NAME))" -ne "0" ]
+
+rdylib: lib.rs
+	$(BUILD_LIB) --crate-type=dylib --emit=obj -o $(TMPDIR)/rdylib.o
+	[ "$$($(SECTION_HEADERS) $(TMPDIR)/rdylib.o | grep -c $(LLVMBC_SECTION_NAME))" -ne "0" ]
+
+exe: lib.rs
+	$(BUILD_EXE) -o $(TMPDIR)/exe.o
+	[ "$$($(SECTION_HEADERS) $(TMPDIR)/exe.o | grep -c $(LLVMBC_SECTION_NAME))" -ne "0" ]

src/test/run-make/cross-lang-lto/lib.rs

@@ -0,0 +1,14 @@
+// Copyright 2018 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+#[no_mangle]
+pub extern "C" fn foo() {
+    println!("abc");
+}

src/test/run-make/cross-lang-lto/main.rs

@@ -0,0 +1,13 @@
+// Copyright 2018 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+fn main() {
+    println!("Hello World");
+}

src/tools/compiletest/src/header.rs

@@ -412,6 +412,15 @@ fn iter_header(testfile: &Path, cfg: Option<&str>, it: &mut FnMut(&str)) {
     if testfile.is_dir() {
         return;
     }
+
+    let comment = if testfile.to_string_lossy().ends_with(".rs") {
+        "//"
+    } else {
+        "#"
+    };
+
+    let comment_with_brace = comment.to_string() + "[";
+
     let rdr = BufReader::new(File::open(testfile).unwrap());
     for ln in rdr.lines() {
         // Assume that any directives will be found before the first
@@ -421,10 +430,11 @@ fn iter_header(testfile: &Path, cfg: Option<&str>, it: &mut FnMut(&str)) {
         let ln = ln.trim();
         if ln.starts_with("fn") || ln.starts_with("mod") {
             return;
-        } else if ln.starts_with("//[") {
+        } else if ln.starts_with(&comment_with_brace) {
             // A comment like `//[foo]` is specific to revision `foo`
             if let Some(close_brace) = ln.find(']') {
-                let lncfg = &ln[3..close_brace];
+                let open_brace = ln.find('[').unwrap();
+                let lncfg = &ln[open_brace + 1 .. close_brace];
                 let matches = match cfg {
                     Some(s) => s == &lncfg[..],
                     None => false,
@@ -433,11 +443,11 @@ fn iter_header(testfile: &Path, cfg: Option<&str>, it: &mut FnMut(&str)) {
                     it(ln[(close_brace + 1) ..].trim_left());
                 }
             } else {
-                panic!("malformed condition directive: expected `//[foo]`, found `{}`",
-                       ln)
+                panic!("malformed condition directive: expected `{}foo]`, found `{}`",
+                        comment_with_brace, ln)
             }
-        } else if ln.starts_with("//") {
-            it(ln[2..].trim_left());
+        } else if ln.starts_with(comment) {
+            it(ln[comment.len() ..].trim_left());
         }
     }
     return;

src/tools/compiletest/src/main.rs

@@ -610,7 +610,12 @@ pub fn is_test(file_name: &OsString) -> bool {
 }
 
 pub fn make_test(config: &Config, testpaths: &TestPaths) -> test::TestDescAndFn {
-    let early_props = EarlyProps::from_file(config, &testpaths.file);
+
+    let early_props = if config.mode == Mode::RunMake {
+        EarlyProps::from_file(config, &testpaths.file.join("Makefile"))
+    } else {
+        EarlyProps::from_file(config, &testpaths.file)
+    };
 
     // The `should-fail` annotation doesn't apply to pretty tests,
     // since we run the pretty printer across all tests by default.