Auto merge of #76349 - Mark-Simulacrum:dl-llvm, r=alexcrichton

Download LLVM from CI to bootstrap (linux-only to start)

This follows #76332, adding support for using CI-built LLVM rather than building it locally. This should essentially "just work," but is left off by default in this PR.

While we can support downloading LLVM for multiple host triples, this currently only downloads it for the build triple. That said, it should be possible to expand this relatively easily should multiple host triples be desired. Most people shouldn't be adjusting host/target triples though, so this should cover most use cases.

Currently this downloads LLVM for the last bors-authored commit in the `git log`. This is a bit suboptimal -- we want the last bors-authored commit that touched the llvm-project submodule in basically all cases. But for now this just adds an extra ~20 MB download when rebasing atop latest master. Once we have a submodule bump landing after #76332, we can fix this behavior to reduce downloads further.

config.toml.example

@@ -14,6 +14,21 @@
 # =============================================================================
 [llvm]
 
+# Whether to use Rust CI built LLVM instead of locally building it.
+#
+# Unless you're developing for a target where Rust CI doesn't build a compiler
+# toolchain or changing LLVM locally, you probably want to set this to true.
+#
+# It's currently false by default due to being newly added; please file bugs if
+# enabling this did not work for you on Linux (macOS and Windows support is
+# coming soon).
+#
+# We also currently only support this when building LLVM for the build triple.
+#
+# Note that many of the LLVM options are not currently supported for
+# downloading. Currently only the "assertions" option can be toggled.
+#download-ci-llvm = false
+
 # Indicates whether LLVM rebuild should be skipped when running bootstrap. If
 # this is `false` then the compiler's LLVM will be rebuilt whenever the built
 # version doesn't have the correct hash. If it is `true` then LLVM will never

src/bootstrap/bootstrap.py

@@ -14,8 +14,17 @@
 
 from time import time
 
-
-def get(url, path, verbose=False):
+def support_xz():
+    try:
+        with tempfile.NamedTemporaryFile(delete=False) as temp_file:
+            temp_path = temp_file.name
+        with tarfile.open(temp_path, "w:xz"):
+            pass
+        return True
+    except tarfile.CompressionError:
+        return False
+
+def get(url, path, verbose=False, do_verify=True):
     suffix = '.sha256'
     sha_url = url + suffix
     with tempfile.NamedTemporaryFile(delete=False) as temp_file:
@@ -24,19 +33,20 @@ def get(url, path, verbose=False):
         sha_path = sha_file.name
 
     try:
-        download(sha_path, sha_url, False, verbose)
-        if os.path.exists(path):
-            if verify(path, sha_path, False):
-                if verbose:
-                    print("using already-download file", path)
-                return
-            else:
-                if verbose:
-                    print("ignoring already-download file",
-                          path, "due to failed verification")
-                os.unlink(path)
+        if do_verify:
+            download(sha_path, sha_url, False, verbose)
+            if os.path.exists(path):
+                if verify(path, sha_path, False):
+                    if verbose:
+                        print("using already-download file", path)
+                    return
+                else:
+                    if verbose:
+                        print("ignoring already-download file",
+                            path, "due to failed verification")
+                    os.unlink(path)
         download(temp_path, url, True, verbose)
-        if not verify(temp_path, sha_path, verbose):
+        if do_verify and not verify(temp_path, sha_path, verbose):
             raise RuntimeError("failed verification")
         if verbose:
             print("moving {} to {}".format(temp_path, path))
@@ -365,16 +375,6 @@ def download_stage0(self):
         cargo_channel = self.cargo_channel
         rustfmt_channel = self.rustfmt_channel
 
-        def support_xz():
-            try:
-                with tempfile.NamedTemporaryFile(delete=False) as temp_file:
-                    temp_path = temp_file.name
-                with tarfile.open(temp_path, "w:xz"):
-                    pass
-                return True
-            except tarfile.CompressionError:
-                return False
-
         if self.rustc().startswith(self.bin_root()) and \
                 (not os.path.exists(self.rustc()) or
                  self.program_out_of_date(self.rustc_stamp())):
@@ -423,6 +423,19 @@ def support_xz():
                 with output(self.rustfmt_stamp()) as rustfmt_stamp:
                     rustfmt_stamp.write(self.date + self.rustfmt_channel)
 
+        if self.downloading_llvm():
+            llvm_sha = subprocess.check_output(["git", "log", "--author=bors",
+                "--format=%H", "-n1"]).decode(sys.getdefaultencoding()).strip()
+            llvm_assertions = self.get_toml('assertions', 'llvm') == 'true'
+            if self.program_out_of_date(self.llvm_stamp(), llvm_sha + str(llvm_assertions)):
+                self._download_ci_llvm(llvm_sha, llvm_assertions)
+                with output(self.llvm_stamp()) as llvm_stamp:
+                    llvm_stamp.write(self.date + llvm_sha + str(llvm_assertions))
+
+    def downloading_llvm(self):
+        opt = self.get_toml('download-ci-llvm', 'llvm')
+        return opt == "true"
+
     def _download_stage0_helper(self, filename, pattern, tarball_suffix, date=None):
         if date is None:
             date = self.date
@@ -437,6 +450,25 @@ def _download_stage0_helper(self, filename, pattern, tarball_suffix, date=None):
             get("{}/{}".format(url, filename), tarball, verbose=self.verbose)
         unpack(tarball, tarball_suffix, self.bin_root(), match=pattern, verbose=self.verbose)
 
+    def _download_ci_llvm(self, llvm_sha, llvm_assertions):
+        cache_prefix = "llvm-{}-{}".format(llvm_sha, llvm_assertions)
+        cache_dst = os.path.join(self.build_dir, "cache")
+        rustc_cache = os.path.join(cache_dst, cache_prefix)
+        if not os.path.exists(rustc_cache):
+            os.makedirs(rustc_cache)
+
+        url = "https://ci-artifacts.rust-lang.org/rustc-builds/{}".format(llvm_sha)
+        if llvm_assertions:
+            url = url.replace('rustc-builds', 'rustc-builds-alt')
+        tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
+        filename = "rust-dev-nightly-" + self.build + tarball_suffix
+        tarball = os.path.join(rustc_cache, filename)
+        if not os.path.exists(tarball):
+            get("{}/{}".format(url, filename), tarball, verbose=self.verbose, do_verify=False)
+        unpack(tarball, tarball_suffix, self.llvm_root(),
+                match="rust-dev",
+                verbose=self.verbose)
+
     def fix_bin_or_dylib(self, fname):
         """Modifies the interpreter section of 'fname' to fix the dynamic linker,
         or the RPATH section, to fix the dynamic library search path
@@ -558,6 +590,17 @@ def rustfmt_stamp(self):
         """
         return os.path.join(self.bin_root(), '.rustfmt-stamp')
 
+    def llvm_stamp(self):
+        """Return the path for .rustfmt-stamp
+
+        >>> rb = RustBuild()
+        >>> rb.build_dir = "build"
+        >>> rb.llvm_stamp() == os.path.join("build", "ci-llvm", ".llvm-stamp")
+        True
+        """
+        return os.path.join(self.llvm_root(), '.llvm-stamp')
+
+
     def program_out_of_date(self, stamp_path, extra=""):
         """Check if the given program stamp is out of date"""
         if not os.path.exists(stamp_path) or self.clean:
@@ -581,6 +624,22 @@ def bin_root(self):
         """
         return os.path.join(self.build_dir, self.build, "stage0")
 
+    def llvm_root(self):
+        """Return the CI LLVM root directory
+
+        >>> rb = RustBuild()
+        >>> rb.build_dir = "build"
+        >>> rb.llvm_root() == os.path.join("build", "ci-llvm")
+        True
+
+        When the 'build' property is given should be a nested directory:
+
+        >>> rb.build = "devel"
+        >>> rb.llvm_root() == os.path.join("build", "devel", "ci-llvm")
+        True
+        """
+        return os.path.join(self.build_dir, self.build, "ci-llvm")
+
     def get_toml(self, key, section=None):
         """Returns the value of the given key in config.toml, otherwise returns None
 

src/bootstrap/compile.rs

@@ -593,7 +593,7 @@ pub fn rustc_cargo_env(builder: &Builder<'_>, cargo: &mut Cargo, target: TargetS
             let file = compiler_file(builder, builder.cxx(target).unwrap(), target, "libstdc++.a");
             cargo.env("LLVM_STATIC_STDCPP", file);
         }
-        if builder.config.llvm_link_shared || builder.config.llvm_thin_lto {
+        if builder.config.llvm_link_shared {
             cargo.env("LLVM_LINK_SHARED", "1");
         }
         if builder.config.llvm_use_libcxx {

src/bootstrap/config.rs

@@ -15,9 +15,20 @@ use std::process;
 use crate::cache::{Interned, INTERNER};
 use crate::flags::Flags;
 pub use crate::flags::Subcommand;
+use crate::util::exe;
 use build_helper::t;
 use serde::Deserialize;
 
+macro_rules! check_ci_llvm {
+    ($name:expr) => {
+        assert!(
+            $name.is_none(),
+            "setting {} is incompatible with download-ci-llvm.",
+            stringify!($name)
+        );
+    };
+}
+
 /// Global configuration for the entire build and/or bootstrap.
 ///
 /// This structure is derived from a combination of both `config.toml` and
@@ -84,6 +95,7 @@ pub struct Config {
     pub llvm_version_suffix: Option<String>,
     pub llvm_use_linker: Option<String>,
     pub llvm_allow_old_toolchain: Option<bool>,
+    pub llvm_from_ci: bool,
 
     pub use_lld: bool,
     pub lld_enabled: bool,
@@ -344,6 +356,7 @@ struct Llvm {
     use_libcxx: Option<bool>,
     use_linker: Option<String>,
     allow_old_toolchain: Option<bool>,
+    download_ci_llvm: Option<bool>,
 }
 
 #[derive(Deserialize, Default, Clone)]
@@ -624,6 +637,43 @@ impl Config {
             set(&mut config.llvm_use_libcxx, llvm.use_libcxx);
             config.llvm_use_linker = llvm.use_linker.clone();
             config.llvm_allow_old_toolchain = llvm.allow_old_toolchain;
+            config.llvm_from_ci = llvm.download_ci_llvm.unwrap_or(false);
+
+            if config.llvm_from_ci {
+                // None of the LLVM options, except assertions, are supported
+                // when using downloaded LLVM. We could just ignore these but
+                // that's potentially confusing, so force them to not be
+                // explicitly set. The defaults and CI defaults don't
+                // necessarily match but forcing people to match (somewhat
+                // arbitrary) CI configuration locally seems bad/hard.
+                check_ci_llvm!(llvm.optimize);
+                check_ci_llvm!(llvm.thin_lto);
+                check_ci_llvm!(llvm.release_debuginfo);
+                check_ci_llvm!(llvm.link_shared);
+                check_ci_llvm!(llvm.static_libstdcpp);
+                check_ci_llvm!(llvm.targets);
+                check_ci_llvm!(llvm.experimental_targets);
+                check_ci_llvm!(llvm.link_jobs);
+                check_ci_llvm!(llvm.link_shared);
+                check_ci_llvm!(llvm.clang_cl);
+                check_ci_llvm!(llvm.version_suffix);
+                check_ci_llvm!(llvm.cflags);
+                check_ci_llvm!(llvm.cxxflags);
+                check_ci_llvm!(llvm.ldflags);
+                check_ci_llvm!(llvm.use_libcxx);
+                check_ci_llvm!(llvm.use_linker);
+                check_ci_llvm!(llvm.allow_old_toolchain);
+
+                // CI-built LLVM is shared
+                config.llvm_link_shared = true;
+            }
+
+            if config.llvm_thin_lto {
+                // If we're building with ThinLTO on, we want to link to LLVM
+                // shared, to avoid re-doing ThinLTO (which happens in the link
+                // step) with each stage.
+                config.llvm_link_shared = true;
+            }
         }
 
         if let Some(ref rust) = toml.rust {
@@ -706,6 +756,20 @@ impl Config {
             }
         }
 
+        if config.llvm_from_ci {
+            let triple = &config.build.triple;
+            let mut build_target = config
+                .target_config
+                .entry(config.build)
+                .or_insert_with(|| Target::from_triple(&triple));
+
+            check_ci_llvm!(build_target.llvm_config);
+            check_ci_llvm!(build_target.llvm_filecheck);
+            let ci_llvm_bin = config.out.join(&*config.build.triple).join("ci-llvm/bin");
+            build_target.llvm_config = Some(ci_llvm_bin.join(exe("llvm-config", config.build)));
+            build_target.llvm_filecheck = Some(ci_llvm_bin.join(exe("FileCheck", config.build)));
+        }
+
         if let Some(ref t) = toml.dist {
             config.dist_sign_folder = t.sign_folder.clone().map(PathBuf::from);
             config.dist_gpg_password_file = t.gpg_password_file.clone().map(PathBuf::from);

src/bootstrap/dist.rs

@@ -2382,26 +2382,32 @@ impl Step for HashSign {
 /// Note: This function does not yet support Windows, but we also don't support
 ///       linking LLVM tools dynamically on Windows yet.
 fn maybe_install_llvm(builder: &Builder<'_>, target: TargetSelection, dst_libdir: &Path) {
-    let src_libdir = builder.llvm_out(target).join("lib");
+    if !builder.config.llvm_link_shared {
+        // We do not need to copy LLVM files into the sysroot if it is not
+        // dynamically linked; it is already included into librustc_llvm
+        // statically.
+        return;
+    }
 
+    // On macOS for some reason the llvm-config binary behaves differently and
+    // and fails on missing .a files if run without --link-shared. If run with
+    // that option, it still fails, but because we only ship a libLLVM.dylib
+    // rather than libLLVM-11-rust-....dylib file.
+    //
+    // For now just don't use llvm-config here on macOS; that will fail to
+    // support CI-built LLVM, but until we work out the different behavior that
+    // is fine as it is off by default.
     if target.contains("apple-darwin") {
+        let src_libdir = builder.llvm_out(target).join("lib");
         let llvm_dylib_path = src_libdir.join("libLLVM.dylib");
         if llvm_dylib_path.exists() {
             builder.install(&llvm_dylib_path, dst_libdir, 0o644);
         }
-        return;
-    }
-
-    // Usually libLLVM.so is a symlink to something like libLLVM-6.0.so.
-    // Since tools link to the latter rather than the former, we have to
-    // follow the symlink to find out what to distribute.
-    let llvm_dylib_path = src_libdir.join("libLLVM.so");
-    if llvm_dylib_path.exists() {
-        let llvm_dylib_path = llvm_dylib_path.canonicalize().unwrap_or_else(|e| {
-            panic!("dist: Error calling canonicalize path `{}`: {}", llvm_dylib_path.display(), e);
-        });
-
-        builder.install(&llvm_dylib_path, dst_libdir, 0o644);
+    } else if let Ok(llvm_config) = crate::native::prebuilt_llvm_config(builder, target) {
+        let files = output(Command::new(llvm_config).arg("--libfiles"));
+        for file in files.lines() {
+            builder.install(Path::new(file), dst_libdir, 0o644);
+        }
     }
 }
 

src/bootstrap/lib.rs

@@ -611,6 +611,10 @@ impl Build {
     ///
     /// If no custom `llvm-config` was specified then Rust's llvm will be used.
     fn is_rust_llvm(&self, target: TargetSelection) -> bool {
+        if self.config.llvm_from_ci && target == self.config.build {
+            return true;
+        }
+
         match self.config.target_config.get(&target) {
             Some(ref c) => c.llvm_config.is_none(),
             None => true,