diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2d8980fcd1a0d..1f64e71d1a480 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -311,7 +311,7 @@ jobs: os: ubuntu-20.04-8core-32gb - name: dist-x86_64-apple env: - SCRIPT: "./x.py dist bootstrap --include-default-paths --host=x86_64-apple-darwin --target=x86_64-apple-darwin" + SCRIPT: "./x.py dist bootstrap bootstrap-shim --include-default-paths --host=x86_64-apple-darwin --target=x86_64-apple-darwin" RUST_CONFIGURE_ARGS: "--enable-full-tools --enable-sanitizers --enable-profiler --set rust.jemalloc --set llvm.ninja=false --set rust.lto=thin" RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 MACOSX_DEPLOYMENT_TARGET: 10.7 @@ -323,7 +323,7 @@ jobs: os: macos-latest - name: dist-apple-various env: - SCRIPT: "./x.py dist bootstrap --include-default-paths --host='' --target=aarch64-apple-ios,x86_64-apple-ios,aarch64-apple-ios-sim" + SCRIPT: "./x.py dist bootstrap bootstrap-shim --include-default-paths --host='' --target=aarch64-apple-ios,x86_64-apple-ios,aarch64-apple-ios-sim" RUST_CONFIGURE_ARGS: "--enable-sanitizers --enable-profiler --set rust.jemalloc --set llvm.ninja=false" RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 MACOSX_DEPLOYMENT_TARGET: 10.7 @@ -334,7 +334,7 @@ jobs: os: macos-latest - name: dist-x86_64-apple-alt env: - SCRIPT: "./x.py dist bootstrap --include-default-paths" + SCRIPT: "./x.py dist bootstrap bootstrap-shim --include-default-paths" RUST_CONFIGURE_ARGS: "--enable-extended --enable-profiler --set rust.jemalloc --set llvm.ninja=false" RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 MACOSX_DEPLOYMENT_TARGET: 10.7 @@ -367,7 +367,7 @@ jobs: os: macos-latest - name: dist-aarch64-apple env: - SCRIPT: "./x.py dist bootstrap --include-default-paths --stage 2" + SCRIPT: "./x.py dist bootstrap bootstrap-shim --include-default-paths --stage 2" RUST_CONFIGURE_ARGS: "--build=x86_64-apple-darwin --host=aarch64-apple-darwin --target=aarch64-apple-darwin --enable-full-tools --enable-sanitizers --enable-profiler --disable-docs --set rust.jemalloc --set llvm.ninja=false" RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 SELECT_XCODE: /Applications/Xcode_13.4.1.app @@ -442,19 +442,19 @@ jobs: - name: dist-x86_64-msvc env: RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-msvc --host=x86_64-pc-windows-msvc --target=x86_64-pc-windows-msvc --enable-full-tools --enable-profiler" - SCRIPT: PGO_HOST=x86_64-pc-windows-msvc python src/ci/stage-build.py python x.py dist bootstrap --include-default-paths + SCRIPT: PGO_HOST=x86_64-pc-windows-msvc python src/ci/stage-build.py python x.py dist bootstrap bootstrap-shim --include-default-paths DIST_REQUIRE_ALL_TOOLS: 1 os: windows-2019-8core-32gb - name: dist-i686-msvc env: RUST_CONFIGURE_ARGS: "--build=i686-pc-windows-msvc --host=i686-pc-windows-msvc --target=i686-pc-windows-msvc,i586-pc-windows-msvc --enable-full-tools --enable-profiler" - SCRIPT: python x.py dist bootstrap --include-default-paths + SCRIPT: python x.py dist bootstrap bootstrap-shim --include-default-paths DIST_REQUIRE_ALL_TOOLS: 1 os: windows-2019-8core-32gb - name: dist-aarch64-msvc env: RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-msvc --host=aarch64-pc-windows-msvc --enable-full-tools --enable-profiler" - SCRIPT: python x.py dist bootstrap --include-default-paths + SCRIPT: python x.py dist bootstrap bootstrap-shim --include-default-paths DIST_REQUIRE_ALL_TOOLS: 1 WINDOWS_SDK_20348_HACK: 1 os: windows-2019-8core-32gb @@ -462,13 +462,13 @@ jobs: env: RUST_CONFIGURE_ARGS: "--build=i686-pc-windows-gnu --enable-full-tools --enable-profiler" NO_DOWNLOAD_CI_LLVM: 1 - SCRIPT: python x.py dist bootstrap --include-default-paths + SCRIPT: python x.py dist bootstrap bootstrap-shim --include-default-paths CUSTOM_MINGW: 1 DIST_REQUIRE_ALL_TOOLS: 1 os: windows-2019-8core-32gb - name: dist-x86_64-mingw env: - SCRIPT: python x.py dist bootstrap --include-default-paths + SCRIPT: python x.py dist bootstrap bootstrap-shim --include-default-paths RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-gnu --enable-full-tools --enable-profiler" NO_DOWNLOAD_CI_LLVM: 1 CUSTOM_MINGW: 1 @@ -477,7 +477,7 @@ jobs: - name: dist-x86_64-msvc-alt env: RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-msvc --enable-extended --enable-profiler" - SCRIPT: python x.py dist bootstrap --include-default-paths + SCRIPT: python x.py dist bootstrap bootstrap-shim --include-default-paths os: windows-2019-8core-32gb timeout-minutes: 600 runs-on: "${{ matrix.os }}" diff --git a/src/bootstrap/Cargo.lock b/src/bootstrap/Cargo.lock index 9420c4fec5fa4..5f0f2c333d54e 100644 --- a/src/bootstrap/Cargo.lock +++ b/src/bootstrap/Cargo.lock @@ -49,6 +49,7 @@ dependencies = [ "cmake", "fd-lock", "filetime", + "getopts", "hex", "ignore", "is-terminal", @@ -317,6 +318,15 @@ dependencies = [ "version_check", ] +[[package]] +name = "getopts" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14dbbfd5c71d70241ecf9e6f13737f7b5ce823821063188d7e46c41d371eebd5" +dependencies = [ + "unicode-width", +] + [[package]] name = "globset" version = "0.4.8" @@ -778,6 +788,12 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d22af068fba1eb5edcb4aea19d382b2a3deb4c8f9d475c589b6ada9e0fd493ee" +[[package]] +name = "unicode-width" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" + [[package]] name = "version_check" version = "0.9.4" diff --git a/src/bootstrap/Cargo.toml b/src/bootstrap/Cargo.toml index 746c8dcfce0cd..170205f1d5c9e 100644 --- a/src/bootstrap/Cargo.toml +++ b/src/bootstrap/Cargo.toml @@ -14,6 +14,11 @@ name = "bootstrap" path = "bin/main.rs" test = false +[[bin]] +name = "bootstrap-shim" +path = "bin/bootstrap-shim.rs" +test = false + [[bin]] name = "rustc" path = "bin/rustc.rs" @@ -35,6 +40,7 @@ build_helper = { path = "../tools/build_helper" } cmake = "0.1.38" filetime = "0.2" cc = "1.0.69" +getopts = "0.2" libc = "0.2" hex = "0.4" object = { version = "0.29.0", default-features = false, features = ["archive", "coff", "read_core", "unaligned"] } diff --git a/src/bootstrap/bin/bootstrap-shim.rs b/src/bootstrap/bin/bootstrap-shim.rs new file mode 100644 index 0000000000000..d44ca8f994716 --- /dev/null +++ b/src/bootstrap/bin/bootstrap-shim.rs @@ -0,0 +1,79 @@ +use std::{ + env, io, + process::{self, Command, ExitStatus}, +}; + +use bootstrap::{Flags, MinimalConfig}; + +#[path = "../../../src/tools/x/src/main.rs"] +mod x; + +/// We are planning to exclude python logic from x script by executing bootstrap-shim +/// immediately. Since `find_and_run_available_bootstrap_script` executes x script first, +/// any changes on bootstrap will not be seen. To prevent this problem, in bootstrap-shim +/// we want to call the python script directly. +fn find_and_run_py_bootstrap_script() { + #[cfg(unix)] + fn exec_or_status(command: &mut Command) -> io::Result { + use std::os::unix::process::CommandExt; + Err(command.exec()) + } + + #[cfg(not(unix))] + fn exec_or_status(command: &mut Command) -> io::Result { + command.status() + } + + let current_path = match env::current_dir() { + Ok(dir) => dir, + Err(err) => { + eprintln!("Failed to get current directory: {err}"); + process::exit(1); + } + }; + + for dir in current_path.ancestors() { + let candidate = dir.join("x.py"); + if candidate.exists() { + let mut cmd: Command; + cmd = Command::new(x::python()); + cmd.arg(&candidate).args(env::args().skip(1)).current_dir(dir); + let result = exec_or_status(&mut cmd); + + match result { + Err(error) => { + eprintln!("Failed to invoke `{:?}`: {}", cmd, error); + } + Ok(status) => { + process::exit(status.code().unwrap_or(1)); + } + } + } + } +} + +fn main() { + let args = env::args().skip(1).collect::>(); + let flags = Flags::parse(&args); + + // If there are no untracked changes to bootstrap, download it from CI. + // Otherwise, build it from source. Use python to build to avoid duplicating the code between python and rust. + let config = MinimalConfig::parse(&flags, None); + let bootstrap_bin = if let Some(commit) = last_modified_bootstrap_commit(&config) { + config.download_bootstrap(&commit) + } else { + return find_and_run_py_bootstrap_script(); + }; + + let args: Vec<_> = std::env::args().skip(1).collect(); + println!("Running pre-compiled bootstrap binary"); + Command::new(bootstrap_bin).args(args).status().expect("failed to spawn bootstrap binairy"); +} + +fn last_modified_bootstrap_commit(config: &MinimalConfig) -> Option { + config.last_modified_commit( + &["src/bootstrap", "src/tools/build_helper", "src/tools/x"], + "download-bootstrap", + true, + ) +} diff --git a/src/bootstrap/bin/main.rs b/src/bootstrap/bin/main.rs index a80379e85c193..4d8b4f2f359e2 100644 --- a/src/bootstrap/bin/main.rs +++ b/src/bootstrap/bin/main.rs @@ -13,7 +13,7 @@ use bootstrap::{Build, Config, Subcommand, VERSION}; fn main() { let args = env::args().skip(1).collect::>(); - let config = Config::parse(&args); + let config = Config::parse(&args, None); #[cfg(all(any(unix, windows), not(target_os = "solaris")))] let mut build_lock; diff --git a/src/bootstrap/builder.rs b/src/bootstrap/builder.rs index 5c37fab547070..a07e050912c8e 100644 --- a/src/bootstrap/builder.rs +++ b/src/bootstrap/builder.rs @@ -703,7 +703,7 @@ impl<'a> Builder<'a> { check::Rls, check::RustAnalyzer, check::Rustfmt, - check::Bootstrap + check::Bootstrap, ), Kind::Test => describe!( crate::toolstate::ToolStateCheck, @@ -808,6 +808,7 @@ impl<'a> Builder<'a> { dist::LlvmTools, dist::RustDev, dist::Bootstrap, + dist::BootstrapShim, dist::Extended, // It seems that PlainSourceTarball somehow changes how some of the tools // perceive their dependencies (see #93033) which would invalidate fingerprints diff --git a/src/bootstrap/builder/tests.rs b/src/bootstrap/builder/tests.rs index c32fe59bbf069..08ea0a1e57c88 100644 --- a/src/bootstrap/builder/tests.rs +++ b/src/bootstrap/builder/tests.rs @@ -7,7 +7,7 @@ fn configure(cmd: &str, host: &[&str], target: &[&str]) -> Config { } fn configure_with_args(cmd: &[String], host: &[&str], target: &[&str]) -> Config { - let mut config = Config::parse(cmd); + let mut config = Config::parse(cmd, None); // don't save toolstates config.save_toolstates = None; config.dry_run = DryRun::SelfCheck; @@ -18,7 +18,7 @@ fn configure_with_args(cmd: &[String], host: &[&str], target: &[&str]) -> Config let submodule_build = Build::new(Config { // don't include LLVM, so CI doesn't require ninja/cmake to be installed rust_codegen_backends: vec![], - ..Config::parse(&["check".to_owned()]) + ..Config::parse(&["check".to_owned()], None) }); submodule_build.update_submodule(Path::new("src/doc/book")); submodule_build.update_submodule(Path::new("src/tools/rust-analyzer")); diff --git a/src/bootstrap/check.rs b/src/bootstrap/check.rs index b11be96cefe62..74a59907e93e0 100644 --- a/src/bootstrap/check.rs +++ b/src/bootstrap/check.rs @@ -448,6 +448,7 @@ tool_check_step!(Rls, "src/tools/rls", SourceType::InTree); tool_check_step!(Rustfmt, "src/tools/rustfmt", SourceType::InTree); tool_check_step!(MiroptTestTools, "src/tools/miropt-test-tools", SourceType::InTree); +// FIXME: currently these are marked as ToolRustc, but they should be ToolBootstrap instead to avoid having to build the compiler first tool_check_step!(Bootstrap, "src/bootstrap", SourceType::InTree, false); /// Cargo's output path for the standard library in a given stage, compiled diff --git a/src/bootstrap/config.rs b/src/bootstrap/config.rs index 710c8b52194b4..4e139a916db50 100644 --- a/src/bootstrap/config.rs +++ b/src/bootstrap/config.rs @@ -7,11 +7,9 @@ mod tests; use std::cell::{Cell, RefCell}; -use std::cmp; use std::collections::{HashMap, HashSet}; -use std::env; -use std::fmt; use std::fs; +use std::ops::{Deref, DerefMut}; use std::path::{Path, PathBuf}; use std::process::Command; use std::str::FromStr; @@ -22,12 +20,16 @@ use crate::cc_detect::{ndk_compiler, Language}; use crate::channel::{self, GitInfo}; pub use crate::flags::Subcommand; use crate::flags::{Color, Flags, Warnings}; +use crate::min_config::{get_toml, set_cfg_path_and_return_toml_cfg}; use crate::util::{exe, output, t}; +use crate::MinimalConfig; use once_cell::sync::OnceCell; use semver::Version; use serde::{Deserialize, Deserializer}; use serde_derive::Deserialize; +pub use crate::min_config::{DryRun, Stage0Metadata, TargetSelection, TargetSelectionList}; + macro_rules! check_ci_llvm { ($name:expr) => { assert!( @@ -38,17 +40,6 @@ macro_rules! check_ci_llvm { }; } -#[derive(Clone, Default)] -pub enum DryRun { - /// This isn't a dry run. - #[default] - Disabled, - /// This is a dry run enabled by bootstrap itself, so it can verify that no work is done. - SelfCheck, - /// This is a dry run enabled by the `--dry-run` flag. - UserSelected, -} - /// Global configuration for the entire build and/or bootstrap. /// /// This structure is parsed from `config.toml`, and some of the fields are inferred from `git` or build-time parameters. @@ -63,7 +54,6 @@ pub struct Config { pub ccache: Option, /// Call Build::ninja() instead of this. pub ninja_in_file: bool, - pub verbose: usize, pub submodules: Option, pub compiler_docs: bool, pub library_docs_private_items: bool, @@ -84,8 +74,6 @@ pub struct Config { pub json_output: bool, pub test_compare_mode: bool, pub color: Color, - pub patch_binaries_for_nix: bool, - pub stage0_metadata: Stage0Metadata, pub stdout_is_tty: bool, pub stderr_is_tty: bool, @@ -94,13 +82,9 @@ pub struct Config { pub stage: u32, pub keep_stage: Vec, pub keep_stage_std: Vec, - pub src: PathBuf, - /// defaults to `config.toml` - pub config: Option, pub jobs: Option, pub cmd: Subcommand, pub incremental: bool, - pub dry_run: DryRun, /// Arguments appearing after `--` to be forwarded to tools, /// e.g. `--fix-broken` or test arguments. pub free_args: Vec, @@ -183,7 +167,6 @@ pub struct Config { pub llvm_bolt_profile_generate: bool, pub llvm_bolt_profile_use: Option, - pub build: TargetSelection, pub hosts: Vec, pub targets: Vec, pub local_rebuild: bool, @@ -227,7 +210,6 @@ pub struct Config { pub reuse: Option, pub cargo_native_static: bool, pub configure_args: Vec, - pub out: PathBuf, pub rust_info: channel::GitInfo, // These are either the stage0 downloaded binaries or the locally installed ones. @@ -240,33 +222,24 @@ pub struct Config { pub initial_rustfmt: RefCell, pub paths: Vec, -} -#[derive(Default, Deserialize, Clone)] -pub struct Stage0Metadata { - pub compiler: CompilerMetadata, - pub config: Stage0Config, - pub checksums_sha256: HashMap, - pub rustfmt: Option, -} -#[derive(Default, Deserialize, Clone)] -pub struct CompilerMetadata { - pub date: String, - pub version: String, + #[cfg(test)] + pub minimal_config: MinimalConfig, + #[cfg(not(test))] + minimal_config: MinimalConfig, } -#[derive(Default, Deserialize, Clone)] -pub struct Stage0Config { - pub dist_server: String, - pub artifacts_server: String, - pub artifacts_with_llvm_assertions_server: String, - pub git_merge_commit_email: String, - pub nightly_branch: String, +impl Deref for Config { + type Target = MinimalConfig; + fn deref(&self) -> &Self::Target { + &self.minimal_config + } } -#[derive(Default, Deserialize, Clone)] -pub struct RustfmtMetadata { - pub date: String, - pub version: String, + +impl DerefMut for Config { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.minimal_config + } } #[derive(Clone, Debug)] @@ -373,83 +346,6 @@ impl std::str::FromStr for RustcLto { } } -#[derive(Copy, Clone, Default, PartialEq, Eq, PartialOrd, Ord, Hash)] -pub struct TargetSelection { - pub triple: Interned, - file: Option>, -} - -/// Newtype over `Vec` so we can implement custom parsing logic -#[derive(Clone, Default, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)] -pub struct TargetSelectionList(Vec); - -pub fn target_selection_list(s: &str) -> Result { - Ok(TargetSelectionList( - s.split(",").filter(|s| !s.is_empty()).map(TargetSelection::from_user).collect(), - )) -} - -impl TargetSelection { - pub fn from_user(selection: &str) -> Self { - let path = Path::new(selection); - - let (triple, file) = if path.exists() { - let triple = path - .file_stem() - .expect("Target specification file has no file stem") - .to_str() - .expect("Target specification file stem is not UTF-8"); - - (triple, Some(selection)) - } else { - (selection, None) - }; - - let triple = INTERNER.intern_str(triple); - let file = file.map(|f| INTERNER.intern_str(f)); - - Self { triple, file } - } - - pub fn rustc_target_arg(&self) -> &str { - self.file.as_ref().unwrap_or(&self.triple) - } - - pub fn contains(&self, needle: &str) -> bool { - self.triple.contains(needle) - } - - pub fn starts_with(&self, needle: &str) -> bool { - self.triple.starts_with(needle) - } - - pub fn ends_with(&self, needle: &str) -> bool { - self.triple.ends_with(needle) - } -} - -impl fmt::Display for TargetSelection { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}", self.triple)?; - if let Some(file) = self.file { - write!(f, "({})", file)?; - } - Ok(()) - } -} - -impl fmt::Debug for TargetSelection { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}", self) - } -} - -impl PartialEq<&str> for TargetSelection { - fn eq(&self, other: &&str) -> bool { - self.triple == *other - } -} - /// Per-target configuration stored in the global configuration structure. #[derive(Default, Clone)] pub struct Target { @@ -490,6 +386,7 @@ impl Target { target } } + /// Structure of the `config.toml` file that configuration is read from. /// /// This structure uses `Decodable` to automatically decode a TOML configuration @@ -542,8 +439,8 @@ macro_rules! define_config { $($field:ident: Option<$field_ty:ty> = $field_key:literal,)* }) => { $(#[$attr])* - struct $name { - $($field: Option<$field_ty>,)* + pub struct $name { + $(pub(crate) $field: Option<$field_ty>,)* } impl Merge for $name { @@ -625,7 +522,7 @@ macro_rules! define_config { define_config! { /// TOML representation of various global build decisions. - #[derive(Default)] + #[derive(Clone, Default)] struct Build { build: Option = "build", host: Option> = "host", @@ -731,7 +628,7 @@ define_config! { #[derive(Debug, Deserialize)] #[serde(untagged)] -enum StringOrBool { +pub(crate) enum StringOrBool { String(String), Bool(bool), } @@ -851,44 +748,47 @@ impl Config { config.stdout_is_tty = std::io::stdout().is_terminal(); config.stderr_is_tty = std::io::stderr().is_terminal(); - // set by build.rs - config.build = TargetSelection::from_user(&env!("BUILD_TRIPLE")); - - let manifest_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")); - // Undo `src/bootstrap` - config.src = manifest_dir.parent().unwrap().parent().unwrap().to_owned(); - config.out = PathBuf::from("build"); - config } - pub fn parse(args: &[String]) -> Config { - #[cfg(test)] - let get_toml = |_: &_| TomlConfig::default(); - #[cfg(not(test))] - let get_toml = |file: &Path| { - let contents = - t!(fs::read_to_string(file), format!("config file {} not found", file.display())); - // Deserialize to Value and then TomlConfig to prevent the Deserialize impl of - // TomlConfig and sub types to be monomorphized 5x by toml. - match toml::from_str(&contents) - .and_then(|table: toml::Value| TomlConfig::deserialize(table)) - { - Ok(table) => table, - Err(err) => { - eprintln!("failed to parse TOML configuration '{}': {}", file.display(), err); - crate::detail_exit(2); - } + pub(crate) fn test_args(&self) -> Vec<&str> { + let mut test_args = match self.cmd { + Subcommand::Test { ref test_args, .. } | Subcommand::Bench { ref test_args, .. } => { + test_args.iter().flat_map(|s| s.split_whitespace()).collect() } + _ => vec![], }; + test_args.extend(self.free_args.iter().map(|s| s.as_str())); + test_args + } - Self::parse_inner(args, get_toml) + pub(crate) fn args(&self) -> Vec<&str> { + let mut args = match self.cmd { + Subcommand::Run { ref args, .. } => { + args.iter().flat_map(|s| s.split_whitespace()).collect() + } + _ => vec![], + }; + args.extend(self.free_args.iter().map(|s| s.as_str())); + args } - fn parse_inner<'a>(args: &[String], get_toml: impl 'a + Fn(&Path) -> TomlConfig) -> Config { + pub fn parse(args: &[String], custom_toml_config: Option<&str>) -> Config { let mut flags = Flags::parse(&args); let mut config = Config::default_opts(); + let mut toml: TomlConfig = if let Some(custom_toml_config) = custom_toml_config { + toml::from_str(custom_toml_config).unwrap() + } else { + set_cfg_path_and_return_toml_cfg( + config.src.clone(), + flags.config.clone(), + &mut config.config, + ) + }; + + config.minimal_config = MinimalConfig::parse(&flags, toml.build.clone()); + // Set flags. config.paths = std::mem::take(&mut flags.paths); config.exclude = flags.exclude.into_iter().map(|path| TaskPath::parse(path)).collect(); @@ -899,7 +799,6 @@ impl Config { config.jobs = Some(threads_from_config(flags.jobs as u32)); config.cmd = flags.cmd; config.incremental = flags.incremental; - config.dry_run = if flags.dry_run { DryRun::UserSelected } else { DryRun::Disabled }; config.keep_stage = flags.keep_stage; config.keep_stage_std = flags.keep_stage_std; config.color = flags.color; @@ -916,81 +815,6 @@ impl Config { crate::detail_exit(1); } - // Infer the rest of the configuration. - - // Infer the source directory. This is non-trivial because we want to support a downloaded bootstrap binary, - // running on a completely machine from where it was compiled. - let mut cmd = Command::new("git"); - // NOTE: we cannot support running from outside the repository because the only path we have available - // is set at compile time, which can be wrong if bootstrap was downloaded from source. - // We still support running outside the repository if we find we aren't in a git directory. - cmd.arg("rev-parse").arg("--show-toplevel"); - // Discard stderr because we expect this to fail when building from a tarball. - let output = cmd - .stderr(std::process::Stdio::null()) - .output() - .ok() - .and_then(|output| if output.status.success() { Some(output) } else { None }); - if let Some(output) = output { - let git_root = String::from_utf8(output.stdout).unwrap(); - // We need to canonicalize this path to make sure it uses backslashes instead of forward slashes. - let git_root = PathBuf::from(git_root.trim()).canonicalize().unwrap(); - let s = git_root.to_str().unwrap(); - - // Bootstrap is quite bad at handling /? in front of paths - let src = match s.strip_prefix("\\\\?\\") { - Some(p) => PathBuf::from(p), - None => PathBuf::from(git_root), - }; - // If this doesn't have at least `stage0.json`, we guessed wrong. This can happen when, - // for example, the build directory is inside of another unrelated git directory. - // In that case keep the original `CARGO_MANIFEST_DIR` handling. - // - // NOTE: this implies that downloadable bootstrap isn't supported when the build directory is outside - // the source directory. We could fix that by setting a variable from all three of python, ./x, and x.ps1. - if src.join("src").join("stage0.json").exists() { - config.src = src; - } - } else { - // We're building from a tarball, not git sources. - // We don't support pre-downloaded bootstrap in this case. - } - - if cfg!(test) { - // Use the build directory of the original x.py invocation, so that we can set `initial_rustc` properly. - config.out = Path::new( - &env::var_os("CARGO_TARGET_DIR").expect("cargo test directly is not supported"), - ) - .parent() - .unwrap() - .to_path_buf(); - } - - let stage0_json = t!(std::fs::read(&config.src.join("src").join("stage0.json"))); - - config.stage0_metadata = t!(serde_json::from_slice::(&stage0_json)); - - // Read from `--config`, then `RUST_BOOTSTRAP_CONFIG`, then `./config.toml`, then `config.toml` in the root directory. - let toml_path = flags - .config - .clone() - .or_else(|| env::var_os("RUST_BOOTSTRAP_CONFIG").map(PathBuf::from)); - let using_default_path = toml_path.is_none(); - let mut toml_path = toml_path.unwrap_or_else(|| PathBuf::from("config.toml")); - if using_default_path && !toml_path.exists() { - toml_path = config.src.join(toml_path); - } - - // Give a hard error if `--config` or `RUST_BOOTSTRAP_CONFIG` are set to a missing path, - // but not if `config.toml` hasn't been created. - let mut toml = if !using_default_path || toml_path.exists() { - config.config = Some(toml_path.clone()); - get_toml(&toml_path) - } else { - config.config = None; - TomlConfig::default() - }; - if let Some(include) = &toml.profile { let mut include_path = config.src.clone(); include_path.push("src"); @@ -1068,7 +892,6 @@ impl Config { set(&mut config.full_bootstrap, build.full_bootstrap); set(&mut config.extended, build.extended); config.tools = build.tools; - set(&mut config.verbose, build.verbose); set(&mut config.sanitizers, build.sanitizers); set(&mut config.profiler, build.profiler); set(&mut config.cargo_native_static, build.cargo_native_static); @@ -1076,9 +899,6 @@ impl Config { set(&mut config.local_rebuild, build.local_rebuild); set(&mut config.print_step_timings, build.print_step_timings); set(&mut config.print_step_rusage, build.print_step_rusage); - set(&mut config.patch_binaries_for_nix, build.patch_binaries_for_nix); - - config.verbose = cmp::max(config.verbose, flags.verbose as usize); if let Some(install) = toml.install { config.prefix = install.prefix.map(PathBuf::from); @@ -1329,17 +1149,19 @@ impl Config { } if config.llvm_from_ci { - let triple = &config.build.triple; + let build_target_selection = config.build; let ci_llvm_bin = config.ci_llvm_root().join("bin"); let build_target = config .target_config .entry(config.build) - .or_insert_with(|| Target::from_triple(&triple)); + .or_insert_with(|| Target::from_triple(&build_target_selection.triple)); check_ci_llvm!(build_target.llvm_config); check_ci_llvm!(build_target.llvm_filecheck); - build_target.llvm_config = Some(ci_llvm_bin.join(exe("llvm-config", config.build))); - build_target.llvm_filecheck = Some(ci_llvm_bin.join(exe("FileCheck", config.build))); + build_target.llvm_config = + Some(ci_llvm_bin.join(exe("llvm-config", build_target_selection))); + build_target.llvm_filecheck = + Some(ci_llvm_bin.join(exe("FileCheck", build_target_selection))); } if let Some(t) = toml.dist { @@ -1448,44 +1270,6 @@ impl Config { config } - pub(crate) fn dry_run(&self) -> bool { - match self.dry_run { - DryRun::Disabled => false, - DryRun::SelfCheck | DryRun::UserSelected => true, - } - } - - /// A git invocation which runs inside the source directory. - /// - /// Use this rather than `Command::new("git")` in order to support out-of-tree builds. - pub(crate) fn git(&self) -> Command { - let mut git = Command::new("git"); - git.current_dir(&self.src); - git - } - - pub(crate) fn test_args(&self) -> Vec<&str> { - let mut test_args = match self.cmd { - Subcommand::Test { ref test_args, .. } | Subcommand::Bench { ref test_args, .. } => { - test_args.iter().flat_map(|s| s.split_whitespace()).collect() - } - _ => vec![], - }; - test_args.extend(self.free_args.iter().map(|s| s.as_str())); - test_args - } - - pub(crate) fn args(&self) -> Vec<&str> { - let mut args = match self.cmd { - Subcommand::Run { ref args, .. } => { - args.iter().flat_map(|s| s.split_whitespace()).collect() - } - _ => vec![], - }; - args.extend(self.free_args.iter().map(|s| s.as_str())); - args - } - /// Bootstrap embeds a version number into the name of shared libraries it uploads in CI. /// Return the version it would have used for the given commit. pub(crate) fn artifact_version_part(&self, commit: &str) -> String { @@ -1632,12 +1416,6 @@ impl Config { } } - pub fn verbose(&self, msg: &str) { - if self.verbose > 0 { - println!("{}", msg); - } - } - pub fn sanitizers_enabled(&self, target: TargetSelection) -> bool { self.target_config.get(&target).map(|t| t.sanitizers).flatten().unwrap_or(self.sanitizers) } @@ -1730,56 +1508,11 @@ impl Config { } }; - // Handle running from a directory other than the top level - let top_level = output(self.git().args(&["rev-parse", "--show-toplevel"])); - let top_level = top_level.trim_end(); - let compiler = format!("{top_level}/compiler/"); - let library = format!("{top_level}/library/"); - - // Look for a version to compare to based on the current commit. - // Only commits merged by bors will have CI artifacts. - let merge_base = output( - self.git() - .arg("rev-list") - .arg(format!("--author={}", self.stage0_metadata.config.git_merge_commit_email)) - .args(&["-n1", "--first-parent", "HEAD"]), - ); - let commit = merge_base.trim_end(); - if commit.is_empty() { - println!("error: could not find commit hash for downloading rustc"); - println!("help: maybe your repository history is too shallow?"); - println!("help: consider disabling `download-rustc`"); - println!("help: or fetch enough history to include one upstream commit"); - crate::detail_exit(1); - } - - // Warn if there were changes to the compiler or standard library since the ancestor commit. - let has_changes = !t!(self - .git() - .args(&["diff-index", "--quiet", &commit, "--", &compiler, &library]) - .status()) - .success(); - if has_changes { - if if_unchanged { - if self.verbose > 0 { - println!( - "warning: saw changes to compiler/ or library/ since {commit}; \ - ignoring `download-rustc`" - ); - } - return None; - } - println!( - "warning: `download-rustc` is enabled, but there are changes to \ - compiler/ or library/" - ); - } - - Some(commit.to_string()) + self.last_modified_commit(&["compiler", "library"], "download-rustc", if_unchanged) } } -fn set(field: &mut T, val: Option) { +pub(crate) fn set(field: &mut T, val: Option) { if let Some(v) = val { *field = v; } diff --git a/src/bootstrap/config/tests.rs b/src/bootstrap/config/tests.rs index d913ca295e29d..e79befc716263 100644 --- a/src/bootstrap/config/tests.rs +++ b/src/bootstrap/config/tests.rs @@ -1,13 +1,9 @@ -use super::{Config, Flags, TomlConfig}; +use super::{Config, Flags}; use clap::CommandFactory; use std::{env, path::Path}; -fn toml(config: &str) -> impl '_ + Fn(&Path) -> TomlConfig { - |&_| toml::from_str(config).unwrap() -} - fn parse(config: &str) -> Config { - Config::parse_inner(&["check".to_owned(), "--config=/does/not/exist".to_owned()], toml(config)) + Config::parse(&["check".to_owned(), "--config=/does/not/exist".to_owned()], Some(config)) } #[test] diff --git a/src/bootstrap/dist.rs b/src/bootstrap/dist.rs index 9cead7adc8c35..78a069af03c32 100644 --- a/src/bootstrap/dist.rs +++ b/src/bootstrap/dist.rs @@ -2314,6 +2314,40 @@ impl Step for Bootstrap { } } +/// Tarball intended for being able to run `rustup component add bootstrap-shim`. +/// Not stable, but user-facing. +#[derive(Clone, Debug, Eq, Hash, PartialEq)] +pub struct BootstrapShim { + pub target: TargetSelection, +} + +impl Step for BootstrapShim { + type Output = Option; + const DEFAULT: bool = false; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.alias("bootstrap-shim") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(BootstrapShim { target: run.target }); + } + + fn run(self, builder: &Builder<'_>) -> Option { + let target = self.target; + let mut tarball = Tarball::new(builder, "bootstrap-shim", &target.triple); + let bootstrap_outdir = &builder.bootstrap_out; + tarball.add_file( + bootstrap_outdir.join(exe("bootstrap-shim", target)), + tarball.image_dir().join("bin"), + 0o755, + ); + tarball.is_preview(true); + Some(tarball.generate()) + } +} + /// Tarball containing a prebuilt version of the build-manifest tool, intended to be used by the /// release process to avoid cloning the monorepo and building stuff. /// diff --git a/src/bootstrap/download.rs b/src/bootstrap/download.rs index 3e82a381a1b2d..d0efd836cb0f5 100644 --- a/src/bootstrap/download.rs +++ b/src/bootstrap/download.rs @@ -11,17 +11,17 @@ use once_cell::sync::OnceCell; use xz2::bufread::XzDecoder; use crate::{ - config::RustfmtMetadata, llvm::detect_llvm_sha, + min_config::RustfmtMetadata, t, - util::{check_run, exe, program_out_of_date, try_run}, - Config, + util::{check_run, exe, output, program_out_of_date, try_run}, + Config, MinimalConfig, }; static SHOULD_FIX_BINS_AND_DYLIBS: OnceCell = OnceCell::new(); /// Generic helpers that are useful anywhere in bootstrap. -impl Config { +impl MinimalConfig { pub fn is_verbose(&self) -> bool { self.verbose > 0 } @@ -329,6 +329,29 @@ impl Config { } return verified; } + + /// Bootstrap embeds a version number into the name of shared libraries it uploads in CI. + /// Return the version it would have used for the given commit. + /// + /// NOTE: this currently doesn't support tarballs, use `Config::artifact_version_part` if you need that support. + pub(crate) fn git_artifact_version_part(&self, commit: &str) -> String { + let (channel, version) = { + let mut channel = self.git(); + channel.arg("show").arg(format!("{}:src/ci/channel", commit)); + let channel = output(&mut channel); + let mut version = self.git(); + version.arg("show").arg(format!("{}:src/version", commit)); + let version = output(&mut version); + (channel.trim().to_owned(), version.trim().to_owned()) + }; + + match channel.as_str() { + "stable" => version, + "beta" => channel, + "nightly" => channel, + other => unreachable!("{:?} is not recognized as a valid channel", other), + } + } } enum DownloadSource { @@ -336,6 +359,110 @@ enum DownloadSource { Dist, } +impl MinimalConfig { + pub fn download_bootstrap(&self, commit: &str) -> PathBuf { + self.verbose(&format!("downloading bootstrap from CI (commit {commit})")); + let host = self.build.triple; + let bin_root = self.out.join(host).join("bootstrap"); + let stamp = bin_root.join(".bootstrap-stamp"); + let bootstrap_bin = bin_root.join("bin").join("bootstrap"); + + if !bootstrap_bin.exists() || program_out_of_date(&stamp, commit) { + let version = self.git_artifact_version_part(commit); + let filename = format!("bootstrap-{version}-{host}.tar.xz"); + self.download_component(DownloadSource::CI, filename, "bootstrap", commit, ""); + if self.should_fix_bins_and_dylibs() { + self.fix_bin_or_dylib(&bootstrap_bin); + } + t!(fs::write(stamp, commit)); + } + + bootstrap_bin + } + + fn download_component( + &self, + mode: DownloadSource, + filename: String, + prefix: &str, + key: &str, + destination: &str, + ) { + let cache_dst = self.out.join("cache"); + let cache_dir = cache_dst.join(key); + if !cache_dir.exists() { + t!(fs::create_dir_all(&cache_dir)); + } + + let bin_root = self.out.join(self.build.triple).join(destination); + let tarball = cache_dir.join(&filename); + let (base_url, url, should_verify) = match mode { + DownloadSource::CI => ( + self.stage0_metadata.config.artifacts_server.clone(), + format!("{key}/{filename}"), + false, + ), + DownloadSource::Dist => { + let dist_server = env::var("RUSTUP_DIST_SERVER") + .unwrap_or(self.stage0_metadata.config.dist_server.to_string()); + // NOTE: make `dist` part of the URL because that's how it's stored in src/stage0.json + (dist_server, format!("dist/{key}/{filename}"), true) + } + }; + + // For the beta compiler, put special effort into ensuring the checksums are valid. + // FIXME: maybe we should do this for download-rustc as well? but it would be a pain to update + // this on each and every nightly ... + let checksum = if should_verify { + let error = format!( + "src/stage0.json doesn't contain a checksum for {url}. \ + Pre-built artifacts might not be available for this \ + target at this time, see https://doc.rust-lang.org/nightly\ + /rustc/platform-support.html for more information." + ); + let sha256 = self.stage0_metadata.checksums_sha256.get(&url).expect(&error); + if tarball.exists() { + if self.verify(&tarball, sha256) { + self.unpack(&tarball, &bin_root, prefix); + return; + } else { + self.verbose(&format!( + "ignoring cached file {} due to failed verification", + tarball.display() + )); + self.remove(&tarball); + } + } + Some(sha256) + } else if tarball.exists() { + self.unpack(&tarball, &bin_root, prefix); + return; + } else { + None + }; + + let mut help_on_error = ""; + if destination == "ci-rustc" { + help_on_error = "error: failed to download pre-built rustc from CI + +note: old builds get deleted after a certain time +help: if trying to compile an old commit of rustc, disable `download-rustc` in config.toml: + +[rust] +download-rustc = false +"; + } + self.download_file(&format!("{base_url}/{url}"), &tarball, help_on_error); + if let Some(sha256) = checksum { + if !self.verify(&tarball, sha256) { + panic!("failed to verify {}", tarball.display()); + } + } + + self.unpack(&tarball, &bin_root, prefix); + } +} + /// Functions that are only ever called once, but named for clarify and to avoid thousand-line functions. impl Config { pub(crate) fn maybe_download_rustfmt(&self) -> Option { @@ -476,89 +603,7 @@ impl Config { /// Download a single component of a CI-built toolchain (not necessarily a published nightly). // NOTE: intentionally takes an owned string to avoid downloading multiple times by accident fn download_ci_component(&self, filename: String, prefix: &str, commit: &str) { - Self::download_component(self, DownloadSource::CI, filename, prefix, commit, "ci-rustc") - } - - fn download_component( - &self, - mode: DownloadSource, - filename: String, - prefix: &str, - key: &str, - destination: &str, - ) { - let cache_dst = self.out.join("cache"); - let cache_dir = cache_dst.join(key); - if !cache_dir.exists() { - t!(fs::create_dir_all(&cache_dir)); - } - - let bin_root = self.out.join(self.build.triple).join(destination); - let tarball = cache_dir.join(&filename); - let (base_url, url, should_verify) = match mode { - DownloadSource::CI => ( - self.stage0_metadata.config.artifacts_server.clone(), - format!("{key}/{filename}"), - false, - ), - DownloadSource::Dist => { - let dist_server = env::var("RUSTUP_DIST_SERVER") - .unwrap_or(self.stage0_metadata.config.dist_server.to_string()); - // NOTE: make `dist` part of the URL because that's how it's stored in src/stage0.json - (dist_server, format!("dist/{key}/{filename}"), true) - } - }; - - // For the beta compiler, put special effort into ensuring the checksums are valid. - // FIXME: maybe we should do this for download-rustc as well? but it would be a pain to update - // this on each and every nightly ... - let checksum = if should_verify { - let error = format!( - "src/stage0.json doesn't contain a checksum for {url}. \ - Pre-built artifacts might not be available for this \ - target at this time, see https://doc.rust-lang.org/nightly\ - /rustc/platform-support.html for more information." - ); - let sha256 = self.stage0_metadata.checksums_sha256.get(&url).expect(&error); - if tarball.exists() { - if self.verify(&tarball, sha256) { - self.unpack(&tarball, &bin_root, prefix); - return; - } else { - self.verbose(&format!( - "ignoring cached file {} due to failed verification", - tarball.display() - )); - self.remove(&tarball); - } - } - Some(sha256) - } else if tarball.exists() { - self.unpack(&tarball, &bin_root, prefix); - return; - } else { - None - }; - - let mut help_on_error = ""; - if destination == "ci-rustc" { - help_on_error = "error: failed to download pre-built rustc from CI - -note: old builds get deleted after a certain time -help: if trying to compile an old commit of rustc, disable `download-rustc` in config.toml: - -[rust] -download-rustc = false -"; - } - self.download_file(&format!("{base_url}/{url}"), &tarball, help_on_error); - if let Some(sha256) = checksum { - if !self.verify(&tarball, sha256) { - panic!("failed to verify {}", tarball.display()); - } - } - - self.unpack(&tarball, &bin_root, prefix); + self.download_component(DownloadSource::CI, filename, prefix, commit, "ci-rustc") } pub(crate) fn maybe_download_ci_llvm(&self) { @@ -571,10 +616,19 @@ download-rustc = false let key = format!("{}{}", llvm_sha, self.llvm_assertions); if program_out_of_date(&llvm_stamp, &key) && !self.dry_run() { self.download_ci_llvm(&llvm_sha); + if self.should_fix_bins_and_dylibs() { for entry in t!(fs::read_dir(llvm_root.join("bin"))) { self.fix_bin_or_dylib(&t!(entry).path()); } + + let llvm_lib = llvm_root.join("lib"); + for entry in t!(fs::read_dir(&llvm_lib)) { + let lib = t!(entry).path(); + if lib.extension().map_or(false, |ext| ext == "so") { + self.fix_bin_or_dylib(&lib); + } + } } // Update the timestamp of llvm-config to force rustc_llvm to be @@ -589,16 +643,6 @@ download-rustc = false let llvm_config = llvm_root.join("bin").join(exe("llvm-config", self.build)); t!(filetime::set_file_times(&llvm_config, now, now)); - if self.should_fix_bins_and_dylibs() { - let llvm_lib = llvm_root.join("lib"); - for entry in t!(fs::read_dir(&llvm_lib)) { - let lib = t!(entry).path(); - if lib.extension().map_or(false, |ext| ext == "so") { - self.fix_bin_or_dylib(&lib); - } - } - } - t!(fs::write(llvm_stamp, key)); } } diff --git a/src/bootstrap/flags.rs b/src/bootstrap/flags.rs index 6e0c0e01af863..35e7b9f5ccfa4 100644 --- a/src/bootstrap/flags.rs +++ b/src/bootstrap/flags.rs @@ -8,7 +8,8 @@ use std::path::{Path, PathBuf}; use clap::{CommandFactory, Parser, ValueEnum}; use crate::builder::{Builder, Kind}; -use crate::config::{target_selection_list, Config, TargetSelectionList}; +use crate::config::Config; +use crate::min_config::{target_selection_list, TargetSelectionList}; use crate::setup::Profile; use crate::{Build, DocTests}; @@ -182,7 +183,7 @@ impl Flags { HelpVerboseOnly::try_parse_from(it.clone()) { println!("note: updating submodules before printing available paths"); - let config = Config::parse(&[String::from("build")]); + let config = Config::parse(&[String::from("build")], None); let build = Build::new(config); let paths = Builder::get_help(&build, subcommand); if let Some(s) = paths { diff --git a/src/bootstrap/lib.rs b/src/bootstrap/lib.rs index 3756976dee062..dda09852439de 100644 --- a/src/bootstrap/lib.rs +++ b/src/bootstrap/lib.rs @@ -55,6 +55,7 @@ mod format; mod install; mod llvm; mod metadata; +mod min_config; mod render_tests; mod run; mod sanity; @@ -89,7 +90,9 @@ mod job { pub use crate::builder::PathSet; use crate::cache::{Interned, INTERNER}; pub use crate::config::Config; +pub use crate::flags::Flags; pub use crate::flags::Subcommand; +pub use crate::min_config::MinimalConfig; use termcolor::{ColorChoice, StandardStream, WriteColor}; const LLVM_TOOLS: &[&str] = &[ diff --git a/src/bootstrap/min_config.rs b/src/bootstrap/min_config.rs new file mode 100644 index 0000000000000..5ee031192ecc5 --- /dev/null +++ b/src/bootstrap/min_config.rs @@ -0,0 +1,443 @@ +use core::fmt; +use std::{ + cmp, + collections::HashMap, + env, fs, + path::{Path, PathBuf}, + process::Command, +}; + +use serde_derive::Deserialize; + +use crate::{ + cache::{Interned, INTERNER}, + config::{self, set}, + flags::Flags, + t, + util::output, +}; + +/// The bare minimum config, suitable for `bootstrap-shim`, but sharing code with the main `bootstrap` binary. +/// Unlike bootstrap itself, bootstrap-shim needs to be compatible across multiple versions of the Rust repo. +/// +/// FIXME: Update the following information once bootstrap-shim is default in shell scripts +/// +/// Once bootstrap-shim is used as default(today it's opt in) in shell scripts, DO NOT CHANGE this configuration +/// without the version bump for the pinned nightly version of bootstrap-shim. +#[derive(Default, Clone)] +pub struct MinimalConfig { + // Needed so we know where to store the unpacked bootstrap binary. + pub build: TargetSelection, + // Needed so we know where to load `src/stage0.json` + pub src: PathBuf, + // Needed so we know where to store the cache. + pub out: PathBuf, + pub patch_binaries_for_nix: bool, + // Needed to know which commit to download. + pub stage0_metadata: Stage0Metadata, + // This isn't currently used, but will eventually let people configure whether to download or build bootstrap. + pub config: Option, + // General need for verbose mode logging in `bootstrap-shim`. + pub verbose: usize, + // Needed for `bootstrap::download` module + pub dry_run: DryRun, +} + +#[derive(Default, Deserialize, Clone)] +pub struct Stage0Metadata { + pub compiler: CompilerMetadata, + pub config: Stage0Config, + pub checksums_sha256: HashMap, + pub rustfmt: Option, +} + +#[derive(Clone, Default, Deserialize)] +pub struct CompilerMetadata { + pub date: String, + pub version: String, +} + +#[derive(Default, Deserialize, Clone)] +pub struct Stage0Config { + pub dist_server: String, + pub artifacts_server: String, + pub artifacts_with_llvm_assertions_server: String, + pub git_merge_commit_email: String, + pub nightly_branch: String, +} + +#[derive(Default, Deserialize, Clone)] +pub struct RustfmtMetadata { + pub date: String, + pub version: String, +} + +impl MinimalConfig { + fn default_opts() -> Self { + let dry_run = DryRun::default(); + let config = None; + let verbose = 0; + let patch_binaries_for_nix = false; + let stage0_metadata = Stage0Metadata::default(); + + let manifest_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + // Undo `src/bootstrap` + let src = manifest_dir.parent().unwrap().parent().unwrap().to_owned(); + let out = PathBuf::from("build"); + + // set by build.rs + let build = TargetSelection::from_user(&env!("BUILD_TRIPLE")); + + MinimalConfig { + build, + src, + out, + config, + dry_run, + verbose, + patch_binaries_for_nix, + stage0_metadata, + } + } + + fn set_shared_fields_from_parent(&mut self, parent_build_config: config::Build) { + set(&mut self.verbose, parent_build_config.verbose); + set(&mut self.patch_binaries_for_nix, parent_build_config.patch_binaries_for_nix); + } + + pub fn parse(flags: &Flags, parent_build_config: Option) -> MinimalConfig { + let mut config = Self::default_opts(); + + if let Some(parent_build_config) = parent_build_config { + config.set_shared_fields_from_parent(parent_build_config); + }; + + if let Some(src) = src() { + config.src = src; + } + + set_config_output_dir(&mut config.out); + + config.stage0_metadata = deserialize_stage0_metadata(&config.src); + config.dry_run = if flags.dry_run { DryRun::UserSelected } else { DryRun::Disabled }; + + let toml: TomlConfig = set_cfg_path_and_return_toml_cfg( + config.src.clone(), + flags.config.clone(), + &mut config.config, + ); + + let build = toml.build.unwrap_or_default(); + if let Some(file_build) = build.build { + config.build = TargetSelection::from_user(&file_build); + } + + config.verbose = cmp::max(config.verbose, flags.verbose as usize); + + if config.dry_run() { + let dir = config.out.join("tmp-dry-run"); + t!(fs::create_dir_all(&dir)); + config.out = dir; + } + // NOTE: Bootstrap spawns various commands with different working directories. + // To avoid writing to random places on the file system, `config.out` needs to be an absolute path. + else if !config.out.is_absolute() { + // `canonicalize` requires the path to already exist. Use our vendored copy of `absolute` instead. + config.out = crate::util::absolute(&config.out); + } + + config.stage0_metadata = deserialize_stage0_metadata(&config.src); + + config + } +} + +impl MinimalConfig { + pub fn verbose(&self, msg: &str) { + if self.verbose > 0 { + println!("{}", msg); + } + } + + pub(crate) fn dry_run(&self) -> bool { + match self.dry_run { + DryRun::Disabled => false, + DryRun::SelfCheck | DryRun::UserSelected => true, + } + } + + /// A git invocation which runs inside the source directory. + /// + /// Use this rather than `Command::new("git")` in order to support out-of-tree builds. + pub(crate) fn git(&self) -> Command { + let mut git = Command::new("git"); + git.current_dir(&self.src); + git + } + + /// Returns the last commit in which any of `modified_paths` were changed, + /// or `None` if there are untracked changes in the working directory and `if_unchanged` is true. + pub fn last_modified_commit( + &self, + modified_paths: &[&str], + option_name: &str, + if_unchanged: bool, + ) -> Option { + // Handle running from a directory other than the top level + let top_level = output(self.git().args(&["rev-parse", "--show-toplevel"])); + let top_level = top_level.trim_end(); + + // Look for a version to compare to based on the current commit. + // Only commits merged by bors will have CI artifacts. + let merge_base = output( + self.git() + .arg("rev-list") + .arg(format!("--author={}", self.stage0_metadata.config.git_merge_commit_email)) + .args(&["-n1", "--first-parent", "HEAD"]), + ); + let commit = merge_base.trim_end(); + if commit.is_empty() { + println!("error: could not find commit hash for downloading components from CI"); + println!("help: maybe your repository history is too shallow?"); + println!("help: consider disabling `{option_name}`"); + println!("help: or fetch enough history to include one upstream commit"); + crate::detail_exit(1); + } + + // Warn if there were changes to the compiler or standard library since the ancestor commit. + let mut git = self.git(); + git.args(&["diff-index", "--quiet", &commit, "--"]); + + for path in modified_paths { + git.arg(format!("{top_level}/{path}")); + } + + let has_changes = !t!(git.status()).success(); + if has_changes { + if if_unchanged { + if self.verbose > 0 { + println!( + "warning: saw changes to one of {modified_paths:?} since {commit}; \ + ignoring `{option_name}`" + ); + } + return None; + } + println!( + "warning: `{option_name}` is enabled, but there are changes to one of {modified_paths:?}" + ); + } + + Some(commit.to_string()) + } +} + +#[cfg(test)] +/// Shared helper function to be used in `MinimalConfig::parse` and `bootstrap::config::Config::parse` +pub(crate) fn get_toml + Default>(_file: &Path) -> T { + T::default() +} +#[cfg(not(test))] +/// Shared helper function to be used in `MinimalConfig::parse` and `bootstrap::config::Config::parse` +pub(crate) fn get_toml + Default>(file: &Path) -> T { + let contents = + t!(fs::read_to_string(file), format!("config file {} not found", file.display())); + // Deserialize to Value and then TomlConfig to prevent the Deserialize impl of + // TomlConfig and sub types to be monomorphized 5x by toml. + match toml::from_str(&contents).and_then(|table: toml::Value| T::deserialize(table)) { + Ok(table) => table, + Err(err) => { + eprintln!("failed to parse TOML configuration '{}': {}", file.display(), err); + crate::detail_exit(2); + } + } +} + +/// Shared helper function to be used in `MinimalConfig::parse` and `bootstrap::config::Config::parse` +/// +/// Use the build directory of the original x.py invocation, so that we can set `initial_rustc` properly. +fn set_config_output_dir(output_path: &mut PathBuf) { + if cfg!(test) { + *output_path = Path::new( + &env::var_os("CARGO_TARGET_DIR").expect("cargo test directly is not supported"), + ) + .parent() + .unwrap() + .to_path_buf(); + } +} + +/// Shared helper function to be used in `MinimalConfig::parse` and `bootstrap::config::Config::parse` +pub(crate) fn set_cfg_path_and_return_toml_cfg + Default>( + src: PathBuf, + config_flag: Option, + cfg_path: &mut Option, +) -> T { + /// Read from `--config`, then `RUST_BOOTSTRAP_CONFIG`, then `./config.toml`, then `config.toml` in the root directory. + /// + /// Give a hard error if `--config` or `RUST_BOOTSTRAP_CONFIG` are set to a missing path, + /// but not if `config.toml` hasn't been created. + fn config_path(src: &PathBuf, config_flag: Option) -> Option { + let toml_path = + config_flag.or_else(|| env::var_os("RUST_BOOTSTRAP_CONFIG").map(PathBuf::from)); + let using_default_path = toml_path.is_none(); + let mut toml_path = toml_path.unwrap_or_else(|| PathBuf::from("config.toml")); + if using_default_path && !toml_path.exists() { + toml_path = src.join(toml_path); + } + + if !using_default_path || toml_path.exists() { Some(toml_path) } else { None } + } + + if let Some(toml_path) = config_path(&src, config_flag) { + *cfg_path = Some(toml_path.clone()); + get_toml(&toml_path) + } else { + *cfg_path = None; + T::default() + } +} + +/// Shared helper function to be used in `MinimalConfig::parse` and `bootstrap::config::Config::parse` +pub(crate) fn deserialize_stage0_metadata(stage0_metadata_path: &PathBuf) -> Stage0Metadata { + let stage0_json = t!(std::fs::read(stage0_metadata_path.join("src").join("stage0.json"))); + t!(serde_json::from_slice::(&stage0_json)) +} + +fn src() -> Option { + // Infer the source directory. This is non-trivial because we want to support a downloaded bootstrap binary, + // running on a completely machine from where it was compiled. + let mut cmd = Command::new("git"); + // NOTE: we cannot support running from outside the repository because the only path we have available + // is set at compile time, which can be wrong if bootstrap was downloaded from source. + // We still support running outside the repository if we find we aren't in a git directory. + cmd.arg("rev-parse").arg("--show-toplevel"); + // Discard stderr because we expect this to fail when building from a tarball. + let output = cmd + .stderr(std::process::Stdio::null()) + .output() + .ok() + .and_then(|output| if output.status.success() { Some(output) } else { None }); + if let Some(output) = output { + let git_root = String::from_utf8(output.stdout).unwrap(); + // We need to canonicalize this path to make sure it uses backslashes instead of forward slashes. + let git_root = PathBuf::from(git_root.trim()).canonicalize().unwrap(); + let s = git_root.to_str().unwrap(); + + // Bootstrap is quite bad at handling /? in front of paths + let src = match s.strip_prefix("\\\\?\\") { + Some(p) => PathBuf::from(p), + None => PathBuf::from(git_root), + }; + // If this doesn't have at least `stage0.json`, we guessed wrong. This can happen when, + // for example, the build directory is inside of another unrelated git directory. + // In that case keep the original `CARGO_MANIFEST_DIR` handling. + // + // NOTE: this implies that downloadable bootstrap isn't supported when the build directory is outside + // the source directory. We could fix that by setting a variable from all three of python, ./x, and x.ps1. + if src.join("src").join("stage0.json").exists() { Some(src) } else { None } + } else { + // We're building from a tarball, not git sources. + // We don't support pre-downloaded bootstrap in this case. + None + } +} + +#[derive(Clone, Default)] +pub enum DryRun { + /// This isn't a dry run. + #[default] + Disabled, + /// This is a dry run enabled by bootstrap itself, so it can verify that no work is done. + SelfCheck, + /// This is a dry run enabled by the `--dry-run` flag. + UserSelected, +} + +#[derive(Copy, Clone, Default, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct TargetSelection { + pub triple: Interned, + file: Option>, +} + +/// Newtype over `Vec` so we can implement custom parsing logic +#[derive(Clone, Default, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)] +pub struct TargetSelectionList(pub(crate) Vec); + +pub fn target_selection_list(s: &str) -> Result { + Ok(TargetSelectionList( + s.split(",").filter(|s| !s.is_empty()).map(TargetSelection::from_user).collect(), + )) +} + +impl TargetSelection { + pub fn from_user(selection: &str) -> Self { + let path = Path::new(selection); + + let (triple, file) = if path.exists() { + let triple = path + .file_stem() + .expect("Target specification file has no file stem") + .to_str() + .expect("Target specification file stem is not UTF-8"); + + (triple, Some(selection)) + } else { + (selection, None) + }; + + let triple = INTERNER.intern_str(triple); + let file = file.map(|f| INTERNER.intern_str(f)); + + Self { triple, file } + } + + pub fn rustc_target_arg(&self) -> &str { + self.file.as_ref().unwrap_or(&self.triple) + } + + pub fn contains(&self, needle: &str) -> bool { + self.triple.contains(needle) + } + + pub fn starts_with(&self, needle: &str) -> bool { + self.triple.starts_with(needle) + } + + pub fn ends_with(&self, needle: &str) -> bool { + self.triple.ends_with(needle) + } +} + +impl fmt::Display for TargetSelection { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.triple)?; + if let Some(file) = self.file { + write!(f, "({})", file)?; + } + Ok(()) + } +} + +impl fmt::Debug for TargetSelection { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self) + } +} + +impl PartialEq<&str> for TargetSelection { + fn eq(&self, other: &&str) -> bool { + self.triple == *other + } +} + +#[derive(Deserialize, Default)] +struct TomlConfig { + build: Option, +} + +/// TOML representation of various global build decisions. +#[derive(Deserialize, Default)] +struct Build { + build: Option, +} diff --git a/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile b/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile index 04fdb15f5acb2..4983a41fc3685 100644 --- a/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile +++ b/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile @@ -84,7 +84,7 @@ ENV RUST_CONFIGURE_ARGS \ ENV SCRIPT python3 ../src/ci/stage-build.py python3 ../x.py dist \ --host $HOSTS --target $HOSTS \ --include-default-paths \ - build-manifest bootstrap + build-manifest bootstrap bootstrap-shim ENV CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER=clang # This is the only builder which will create source tarballs diff --git a/src/ci/github-actions/ci.yml b/src/ci/github-actions/ci.yml index f81e740936ba5..ee993eb02f22b 100644 --- a/src/ci/github-actions/ci.yml +++ b/src/ci/github-actions/ci.yml @@ -489,7 +489,7 @@ jobs: - name: dist-x86_64-apple env: - SCRIPT: ./x.py dist bootstrap --include-default-paths --host=x86_64-apple-darwin --target=x86_64-apple-darwin + SCRIPT: ./x.py dist bootstrap bootstrap-shim --include-default-paths --host=x86_64-apple-darwin --target=x86_64-apple-darwin RUST_CONFIGURE_ARGS: --enable-full-tools --enable-sanitizers --enable-profiler --set rust.jemalloc --set llvm.ninja=false --set rust.lto=thin RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 MACOSX_DEPLOYMENT_TARGET: 10.7 @@ -502,7 +502,7 @@ jobs: - name: dist-apple-various env: - SCRIPT: ./x.py dist bootstrap --include-default-paths --host='' --target=aarch64-apple-ios,x86_64-apple-ios,aarch64-apple-ios-sim + SCRIPT: ./x.py dist bootstrap bootstrap-shim --include-default-paths --host='' --target=aarch64-apple-ios,x86_64-apple-ios,aarch64-apple-ios-sim RUST_CONFIGURE_ARGS: --enable-sanitizers --enable-profiler --set rust.jemalloc --set llvm.ninja=false RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 MACOSX_DEPLOYMENT_TARGET: 10.7 @@ -514,7 +514,7 @@ jobs: - name: dist-x86_64-apple-alt env: - SCRIPT: ./x.py dist bootstrap --include-default-paths + SCRIPT: ./x.py dist bootstrap bootstrap-shim --include-default-paths RUST_CONFIGURE_ARGS: --enable-extended --enable-profiler --set rust.jemalloc --set llvm.ninja=false RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 MACOSX_DEPLOYMENT_TARGET: 10.7 @@ -545,7 +545,7 @@ jobs: # This target only needs to support 11.0 and up as nothing else supports the hardware - name: dist-aarch64-apple env: - SCRIPT: ./x.py dist bootstrap --include-default-paths --stage 2 + SCRIPT: ./x.py dist bootstrap bootstrap-shim --include-default-paths --stage 2 RUST_CONFIGURE_ARGS: >- --build=x86_64-apple-darwin --host=aarch64-apple-darwin @@ -684,7 +684,7 @@ jobs: --target=x86_64-pc-windows-msvc --enable-full-tools --enable-profiler - SCRIPT: PGO_HOST=x86_64-pc-windows-msvc python src/ci/stage-build.py python x.py dist bootstrap --include-default-paths + SCRIPT: PGO_HOST=x86_64-pc-windows-msvc python src/ci/stage-build.py python x.py dist bootstrap bootstrap-shim --include-default-paths DIST_REQUIRE_ALL_TOOLS: 1 <<: *job-windows-8c @@ -696,7 +696,7 @@ jobs: --target=i686-pc-windows-msvc,i586-pc-windows-msvc --enable-full-tools --enable-profiler - SCRIPT: python x.py dist bootstrap --include-default-paths + SCRIPT: python x.py dist bootstrap bootstrap-shim --include-default-paths DIST_REQUIRE_ALL_TOOLS: 1 <<: *job-windows-8c @@ -707,7 +707,7 @@ jobs: --host=aarch64-pc-windows-msvc --enable-full-tools --enable-profiler - SCRIPT: python x.py dist bootstrap --include-default-paths + SCRIPT: python x.py dist bootstrap bootstrap-shim --include-default-paths DIST_REQUIRE_ALL_TOOLS: 1 # Hack around this SDK version, because it doesn't work with clang. # See https://github.com/rust-lang/rust/issues/88796 @@ -723,14 +723,14 @@ jobs: # We are intentionally allowing an old toolchain on this builder (and that's # incompatible with LLVM downloads today). NO_DOWNLOAD_CI_LLVM: 1 - SCRIPT: python x.py dist bootstrap --include-default-paths + SCRIPT: python x.py dist bootstrap bootstrap-shim --include-default-paths CUSTOM_MINGW: 1 DIST_REQUIRE_ALL_TOOLS: 1 <<: *job-windows-8c - name: dist-x86_64-mingw env: - SCRIPT: python x.py dist bootstrap --include-default-paths + SCRIPT: python x.py dist bootstrap bootstrap-shim --include-default-paths RUST_CONFIGURE_ARGS: >- --build=x86_64-pc-windows-gnu --enable-full-tools @@ -745,7 +745,7 @@ jobs: - name: dist-x86_64-msvc-alt env: RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-extended --enable-profiler - SCRIPT: python x.py dist bootstrap --include-default-paths + SCRIPT: python x.py dist bootstrap bootstrap-shim --include-default-paths <<: *job-windows-8c try: diff --git a/src/tools/build-manifest/src/main.rs b/src/tools/build-manifest/src/main.rs index 21dad9eb74aa9..ec0dbc3c3badc 100644 --- a/src/tools/build-manifest/src/main.rs +++ b/src/tools/build-manifest/src/main.rs @@ -183,7 +183,8 @@ static PKG_INSTALLERS: &[&str] = &["x86_64-apple-darwin", "aarch64-apple-darwin" static MINGW: &[&str] = &["i686-pc-windows-gnu", "x86_64-pc-windows-gnu"]; -static NIGHTLY_ONLY_COMPONENTS: &[PkgType] = &[PkgType::Miri, PkgType::JsonDocs]; +static NIGHTLY_ONLY_COMPONENTS: &[PkgType] = + &[PkgType::Miri, PkgType::JsonDocs, PkgType::BootstrapShim]; macro_rules! t { ($e:expr) => { @@ -414,7 +415,8 @@ impl Builder { | PkgType::Rustfmt | PkgType::LlvmTools | PkgType::RustAnalysis - | PkgType::JsonDocs => { + | PkgType::JsonDocs + | PkgType::BootstrapShim => { extensions.push(host_component(pkg)); } PkgType::RustcDev | PkgType::RustcDocs => { diff --git a/src/tools/build-manifest/src/versions.rs b/src/tools/build-manifest/src/versions.rs index 7a4c15d01eadc..9fd84b011c523 100644 --- a/src/tools/build-manifest/src/versions.rs +++ b/src/tools/build-manifest/src/versions.rs @@ -57,6 +57,7 @@ pkg_type! { LlvmTools = "llvm-tools"; preview = true, Miri = "miri"; preview = true, JsonDocs = "rust-docs-json"; preview = true, + BootstrapShim = "bootstrap-shim"; preview = true, } impl PkgType { @@ -92,6 +93,7 @@ impl PkgType { PkgType::ReproducibleArtifacts => true, PkgType::RustMingw => true, PkgType::RustAnalysis => true, + PkgType::BootstrapShim => true, } } @@ -115,6 +117,7 @@ impl PkgType { RustAnalyzer => HOSTS, Clippy => HOSTS, Miri => HOSTS, + BootstrapShim => HOSTS, Rustfmt => HOSTS, RustAnalysis => TARGETS, LlvmTools => TARGETS, diff --git a/src/tools/x/src/main.rs b/src/tools/x/src/main.rs index 5da8a2888ec83..bb506ca4ebf4f 100644 --- a/src/tools/x/src/main.rs +++ b/src/tools/x/src/main.rs @@ -19,7 +19,7 @@ const PYTHON: &str = "python"; const PYTHON2: &str = "python2"; const PYTHON3: &str = "python3"; -fn python() -> &'static str { +pub fn python() -> &'static str { let val = match env::var_os("PATH") { Some(val) => val, None => return PYTHON, @@ -98,16 +98,8 @@ fn handle_result(result: io::Result, cmd: Command) { } } -fn main() { - match env::args().skip(1).next().as_deref() { - Some("--wrapper-version") => { - let version = env!("CARGO_PKG_VERSION"); - println!("{}", version); - return; - } - _ => {} - } - let current = match env::current_dir() { +fn find_and_run_available_bootstrap_script() { + let current_path = match env::current_dir() { Ok(dir) => dir, Err(err) => { eprintln!("Failed to get current directory: {err}"); @@ -115,7 +107,7 @@ fn main() { } }; - for dir in current.ancestors() { + for dir in current_path.ancestors() { let candidate = dir.join("x.py"); if candidate.exists() { let shell_script_candidate = dir.join("x"); @@ -132,6 +124,20 @@ fn main() { handle_result(result, cmd); } } +} + +#[allow(dead_code)] +fn main() { + match env::args().skip(1).next().as_deref() { + Some("--wrapper-version") => { + let version = env!("CARGO_PKG_VERSION"); + println!("{}", version); + return; + } + _ => {} + } + + find_and_run_available_bootstrap_script(); eprintln!( "x.py not found. Please run inside of a checkout of `https://github.com/rust-lang/rust`."