From f012427aa6abeb4013971108211d1bd6819b46f3 Mon Sep 17 00:00:00 2001 From: onur-ozkan Date: Tue, 18 Feb 2025 12:56:39 +0300 Subject: [PATCH 01/16] handle `ToolRustc` build stages automatically Signed-off-by: onur-ozkan --- src/bootstrap/src/core/build_steps/tool.rs | 23 +++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/src/bootstrap/src/core/build_steps/tool.rs b/src/bootstrap/src/core/build_steps/tool.rs index 75bfff340862a..b8ce92ac8af1a 100644 --- a/src/bootstrap/src/core/build_steps/tool.rs +++ b/src/bootstrap/src/core/build_steps/tool.rs @@ -76,11 +76,16 @@ impl Step for ToolBuild { /// This will build the specified tool with the specified `host` compiler in /// `stage` into the normal cargo output directory. fn run(self, builder: &Builder<'_>) -> PathBuf { - let compiler = self.compiler; let target = self.target; let mut tool = self.tool; let path = self.path; + let compiler = if self.mode == Mode::ToolRustc { + get_tool_rustc_compiler(builder, self.compiler) + } else { + self.compiler + }; + match self.mode { Mode::ToolRustc => { builder.ensure(compile::Std::new(compiler, compiler.host)); @@ -147,6 +152,9 @@ pub fn prepare_tool_cargo( source_type: SourceType, extra_features: &[String], ) -> CargoCommand { + let compiler = + if mode == Mode::ToolRustc { get_tool_rustc_compiler(builder, compiler) } else { compiler }; + let mut cargo = builder::Cargo::new(builder, compiler, mode, source_type, target, cmd_kind); let dir = builder.src.join(path); @@ -240,6 +248,19 @@ pub fn prepare_tool_cargo( cargo } +fn get_tool_rustc_compiler(builder: &Builder<'_>, target_compiler: Compiler) -> Compiler { + if builder.download_rustc() && target_compiler.stage == 1 { + // We already have the stage 1 compiler, we don't need to cut the stage. + builder.compiler(target_compiler.stage, builder.config.build) + } else { + // Similar to `compile::Assemble`, build with the previous stage's compiler. Otherwise + // we'd have stageN/bin/rustc and stageN/bin/$rustc_tool be effectively different stage + // compilers, which isn't what we want. Rustc tools should be linked in the same way as the + // compiler it's paired with, so it must be built with the previous stage compiler. + builder.compiler(target_compiler.stage.saturating_sub(1), builder.config.build) + } +} + /// Links a built tool binary with the given `name` from the build directory to the /// tools directory. fn copy_link_tool_bin( From 164f2408a215a33f3b9d638dde2944a4dbeae6ea Mon Sep 17 00:00:00 2001 From: onur-ozkan Date: Tue, 18 Feb 2025 12:59:20 +0300 Subject: [PATCH 02/16] pass target_compiler from Rustdoc Signed-off-by: onur-ozkan --- src/bootstrap/src/core/build_steps/tool.rs | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/src/bootstrap/src/core/build_steps/tool.rs b/src/bootstrap/src/core/build_steps/tool.rs index b8ce92ac8af1a..05c9ab29e057f 100644 --- a/src/bootstrap/src/core/build_steps/tool.rs +++ b/src/bootstrap/src/core/build_steps/tool.rs @@ -650,16 +650,7 @@ impl Step for Rustdoc { } } - let build_compiler = if builder.download_rustc() && target_compiler.stage == 1 { - // We already have the stage 1 compiler, we don't need to cut the stage. - builder.compiler(target_compiler.stage, builder.config.build) - } else { - // Similar to `compile::Assemble`, build with the previous stage's compiler. Otherwise - // we'd have stageN/bin/rustc and stageN/bin/rustdoc be effectively different stage - // compilers, which isn't what we want. Rustdoc should be linked in the same way as the - // rustc compiler it's paired with, so it must be built with the previous stage compiler. - builder.compiler(target_compiler.stage - 1, builder.config.build) - }; + let build_compiler = get_tool_rustc_compiler(builder, target_compiler); // When using `download-rustc` and a stage0 build_compiler, copying rustc doesn't actually // build stage0 libstd (because the libstd in sysroot has the wrong ABI). Explicitly build @@ -682,7 +673,7 @@ impl Step for Rustdoc { // NOTE: Never modify the rustflags here, it breaks the build cache for other tools! let mut cargo = prepare_tool_cargo( builder, - build_compiler, + target_compiler, Mode::ToolRustc, target, Kind::Build, From 7638abb5807cb77548092420af25c012a399568e Mon Sep 17 00:00:00 2001 From: onur-ozkan Date: Tue, 18 Feb 2025 13:05:36 +0300 Subject: [PATCH 03/16] bless tests Signed-off-by: onur-ozkan --- src/bootstrap/src/core/builder/tests.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/bootstrap/src/core/builder/tests.rs b/src/bootstrap/src/core/builder/tests.rs index 445b5dfbeab22..433e02299c148 100644 --- a/src/bootstrap/src/core/builder/tests.rs +++ b/src/bootstrap/src/core/builder/tests.rs @@ -525,6 +525,7 @@ mod dist { first(cache.all::()), &[ rustc!(TEST_TRIPLE_1 => TEST_TRIPLE_1, stage = 0), + rustc!(TEST_TRIPLE_1 => TEST_TRIPLE_2, stage = 0), rustc!(TEST_TRIPLE_1 => TEST_TRIPLE_2, stage = 1), ] ); @@ -796,7 +797,10 @@ mod dist { // stage minus 1 if --stage is not 0. Very confusing! assert_eq!( first(builder.cache.all::()), - &[tool::Rustdoc { compiler: Compiler { host: a, stage: 2 } },] + &[ + tool::Rustdoc { compiler: Compiler { host: a, stage: 1 } }, + tool::Rustdoc { compiler: Compiler { host: a, stage: 2 } }, + ] ); } From 72e67e87da600492930c5289cfb018441ca8c732 Mon Sep 17 00:00:00 2001 From: onur-ozkan Date: Tue, 18 Feb 2025 13:34:33 +0300 Subject: [PATCH 04/16] return more advanced type from `ToolBuild` Signed-off-by: onur-ozkan --- src/bootstrap/src/core/build_steps/tool.rs | 223 ++++++++++++--------- 1 file changed, 125 insertions(+), 98 deletions(-) diff --git a/src/bootstrap/src/core/build_steps/tool.rs b/src/bootstrap/src/core/build_steps/tool.rs index 05c9ab29e057f..8175c9213b97d 100644 --- a/src/bootstrap/src/core/build_steps/tool.rs +++ b/src/bootstrap/src/core/build_steps/tool.rs @@ -64,8 +64,15 @@ impl Builder<'_> { } } +#[derive(Clone)] +struct ToolBuildResult { + tool_path: PathBuf, + build_compiler: Compiler, + target_compiler: Compiler, +} + impl Step for ToolBuild { - type Output = PathBuf; + type Output = ToolBuildResult; fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { run.never() @@ -75,12 +82,13 @@ impl Step for ToolBuild { /// /// This will build the specified tool with the specified `host` compiler in /// `stage` into the normal cargo output directory. - fn run(self, builder: &Builder<'_>) -> PathBuf { + fn run(mut self, builder: &Builder<'_>) -> ToolBuildResult { let target = self.target; let mut tool = self.tool; let path = self.path; - let compiler = if self.mode == Mode::ToolRustc { + let target_compiler = self.compiler; + self.compiler = if self.mode == Mode::ToolRustc { get_tool_rustc_compiler(builder, self.compiler) } else { self.compiler @@ -88,17 +96,17 @@ impl Step for ToolBuild { match self.mode { Mode::ToolRustc => { - builder.ensure(compile::Std::new(compiler, compiler.host)); - builder.ensure(compile::Rustc::new(compiler, target)); + builder.ensure(compile::Std::new(self.compiler, self.compiler.host)); + builder.ensure(compile::Rustc::new(self.compiler, target)); } - Mode::ToolStd => builder.ensure(compile::Std::new(compiler, target)), + Mode::ToolStd => builder.ensure(compile::Std::new(self.compiler, target)), Mode::ToolBootstrap => {} // uses downloaded stage0 compiler libs _ => panic!("unexpected Mode for tool build"), } let mut cargo = prepare_tool_cargo( builder, - compiler, + self.compiler, self.mode, target, Kind::Build, @@ -136,7 +144,10 @@ impl Step for ToolBuild { if tool == "tidy" { tool = "rust-tidy"; } - copy_link_tool_bin(builder, self.compiler, self.target, self.mode, tool) + let tool_path = + copy_link_tool_bin(builder, target_compiler, self.target, self.mode, tool); + + ToolBuildResult { tool_path, build_compiler: self.compiler, target_compiler } } } } @@ -361,7 +372,7 @@ macro_rules! bootstrap_tool { extra_features: vec![], allow_features: concat!($($allow_features)*), cargo_args: vec![] - }) + }).tool_path } } )+ @@ -429,17 +440,19 @@ impl Step for OptimizedDist { // the tool requires it to be in place to run. builder.require_submodule("src/tools/rustc-perf", None); - builder.ensure(ToolBuild { - compiler: self.compiler, - target: self.target, - tool: "opt-dist", - mode: Mode::ToolBootstrap, - path: "src/tools/opt-dist", - source_type: SourceType::InTree, - extra_features: Vec::new(), - allow_features: "", - cargo_args: Vec::new(), - }) + builder + .ensure(ToolBuild { + compiler: self.compiler, + target: self.target, + tool: "opt-dist", + mode: Mode::ToolBootstrap, + path: "src/tools/opt-dist", + source_type: SourceType::InTree, + extra_features: Vec::new(), + allow_features: "", + cargo_args: Vec::new(), + }) + .tool_path } } @@ -483,7 +496,7 @@ impl Step for RustcPerf { // a CLI. cargo_args: vec!["-p".to_string(), "collector".to_string()], }; - let collector_bin = builder.ensure(tool.clone()); + let collector_bin = builder.ensure(tool.clone()).tool_path; // We also need to symlink the `rustc-fake` binary to the corresponding directory, // because `collector` expects it in the same directory. copy_link_tool_bin(builder, tool.compiler, tool.target, tool.mode, "rustc-fake"); @@ -533,17 +546,19 @@ impl Step for ErrorIndex { } fn run(self, builder: &Builder<'_>) -> PathBuf { - builder.ensure(ToolBuild { - compiler: self.compiler, - target: self.compiler.host, - tool: "error_index_generator", - mode: Mode::ToolRustc, - path: "src/tools/error_index_generator", - source_type: SourceType::InTree, - extra_features: Vec::new(), - allow_features: "", - cargo_args: Vec::new(), - }) + builder + .ensure(ToolBuild { + compiler: self.compiler, + target: self.compiler.host, + tool: "error_index_generator", + mode: Mode::ToolRustc, + path: "src/tools/error_index_generator", + source_type: SourceType::InTree, + extra_features: Vec::new(), + allow_features: "", + cargo_args: Vec::new(), + }) + .tool_path } } @@ -568,17 +583,19 @@ impl Step for RemoteTestServer { } fn run(self, builder: &Builder<'_>) -> PathBuf { - builder.ensure(ToolBuild { - compiler: self.compiler, - target: self.target, - tool: "remote-test-server", - mode: Mode::ToolStd, - path: "src/tools/remote-test-server", - source_type: SourceType::InTree, - extra_features: Vec::new(), - allow_features: "", - cargo_args: Vec::new(), - }) + builder + .ensure(ToolBuild { + compiler: self.compiler, + target: self.target, + tool: "remote-test-server", + mode: Mode::ToolStd, + path: "src/tools/remote-test-server", + source_type: SourceType::InTree, + extra_features: Vec::new(), + allow_features: "", + cargo_args: Vec::new(), + }) + .tool_path } } @@ -754,17 +771,19 @@ impl Step for Cargo { fn run(self, builder: &Builder<'_>) -> PathBuf { builder.build.require_submodule("src/tools/cargo", None); - builder.ensure(ToolBuild { - compiler: self.compiler, - target: self.target, - tool: "cargo", - mode: Mode::ToolRustc, - path: "src/tools/cargo", - source_type: SourceType::Submodule, - extra_features: Vec::new(), - allow_features: "", - cargo_args: Vec::new(), - }) + builder + .ensure(ToolBuild { + compiler: self.compiler, + target: self.target, + tool: "cargo", + mode: Mode::ToolRustc, + path: "src/tools/cargo", + source_type: SourceType::Submodule, + extra_features: Vec::new(), + allow_features: "", + cargo_args: Vec::new(), + }) + .tool_path } } @@ -797,17 +816,19 @@ impl Step for LldWrapper { let target = self.target_compiler.host; - let executable = builder.ensure(ToolBuild { - compiler: self.build_compiler, - target, - tool: "lld-wrapper", - mode: Mode::ToolStd, - path: "src/tools/lld-wrapper", - source_type: SourceType::InTree, - extra_features: Vec::new(), - allow_features: "", - cargo_args: Vec::new(), - }); + let executable = builder + .ensure(ToolBuild { + compiler: self.build_compiler, + target, + tool: "lld-wrapper", + mode: Mode::ToolStd, + path: "src/tools/lld-wrapper", + source_type: SourceType::InTree, + extra_features: Vec::new(), + allow_features: "", + cargo_args: Vec::new(), + }) + .tool_path; let libdir_bin = builder.sysroot_target_bindir(self.target_compiler, target); t!(fs::create_dir_all(&libdir_bin)); @@ -854,17 +875,19 @@ impl Step for RustAnalyzer { } fn run(self, builder: &Builder<'_>) -> PathBuf { - builder.ensure(ToolBuild { - compiler: self.compiler, - target: self.target, - tool: "rust-analyzer", - mode: Mode::ToolRustc, - path: "src/tools/rust-analyzer", - extra_features: vec!["in-rust-tree".to_owned()], - source_type: SourceType::InTree, - allow_features: RustAnalyzer::ALLOW_FEATURES, - cargo_args: Vec::new(), - }) + builder + .ensure(ToolBuild { + compiler: self.compiler, + target: self.target, + tool: "rust-analyzer", + mode: Mode::ToolRustc, + path: "src/tools/rust-analyzer", + extra_features: vec!["in-rust-tree".to_owned()], + source_type: SourceType::InTree, + allow_features: RustAnalyzer::ALLOW_FEATURES, + cargo_args: Vec::new(), + }) + .tool_path } } @@ -898,17 +921,19 @@ impl Step for RustAnalyzerProcMacroSrv { } fn run(self, builder: &Builder<'_>) -> Option { - let path = builder.ensure(ToolBuild { - compiler: self.compiler, - target: self.target, - tool: "rust-analyzer-proc-macro-srv", - mode: Mode::ToolRustc, - path: "src/tools/rust-analyzer/crates/proc-macro-srv-cli", - extra_features: vec!["in-rust-tree".to_owned()], - source_type: SourceType::InTree, - allow_features: RustAnalyzer::ALLOW_FEATURES, - cargo_args: Vec::new(), - }); + let path = builder + .ensure(ToolBuild { + compiler: self.compiler, + target: self.target, + tool: "rust-analyzer-proc-macro-srv", + mode: Mode::ToolRustc, + path: "src/tools/rust-analyzer/crates/proc-macro-srv-cli", + extra_features: vec!["in-rust-tree".to_owned()], + source_type: SourceType::InTree, + allow_features: RustAnalyzer::ALLOW_FEATURES, + cargo_args: Vec::new(), + }) + .tool_path; // Copy `rust-analyzer-proc-macro-srv` to `/libexec/` // so that r-a can use it. @@ -1146,17 +1171,19 @@ fn run_tool_build_step( path: &'static str, add_bins_to_sysroot: Option<&[&str]>, ) -> PathBuf { - let tool = builder.ensure(ToolBuild { - compiler, - target, - tool: tool_name, - mode: Mode::ToolRustc, - path, - extra_features: vec![], - source_type: SourceType::InTree, - allow_features: "", - cargo_args: vec![], - }); + let tool = builder + .ensure(ToolBuild { + compiler, + target, + tool: tool_name, + mode: Mode::ToolRustc, + path, + extra_features: vec![], + source_type: SourceType::InTree, + allow_features: "", + cargo_args: vec![], + }) + .tool_path; // FIXME: This should just be an if-let-chain, but those are unstable. if let Some(add_bins_to_sysroot) = From 8e011e5e672440e9fb45476385d6e9866b72ee12 Mon Sep 17 00:00:00 2001 From: onur-ozkan Date: Tue, 18 Feb 2025 14:01:22 +0300 Subject: [PATCH 05/16] migrate llvm-bitcode-linker to `ToolBuild` Signed-off-by: onur-ozkan --- src/bootstrap/src/core/build_steps/tool.rs | 78 ++++++++-------------- 1 file changed, 27 insertions(+), 51 deletions(-) diff --git a/src/bootstrap/src/core/build_steps/tool.rs b/src/bootstrap/src/core/build_steps/tool.rs index 8175c9213b97d..8bcb5770e39ff 100644 --- a/src/bootstrap/src/core/build_steps/tool.rs +++ b/src/bootstrap/src/core/build_steps/tool.rs @@ -145,7 +145,7 @@ impl Step for ToolBuild { tool = "rust-tidy"; } let tool_path = - copy_link_tool_bin(builder, target_compiler, self.target, self.mode, tool); + copy_link_tool_bin(builder, self.compiler, self.target, self.mode, tool); ToolBuildResult { tool_path, build_compiler: self.compiler, target_compiler } } @@ -976,50 +976,30 @@ impl Step for LlvmBitcodeLinker { instrument(level = "debug", name = "LlvmBitcodeLinker::run", skip_all) )] fn run(self, builder: &Builder<'_>) -> PathBuf { - let bin_name = "llvm-bitcode-linker"; - - // If enabled, use ci-rustc and skip building the in-tree compiler. - if !builder.download_rustc() { - builder.ensure(compile::Std::new(self.compiler, self.compiler.host)); - builder.ensure(compile::Rustc::new(self.compiler, self.target)); - } - - let cargo = prepare_tool_cargo( - builder, - self.compiler, - Mode::ToolRustc, - self.target, - Kind::Build, - "src/tools/llvm-bitcode-linker", - SourceType::InTree, - &self.extra_features, - ); - - let _guard = builder.msg_tool( - Kind::Build, - Mode::ToolRustc, - bin_name, - self.compiler.stage, - &self.compiler.host, - &self.target, - ); - - cargo.into_cmd().run(builder); - - let tool_out = builder - .cargo_out(self.compiler, Mode::ToolRustc, self.target) - .join(exe(bin_name, self.compiler.host)); + let ToolBuildResult { tool_path, build_compiler: _build_compiler, target_compiler } = + builder.ensure(ToolBuild { + compiler: self.compiler, + target: self.target, + tool: "llvm-bitcode-linker", + mode: Mode::ToolRustc, + path: "src/tools/llvm-bitcode-linker", + source_type: SourceType::InTree, + extra_features: self.extra_features, + allow_features: "", + cargo_args: Vec::new(), + }); - if self.compiler.stage > 0 { + if target_compiler.stage > 0 { let bindir_self_contained = builder - .sysroot(self.compiler) + .sysroot(target_compiler) .join(format!("lib/rustlib/{}/bin/self-contained", self.target.triple)); t!(fs::create_dir_all(&bindir_self_contained)); - let bin_destination = bindir_self_contained.join(exe(bin_name, self.compiler.host)); - builder.copy_link(&tool_out, &bin_destination); + let bin_destination = + bindir_self_contained.join(exe("llvm-bitcode-linker", target_compiler.host)); + builder.copy_link(&tool_path, &bin_destination); bin_destination } else { - tool_out + tool_path } } } @@ -1171,7 +1151,7 @@ fn run_tool_build_step( path: &'static str, add_bins_to_sysroot: Option<&[&str]>, ) -> PathBuf { - let tool = builder + let ToolBuildResult { tool_path, build_compiler: _build_compiler, target_compiler } = builder .ensure(ToolBuild { compiler, target, @@ -1182,28 +1162,24 @@ fn run_tool_build_step( source_type: SourceType::InTree, allow_features: "", cargo_args: vec![], - }) - .tool_path; + }); // FIXME: This should just be an if-let-chain, but those are unstable. if let Some(add_bins_to_sysroot) = - add_bins_to_sysroot.filter(|bins| !bins.is_empty() && compiler.stage > 0) + add_bins_to_sysroot.filter(|bins| !bins.is_empty() && target_compiler.stage > 0) { - let bindir = builder.sysroot(compiler).join("bin"); + let bindir = builder.sysroot(target_compiler).join("bin"); t!(fs::create_dir_all(&bindir)); - let tools_out = builder.cargo_out(compiler, Mode::ToolRustc, target); - for add_bin in add_bins_to_sysroot { - let bin_source = tools_out.join(exe(add_bin, target)); - let bin_destination = bindir.join(exe(add_bin, compiler.host)); - builder.copy_link(&bin_source, &bin_destination); + let bin_destination = bindir.join(exe(add_bin, target_compiler.host)); + builder.copy_link(&tool_path, &bin_destination); } // Return a path into the bin dir. - bindir.join(exe(tool_name, compiler.host)) + bindir.join(exe(tool_name, target_compiler.host)) } else { - tool + tool_path } } From a33843968ceae214b2a2c8699e0612bb582a3a4e Mon Sep 17 00:00:00 2001 From: onur-ozkan Date: Tue, 18 Feb 2025 15:06:23 +0300 Subject: [PATCH 06/16] return `ToolBuildResult` to utilize them from callers Signed-off-by: onur-ozkan --- src/bootstrap/src/core/build_steps/dist.rs | 14 +-- src/bootstrap/src/core/build_steps/test.rs | 8 +- src/bootstrap/src/core/build_steps/tool.rs | 118 +++++++++------------ src/bootstrap/src/core/builder/mod.rs | 6 +- src/bootstrap/src/core/builder/tests.rs | 5 +- 5 files changed, 66 insertions(+), 85 deletions(-) diff --git a/src/bootstrap/src/core/build_steps/dist.rs b/src/bootstrap/src/core/build_steps/dist.rs index 85b224771bb31..403aa48bad9b7 100644 --- a/src/bootstrap/src/core/build_steps/dist.rs +++ b/src/bootstrap/src/core/build_steps/dist.rs @@ -1180,7 +1180,7 @@ impl Step for Rls { let mut tarball = Tarball::new(builder, "rls", &target.triple); tarball.set_overlay(OverlayKind::Rls); tarball.is_preview(true); - tarball.add_file(rls, "bin", 0o755); + tarball.add_file(rls.tool_path, "bin", 0o755); tarball.add_legal_and_readme_to("share/doc/rls"); Some(tarball.generate()) } @@ -1268,8 +1268,8 @@ impl Step for Clippy { let mut tarball = Tarball::new(builder, "clippy", &target.triple); tarball.set_overlay(OverlayKind::Clippy); tarball.is_preview(true); - tarball.add_file(clippy, "bin", 0o755); - tarball.add_file(cargoclippy, "bin", 0o755); + tarball.add_file(clippy.tool_path, "bin", 0o755); + tarball.add_file(cargoclippy.tool_path, "bin", 0o755); tarball.add_legal_and_readme_to("share/doc/clippy"); Some(tarball.generate()) } @@ -1318,8 +1318,8 @@ impl Step for Miri { let mut tarball = Tarball::new(builder, "miri", &target.triple); tarball.set_overlay(OverlayKind::Miri); tarball.is_preview(true); - tarball.add_file(miri, "bin", 0o755); - tarball.add_file(cargomiri, "bin", 0o755); + tarball.add_file(miri.tool_path, "bin", 0o755); + tarball.add_file(cargomiri.tool_path, "bin", 0o755); tarball.add_legal_and_readme_to("share/doc/miri"); Some(tarball.generate()) } @@ -1449,8 +1449,8 @@ impl Step for Rustfmt { let mut tarball = Tarball::new(builder, "rustfmt", &target.triple); tarball.set_overlay(OverlayKind::Rustfmt); tarball.is_preview(true); - tarball.add_file(rustfmt, "bin", 0o755); - tarball.add_file(cargofmt, "bin", 0o755); + tarball.add_file(rustfmt.tool_path, "bin", 0o755); + tarball.add_file(cargofmt.tool_path, "bin", 0o755); tarball.add_legal_and_readme_to("share/doc/rustfmt"); Some(tarball.generate()) } diff --git a/src/bootstrap/src/core/build_steps/test.rs b/src/bootstrap/src/core/build_steps/test.rs index b3f4a7bad99c2..1820c13a861cd 100644 --- a/src/bootstrap/src/core/build_steps/test.rs +++ b/src/bootstrap/src/core/build_steps/test.rs @@ -427,7 +427,8 @@ impl Step for Rustfmt { let host = self.host; let compiler = builder.compiler(stage, host); - builder.ensure(tool::Rustfmt { compiler, target: self.host }); + let tool_result = builder.ensure(tool::Rustfmt { compiler, target: self.host }); + let compiler = tool_result.build_compiler; let mut cargo = tool::prepare_tool_cargo( builder, @@ -571,7 +572,7 @@ impl Step for Miri { // miri tests need to know about the stage sysroot cargo.env("MIRI_SYSROOT", &miri_sysroot); cargo.env("MIRI_HOST_SYSROOT", &host_sysroot); - cargo.env("MIRI", &miri); + cargo.env("MIRI", &miri.tool_path); // Set the target. cargo.env("MIRI_TEST_TARGET", target.rustc_target_arg()); @@ -743,7 +744,8 @@ impl Step for Clippy { let host = self.host; let compiler = builder.compiler(stage, host); - builder.ensure(tool::Clippy { compiler, target: self.host }); + let tool_result = builder.ensure(tool::Clippy { compiler, target: self.host }); + let compiler = tool_result.build_compiler; let mut cargo = tool::prepare_tool_cargo( builder, compiler, diff --git a/src/bootstrap/src/core/build_steps/tool.rs b/src/bootstrap/src/core/build_steps/tool.rs index 8bcb5770e39ff..cbb973428e2ca 100644 --- a/src/bootstrap/src/core/build_steps/tool.rs +++ b/src/bootstrap/src/core/build_steps/tool.rs @@ -65,10 +65,10 @@ impl Builder<'_> { } #[derive(Clone)] -struct ToolBuildResult { - tool_path: PathBuf, - build_compiler: Compiler, - target_compiler: Compiler, +pub struct ToolBuildResult { + pub tool_path: PathBuf, + pub build_compiler: Compiler, + pub target_compiler: Compiler, } impl Step for ToolBuild { @@ -114,10 +114,28 @@ impl Step for ToolBuild { self.source_type, &self.extra_features, ); + + if path.ends_with("/rustdoc") && + // rustdoc is performance sensitive, so apply LTO to it. + is_lto_stage(&self.compiler) + { + let lto = match builder.config.rust_lto { + RustcLto::Off => Some("off"), + RustcLto::Thin => Some("thin"), + RustcLto::Fat => Some("fat"), + RustcLto::ThinLocal => None, + }; + if let Some(lto) = lto { + cargo.env(cargo_profile_var("LTO", &builder.config), lto); + } + } + if !self.allow_features.is_empty() { cargo.allow_features(self.allow_features); } + cargo.args(self.cargo_args); + let _guard = builder.msg_tool( Kind::Build, self.mode, @@ -163,9 +181,6 @@ pub fn prepare_tool_cargo( source_type: SourceType, extra_features: &[String], ) -> CargoCommand { - let compiler = - if mode == Mode::ToolRustc { get_tool_rustc_compiler(builder, compiler) } else { compiler }; - let mut cargo = builder::Cargo::new(builder, compiler, mode, source_type, target, cmd_kind); let dir = builder.src.join(path); @@ -667,14 +682,6 @@ impl Step for Rustdoc { } } - let build_compiler = get_tool_rustc_compiler(builder, target_compiler); - - // When using `download-rustc` and a stage0 build_compiler, copying rustc doesn't actually - // build stage0 libstd (because the libstd in sysroot has the wrong ABI). Explicitly build - // it. - builder.ensure(compile::Std::new(build_compiler, target_compiler.host)); - builder.ensure(compile::Rustc::new(build_compiler, target_compiler.host)); - // The presence of `target_compiler` ensures that the necessary libraries (codegen backends, // compiler libraries, ...) are built. Rustdoc does not require the presence of any // libraries within sysroot_libdir (i.e., rustlib), though doctests may want it (since @@ -682,65 +689,39 @@ impl Step for Rustdoc { // libraries here. The intuition here is that If we've built a compiler, we should be able // to build rustdoc. // - let mut features = Vec::new(); + let mut extra_features = Vec::new(); if builder.config.jemalloc(target) { - features.push("jemalloc".to_string()); + extra_features.push("jemalloc".to_string()); } - // NOTE: Never modify the rustflags here, it breaks the build cache for other tools! - let mut cargo = prepare_tool_cargo( - builder, - target_compiler, - Mode::ToolRustc, - target, - Kind::Build, - "src/tools/rustdoc", - SourceType::InTree, - features.as_slice(), - ); - - // rustdoc is performance sensitive, so apply LTO to it. - if is_lto_stage(&build_compiler) { - let lto = match builder.config.rust_lto { - RustcLto::Off => Some("off"), - RustcLto::Thin => Some("thin"), - RustcLto::Fat => Some("fat"), - RustcLto::ThinLocal => None, - }; - if let Some(lto) = lto { - cargo.env(cargo_profile_var("LTO", &builder.config), lto); - } - } - - let _guard = builder.msg_tool( - Kind::Build, - Mode::ToolRustc, - "rustdoc", - build_compiler.stage, - &self.compiler.host, - &target, - ); - cargo.into_cmd().run(builder); - - // Cargo adds a number of paths to the dylib search path on windows, which results in - // the wrong rustdoc being executed. To avoid the conflicting rustdocs, we name the "tool" - // rustdoc a different name. - let tool_rustdoc = builder - .cargo_out(build_compiler, Mode::ToolRustc, target) - .join(exe("rustdoc_tool_binary", target_compiler.host)); + let ToolBuildResult { tool_path, build_compiler: _build_compiler, target_compiler } = + builder.ensure(ToolBuild { + compiler: target_compiler, + target, + // Cargo adds a number of paths to the dylib search path on windows, which results in + // the wrong rustdoc being executed. To avoid the conflicting rustdocs, we name the "tool" + // rustdoc a different name. + tool: "rustdoc_tool_binary", + mode: Mode::ToolRustc, + path: "src/tools/rustdoc", + source_type: SourceType::InTree, + extra_features, + allow_features: "", + cargo_args: Vec::new(), + }); // don't create a stage0-sysroot/bin directory. if target_compiler.stage > 0 { if builder.config.rust_debuginfo_level_tools == DebuginfoLevel::None { // Due to LTO a lot of debug info from C++ dependencies such as jemalloc can make it into // our final binaries - compile::strip_debug(builder, target, &tool_rustdoc); + compile::strip_debug(builder, target, &tool_path); } let bin_rustdoc = bin_rustdoc(); - builder.copy_link(&tool_rustdoc, &bin_rustdoc); + builder.copy_link(&tool_path, &bin_rustdoc); bin_rustdoc } else { - tool_rustdoc + tool_path } } } @@ -1084,7 +1065,7 @@ macro_rules! tool_extended { } impl Step for $name { - type Output = PathBuf; + type Output = ToolBuildResult; const DEFAULT: bool = true; // Overridden by `should_run_tool_build_step` const ONLY_HOSTS: bool = true; @@ -1104,7 +1085,7 @@ macro_rules! tool_extended { }); } - fn run(self, builder: &Builder<'_>) -> PathBuf { + fn run(self, builder: &Builder<'_>) -> ToolBuildResult { let Self { compiler, target } = self; run_tool_build_step( builder, @@ -1150,9 +1131,9 @@ fn run_tool_build_step( tool_name: &'static str, path: &'static str, add_bins_to_sysroot: Option<&[&str]>, -) -> PathBuf { - let ToolBuildResult { tool_path, build_compiler: _build_compiler, target_compiler } = builder - .ensure(ToolBuild { +) -> ToolBuildResult { + let ToolBuildResult { tool_path, build_compiler, target_compiler } = + builder.ensure(ToolBuild { compiler, target, tool: tool_name, @@ -1177,9 +1158,10 @@ fn run_tool_build_step( } // Return a path into the bin dir. - bindir.join(exe(tool_name, target_compiler.host)) + let path = bindir.join(exe(tool_name, target_compiler.host)); + ToolBuildResult { tool_path: path, build_compiler, target_compiler } } else { - tool_path + ToolBuildResult { tool_path, build_compiler, target_compiler } } } diff --git a/src/bootstrap/src/core/builder/mod.rs b/src/bootstrap/src/core/builder/mod.rs index daef8fa3c8a34..b7820acc7e8b3 100644 --- a/src/bootstrap/src/core/builder/mod.rs +++ b/src/bootstrap/src/core/builder/mod.rs @@ -1415,7 +1415,7 @@ impl<'a> Builder<'a> { let mut dylib_path = helpers::dylib_path(); dylib_path.insert(0, self.sysroot(run_compiler).join("lib")); - let mut cmd = command(cargo_clippy); + let mut cmd = command(cargo_clippy.tool_path); cmd.env(helpers::dylib_path_var(), env::join_paths(&dylib_path).unwrap()); cmd.env("CARGO", &self.initial_cargo); cmd @@ -1430,8 +1430,8 @@ impl<'a> Builder<'a> { let cargo_miri = self.ensure(tool::CargoMiri { compiler: build_compiler, target: self.build.build }); // Invoke cargo-miri, make sure it can find miri and cargo. - let mut cmd = command(cargo_miri); - cmd.env("MIRI", &miri); + let mut cmd = command(cargo_miri.tool_path); + cmd.env("MIRI", &miri.tool_path); cmd.env("CARGO", &self.initial_cargo); // Need to add the `run_compiler` libs. Those are the libs produces *by* `build_compiler`, // so they match the Miri we just built. However this means they are actually living one diff --git a/src/bootstrap/src/core/builder/tests.rs b/src/bootstrap/src/core/builder/tests.rs index 433e02299c148..a828a844f8103 100644 --- a/src/bootstrap/src/core/builder/tests.rs +++ b/src/bootstrap/src/core/builder/tests.rs @@ -797,10 +797,7 @@ mod dist { // stage minus 1 if --stage is not 0. Very confusing! assert_eq!( first(builder.cache.all::()), - &[ - tool::Rustdoc { compiler: Compiler { host: a, stage: 1 } }, - tool::Rustdoc { compiler: Compiler { host: a, stage: 2 } }, - ] + &[tool::Rustdoc { compiler: Compiler { host: a, stage: 2 } },] ); } From 6cf650fce7c9a599b18df72b1bac202a19eb81cd Mon Sep 17 00:00:00 2001 From: onur-ozkan Date: Tue, 18 Feb 2025 16:32:13 +0300 Subject: [PATCH 07/16] remove manually handled stage offs Signed-off-by: onur-ozkan --- src/bootstrap/src/core/build_steps/run.rs | 6 +----- src/bootstrap/src/core/build_steps/test.rs | 25 +++++----------------- src/bootstrap/src/core/build_steps/tool.rs | 17 ++++++--------- src/bootstrap/src/core/builder/mod.rs | 25 ++++++++++------------ 4 files changed, 24 insertions(+), 49 deletions(-) diff --git a/src/bootstrap/src/core/build_steps/run.rs b/src/bootstrap/src/core/build_steps/run.rs index 2b17e02cae5a4..fea8232296eca 100644 --- a/src/bootstrap/src/core/build_steps/run.rs +++ b/src/bootstrap/src/core/build_steps/run.rs @@ -126,11 +126,7 @@ impl Step for Miri { // This compiler runs on the host, we'll just use it for the target. let target_compiler = builder.compiler(stage, host); - // Similar to `compile::Assemble`, build with the previous stage's compiler. Otherwise - // we'd have stageN/bin/rustc and stageN/bin/rustdoc be effectively different stage - // compilers, which isn't what we want. Rustdoc should be linked in the same way as the - // rustc compiler it's paired with, so it must be built with the previous stage compiler. - let host_compiler = builder.compiler(stage - 1, host); + let host_compiler = tool::get_tool_rustc_compiler(builder, target_compiler); // Get a target sysroot for Miri. let miri_sysroot = test::Miri::build_miri_sysroot(builder, target_compiler, target); diff --git a/src/bootstrap/src/core/build_steps/test.rs b/src/bootstrap/src/core/build_steps/test.rs index 1820c13a861cd..e871a40731e34 100644 --- a/src/bootstrap/src/core/build_steps/test.rs +++ b/src/bootstrap/src/core/build_steps/test.rs @@ -523,16 +523,11 @@ impl Step for Miri { // This compiler runs on the host, we'll just use it for the target. let target_compiler = builder.compiler(stage, host); - // Similar to `compile::Assemble`, build with the previous stage's compiler. Otherwise - // we'd have stageN/bin/rustc and stageN/bin/rustdoc be effectively different stage - // compilers, which isn't what we want. Rustdoc should be linked in the same way as the - // rustc compiler it's paired with, so it must be built with the previous stage compiler. - let host_compiler = builder.compiler(stage - 1, host); // Build our tools. - let miri = builder.ensure(tool::Miri { compiler: host_compiler, target: host }); + let miri = builder.ensure(tool::Miri { compiler: target_compiler, target: host }); // the ui tests also assume cargo-miri has been built - builder.ensure(tool::CargoMiri { compiler: host_compiler, target: host }); + builder.ensure(tool::CargoMiri { compiler: target_compiler, target: host }); // We also need sysroots, for Miri and for the host (the latter for build scripts). // This is for the tests so everything is done with the target compiler. @@ -543,7 +538,8 @@ impl Step for Miri { // Miri has its own "target dir" for ui test dependencies. Make sure it gets cleared when // the sysroot gets rebuilt, to avoid "found possibly newer version of crate `std`" errors. if !builder.config.dry_run() { - let ui_test_dep_dir = builder.stage_out(host_compiler, Mode::ToolStd).join("miri_ui"); + let ui_test_dep_dir = + builder.stage_out(miri.build_compiler, Mode::ToolStd).join("miri_ui"); // The mtime of `miri_sysroot` changes when the sysroot gets rebuilt (also see // ). // We can hence use that directly as a signal to clear the ui test dir. @@ -554,7 +550,7 @@ impl Step for Miri { // This is with the Miri crate, so it uses the host compiler. let mut cargo = tool::prepare_tool_cargo( builder, - host_compiler, + miri.build_compiler, Mode::ToolRustc, host, Kind::Test, @@ -1722,17 +1718,6 @@ NOTE: if you're sure you want to do this, please open an issue as to why. In the // If we're using `--stage 0`, we should provide the bootstrap cargo. builder.initial_cargo.clone() } else { - // We need to properly build cargo using the suitable stage compiler. - - let compiler = builder.download_rustc().then_some(compiler).unwrap_or_else(|| - // HACK: currently tool stages are off-by-one compared to compiler stages, i.e. if - // you give `tool::Cargo` a stage 1 rustc, it will cause stage 2 rustc to be built - // and produce a cargo built with stage 2 rustc. To fix this, we need to chop off - // the compiler stage by 1 to align with expected `./x test run-make --stage N` - // behavior, i.e. we need to pass `N - 1` compiler stage to cargo. See also Miri - // which does a similar hack. - builder.compiler(builder.top_stage - 1, compiler.host)); - builder.ensure(tool::Cargo { compiler, target: compiler.host }) }; diff --git a/src/bootstrap/src/core/build_steps/tool.rs b/src/bootstrap/src/core/build_steps/tool.rs index cbb973428e2ca..f734776a3d871 100644 --- a/src/bootstrap/src/core/build_steps/tool.rs +++ b/src/bootstrap/src/core/build_steps/tool.rs @@ -274,7 +274,10 @@ pub fn prepare_tool_cargo( cargo } -fn get_tool_rustc_compiler(builder: &Builder<'_>, target_compiler: Compiler) -> Compiler { +pub(crate) fn get_tool_rustc_compiler( + builder: &Builder<'_>, + target_compiler: Compiler, +) -> Compiler { if builder.download_rustc() && target_compiler.stage == 1 { // We already have the stage 1 compiler, we don't need to cut the stage. builder.compiler(target_compiler.stage, builder.config.build) @@ -555,8 +558,7 @@ impl Step for ErrorIndex { // src/tools/error-index-generator` which almost nobody does. // Normally, `x.py test` or `x.py doc` will use the // `ErrorIndex::command` function instead. - let compiler = - run.builder.compiler(run.builder.top_stage.saturating_sub(1), run.builder.config.build); + let compiler = run.builder.compiler(run.builder.top_stage, run.builder.config.build); run.builder.ensure(ErrorIndex { compiler }); } @@ -631,13 +633,8 @@ impl Step for Rustdoc { } fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Rustdoc { - // NOTE: this is somewhat unique in that we actually want a *target* - // compiler here, because rustdoc *is* a compiler. We won't be using - // this as the compiler to build with, but rather this is "what - // compiler are we producing"? - compiler: run.builder.compiler(run.builder.top_stage, run.target), - }); + run.builder + .ensure(Rustdoc { compiler: run.builder.compiler(run.builder.top_stage, run.target) }); } fn run(self, builder: &Builder<'_>) -> PathBuf { diff --git a/src/bootstrap/src/core/builder/mod.rs b/src/bootstrap/src/core/builder/mod.rs index b7820acc7e8b3..c94bdd0a24684 100644 --- a/src/bootstrap/src/core/builder/mod.rs +++ b/src/bootstrap/src/core/builder/mod.rs @@ -1408,10 +1408,9 @@ impl<'a> Builder<'a> { return cmd; } - let build_compiler = self.compiler(run_compiler.stage - 1, self.build.build); - self.ensure(tool::Clippy { compiler: build_compiler, target: self.build.build }); + let _ = self.ensure(tool::Clippy { compiler: run_compiler, target: self.build.build }); let cargo_clippy = - self.ensure(tool::CargoClippy { compiler: build_compiler, target: self.build.build }); + self.ensure(tool::CargoClippy { compiler: run_compiler, target: self.build.build }); let mut dylib_path = helpers::dylib_path(); dylib_path.insert(0, self.sysroot(run_compiler).join("lib")); @@ -1423,23 +1422,21 @@ impl<'a> Builder<'a> { pub fn cargo_miri_cmd(&self, run_compiler: Compiler) -> BootstrapCommand { assert!(run_compiler.stage > 0, "miri can not be invoked at stage 0"); - let build_compiler = self.compiler(run_compiler.stage - 1, self.build.build); - // Prepare the tools - let miri = self.ensure(tool::Miri { compiler: build_compiler, target: self.build.build }); + let miri = self.ensure(tool::Miri { compiler: run_compiler, target: self.build.build }); let cargo_miri = - self.ensure(tool::CargoMiri { compiler: build_compiler, target: self.build.build }); + self.ensure(tool::CargoMiri { compiler: run_compiler, target: self.build.build }); // Invoke cargo-miri, make sure it can find miri and cargo. let mut cmd = command(cargo_miri.tool_path); cmd.env("MIRI", &miri.tool_path); cmd.env("CARGO", &self.initial_cargo); - // Need to add the `run_compiler` libs. Those are the libs produces *by* `build_compiler`, - // so they match the Miri we just built. However this means they are actually living one - // stage up, i.e. we are running `stage0-tools-bin/miri` with the libraries in `stage1/lib`. - // This is an unfortunate off-by-1 caused (possibly) by the fact that Miri doesn't have an - // "assemble" step like rustc does that would cross the stage boundary. We can't use - // `add_rustc_lib_path` as that's a NOP on Windows but we do need these libraries added to - // the PATH due to the stage mismatch. + // Need to add the `run_compiler` libs. Those are the libs produces *by* `build_compiler` + // in `tool::ToolBuild` step, so they match the Miri we just built. However this means they + // are actually living one stage up, i.e. we are running `stage0-tools-bin/miri` with the + // libraries in `stage1/lib`. This is an unfortunate off-by-1 caused (possibly) by the fact + // that Miri doesn't have an "assemble" step like rustc does that would cross the stage boundary. + // We can't use `add_rustc_lib_path` as that's a NOP on Windows but we do need these libraries + // added to the PATH due to the stage mismatch. // Also see https://github.com/rust-lang/rust/pull/123192#issuecomment-2028901503. add_dylib_path(self.rustc_lib_paths(run_compiler), &mut cmd); cmd From 5e5b1b054aae3407f4226ccae2b321e114990776 Mon Sep 17 00:00:00 2001 From: onur-ozkan Date: Tue, 18 Feb 2025 15:34:34 +0000 Subject: [PATCH 08/16] document tool implementations Signed-off-by: onur-ozkan --- src/bootstrap/src/core/build_steps/tool.rs | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/bootstrap/src/core/build_steps/tool.rs b/src/bootstrap/src/core/build_steps/tool.rs index f734776a3d871..7adb7b4681fbd 100644 --- a/src/bootstrap/src/core/build_steps/tool.rs +++ b/src/bootstrap/src/core/build_steps/tool.rs @@ -1,3 +1,14 @@ +//! This module handles building and managing various tools in bootstrap +//! build system. +//! +//! **What It Does** +//! - Defines how tools are built, configured and installed. +//! - Manages tool dependencies and build steps. +//! - Copies built tool binaries to the correct locations. +//! +//! Each Rust tool **MUST** utilize `ToolBuild` inside their `Step` logic, +//! return `ToolBuildResult` and should never prepare `cargo` invocations manually. + use std::path::PathBuf; use std::{env, fs}; @@ -64,10 +75,16 @@ impl Builder<'_> { } } +/// Result of the tool build process. Each `Step` in this module is responsible +/// for using this type as `type Output = ToolBuildResult;` #[derive(Clone)] pub struct ToolBuildResult { + /// Executable path of the corresponding tool that was built. pub tool_path: PathBuf, + /// Compiler used to build the tool. For non-`ToolRustc` tools this is equal to `target_compiler`. + /// For `ToolRustc` this is one stage before of the `target_compiler`. pub build_compiler: Compiler, + /// Target compiler passed to `Step`. pub target_compiler: Compiler, } @@ -274,6 +291,7 @@ pub fn prepare_tool_cargo( cargo } +/// Handle stage-off logic for `ToolRustc` tools when necessary. pub(crate) fn get_tool_rustc_compiler( builder: &Builder<'_>, target_compiler: Compiler, From baef666f788108620263b5d851d845735346acaa Mon Sep 17 00:00:00 2001 From: onur-ozkan Date: Tue, 18 Feb 2025 16:00:06 +0000 Subject: [PATCH 09/16] adapt tool module to `ToolBuildResult` Signed-off-by: onur-ozkan --- src/bootstrap/src/core/build_steps/compile.rs | 26 +- src/bootstrap/src/core/build_steps/dist.rs | 8 +- src/bootstrap/src/core/build_steps/perf.rs | 2 +- src/bootstrap/src/core/build_steps/test.rs | 18 +- src/bootstrap/src/core/build_steps/tool.rs | 312 +++++++++--------- src/bootstrap/src/core/builder/mod.rs | 2 +- 6 files changed, 189 insertions(+), 179 deletions(-) diff --git a/src/bootstrap/src/core/build_steps/compile.rs b/src/bootstrap/src/core/build_steps/compile.rs index aa08dd7e42488..8a0d8b979c37b 100644 --- a/src/bootstrap/src/core/build_steps/compile.rs +++ b/src/bootstrap/src/core/build_steps/compile.rs @@ -1977,13 +1977,14 @@ impl Step for Assemble { let maybe_install_llvm_bitcode_linker = |compiler| { if builder.config.llvm_bitcode_linker_enabled { trace!("llvm-bitcode-linker enabled, installing"); - let src_path = builder.ensure(crate::core::build_steps::tool::LlvmBitcodeLinker { - compiler, - target: target_compiler.host, - extra_features: vec![], - }); + let llvm_bitcode_linker = + builder.ensure(crate::core::build_steps::tool::LlvmBitcodeLinker { + compiler, + target: target_compiler.host, + extra_features: vec![], + }); let tool_exe = exe("llvm-bitcode-linker", target_compiler.host); - builder.copy_link(&src_path, &libdir_bin.join(tool_exe)); + builder.copy_link(&llvm_bitcode_linker.tool_path, &libdir_bin.join(tool_exe)); } }; @@ -2171,14 +2172,13 @@ impl Step for Assemble { // logic to create the final binary. This is used by the // `wasm32-wasip2` target of Rust. if builder.tool_enabled("wasm-component-ld") { - let wasm_component_ld_exe = - builder.ensure(crate::core::build_steps::tool::WasmComponentLd { - compiler: build_compiler, - target: target_compiler.host, - }); + let wasm_component = builder.ensure(crate::core::build_steps::tool::WasmComponentLd { + compiler: build_compiler, + target: target_compiler.host, + }); builder.copy_link( - &wasm_component_ld_exe, - &libdir_bin.join(wasm_component_ld_exe.file_name().unwrap()), + &wasm_component.tool_path, + &libdir_bin.join(wasm_component.tool_path.file_name().unwrap()), ); } diff --git a/src/bootstrap/src/core/build_steps/dist.rs b/src/bootstrap/src/core/build_steps/dist.rs index 403aa48bad9b7..f742a277a95a7 100644 --- a/src/bootstrap/src/core/build_steps/dist.rs +++ b/src/bootstrap/src/core/build_steps/dist.rs @@ -430,7 +430,7 @@ impl Step for Rustc { }, builder.kind, ) { - builder.install(&ra_proc_macro_srv, &image.join("libexec"), 0o755); + builder.install(&ra_proc_macro_srv.tool_path, &image.join("libexec"), 0o755); } let libdir_relative = builder.libdir_relative(compiler); @@ -1134,7 +1134,7 @@ impl Step for Cargo { let mut tarball = Tarball::new(builder, "cargo", &target.triple); tarball.set_overlay(OverlayKind::Cargo); - tarball.add_file(cargo, "bin", 0o755); + tarball.add_file(cargo.tool_path, "bin", 0o755); tarball.add_file(etc.join("_cargo"), "share/zsh/site-functions", 0o644); tarball.add_renamed_file(etc.join("cargo.bashcomp.sh"), "etc/bash_completion.d", "cargo"); tarball.add_dir(etc.join("man"), "share/man/man1"); @@ -1222,7 +1222,7 @@ impl Step for RustAnalyzer { let mut tarball = Tarball::new(builder, "rust-analyzer", &target.triple); tarball.set_overlay(OverlayKind::RustAnalyzer); tarball.is_preview(true); - tarball.add_file(rust_analyzer, "bin", 0o755); + tarball.add_file(rust_analyzer.tool_path, "bin", 0o755); tarball.add_legal_and_readme_to("share/doc/rust-analyzer"); Some(tarball.generate()) } @@ -2272,7 +2272,7 @@ impl Step for LlvmBitcodeLinker { tarball.set_overlay(OverlayKind::LlvmBitcodeLinker); tarball.is_preview(true); - tarball.add_file(llbc_linker, self_contained_bin_dir, 0o755); + tarball.add_file(llbc_linker.tool_path, self_contained_bin_dir, 0o755); Some(tarball.generate()) } diff --git a/src/bootstrap/src/core/build_steps/perf.rs b/src/bootstrap/src/core/build_steps/perf.rs index 98c63a41e768b..6962001fdc240 100644 --- a/src/bootstrap/src/core/build_steps/perf.rs +++ b/src/bootstrap/src/core/build_steps/perf.rs @@ -166,7 +166,7 @@ Consider setting `rust.debuginfo-level = 1` in `config.toml`."#); let results_dir = rustc_perf_dir.join("results"); builder.create_dir(&results_dir); - let mut cmd = command(collector); + let mut cmd = command(collector.tool_path); // We need to set the working directory to `src/tools/rustc-perf`, so that it can find the directory // with compile-time benchmarks. diff --git a/src/bootstrap/src/core/build_steps/test.rs b/src/bootstrap/src/core/build_steps/test.rs index e871a40731e34..c7e9675faa2a1 100644 --- a/src/bootstrap/src/core/build_steps/test.rs +++ b/src/bootstrap/src/core/build_steps/test.rs @@ -263,7 +263,7 @@ impl Step for Cargotest { let _time = helpers::timeit(builder); let mut cmd = builder.tool_cmd(Tool::CargoTest); - cmd.arg(&cargo) + cmd.arg(&cargo.tool_path) .arg(&out_dir) .args(builder.config.test_args()) .env("RUSTC", builder.rustc(compiler)) @@ -1718,7 +1718,7 @@ NOTE: if you're sure you want to do this, please open an issue as to why. In the // If we're using `--stage 0`, we should provide the bootstrap cargo. builder.initial_cargo.clone() } else { - builder.ensure(tool::Cargo { compiler, target: compiler.host }) + builder.ensure(tool::Cargo { compiler, target: compiler.host }).tool_path }; cmd.arg("--cargo-path").arg(cargo_path); @@ -1739,9 +1739,10 @@ NOTE: if you're sure you want to do this, please open an issue as to why. In the // Use the beta compiler for jsondocck let json_compiler = compiler.with_stage(0); cmd.arg("--jsondocck-path") - .arg(builder.ensure(tool::JsonDocCk { compiler: json_compiler, target })); - cmd.arg("--jsondoclint-path") - .arg(builder.ensure(tool::JsonDocLint { compiler: json_compiler, target })); + .arg(builder.ensure(tool::JsonDocCk { compiler: json_compiler, target }).tool_path); + cmd.arg("--jsondoclint-path").arg( + builder.ensure(tool::JsonDocLint { compiler: json_compiler, target }).tool_path, + ); } if matches!(mode, "coverage-map" | "coverage-run") { @@ -2976,12 +2977,15 @@ impl Step for RemoteCopyLibs { builder.info(&format!("REMOTE copy libs to emulator ({target})")); - let server = builder.ensure(tool::RemoteTestServer { compiler, target }); + let remote_test_server = builder.ensure(tool::RemoteTestServer { compiler, target }); // Spawn the emulator and wait for it to come online let tool = builder.tool_exe(Tool::RemoteTestClient); let mut cmd = command(&tool); - cmd.arg("spawn-emulator").arg(target.triple).arg(&server).arg(builder.tempdir()); + cmd.arg("spawn-emulator") + .arg(target.triple) + .arg(&remote_test_server.tool_path) + .arg(builder.tempdir()); if let Some(rootfs) = builder.qemu_rootfs(target) { cmd.arg(rootfs); } diff --git a/src/bootstrap/src/core/build_steps/tool.rs b/src/bootstrap/src/core/build_steps/tool.rs index 7adb7b4681fbd..39acb646dff4d 100644 --- a/src/bootstrap/src/core/build_steps/tool.rs +++ b/src/bootstrap/src/core/build_steps/tool.rs @@ -347,7 +347,7 @@ macro_rules! bootstrap_tool { self.ensure($name { compiler: self.compiler(0, self.config.build), target: self.config.build, - }), + }).tool_path, )+ } } @@ -361,7 +361,7 @@ macro_rules! bootstrap_tool { } impl Step for $name { - type Output = PathBuf; + type Output = ToolBuildResult; fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { run.path($path) @@ -383,7 +383,7 @@ macro_rules! bootstrap_tool { skip_all, ), )] - fn run(self, builder: &Builder<'_>) -> PathBuf { + fn run(self, builder: &Builder<'_>) -> ToolBuildResult { $( for submodule in $submodules { builder.require_submodule(submodule, None); @@ -408,7 +408,7 @@ macro_rules! bootstrap_tool { extra_features: vec![], allow_features: concat!($($allow_features)*), cargo_args: vec![] - }).tool_path + }) } } )+ @@ -458,7 +458,7 @@ pub struct OptimizedDist { } impl Step for OptimizedDist { - type Output = PathBuf; + type Output = ToolBuildResult; fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { run.path("src/tools/opt-dist") @@ -471,24 +471,22 @@ impl Step for OptimizedDist { }); } - fn run(self, builder: &Builder<'_>) -> PathBuf { + fn run(self, builder: &Builder<'_>) -> ToolBuildResult { // We need to ensure the rustc-perf submodule is initialized when building opt-dist since // the tool requires it to be in place to run. builder.require_submodule("src/tools/rustc-perf", None); - builder - .ensure(ToolBuild { - compiler: self.compiler, - target: self.target, - tool: "opt-dist", - mode: Mode::ToolBootstrap, - path: "src/tools/opt-dist", - source_type: SourceType::InTree, - extra_features: Vec::new(), - allow_features: "", - cargo_args: Vec::new(), - }) - .tool_path + builder.ensure(ToolBuild { + compiler: self.compiler, + target: self.target, + tool: "opt-dist", + mode: Mode::ToolBootstrap, + path: "src/tools/opt-dist", + source_type: SourceType::InTree, + extra_features: Vec::new(), + allow_features: "", + cargo_args: Vec::new(), + }) } } @@ -502,7 +500,7 @@ pub struct RustcPerf { impl Step for RustcPerf { /// Path to the built `collector` binary. - type Output = PathBuf; + type Output = ToolBuildResult; fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { run.path("src/tools/rustc-perf") @@ -515,7 +513,7 @@ impl Step for RustcPerf { }); } - fn run(self, builder: &Builder<'_>) -> PathBuf { + fn run(self, builder: &Builder<'_>) -> ToolBuildResult { // We need to ensure the rustc-perf submodule is initialized. builder.require_submodule("src/tools/rustc-perf", None); @@ -532,12 +530,12 @@ impl Step for RustcPerf { // a CLI. cargo_args: vec!["-p".to_string(), "collector".to_string()], }; - let collector_bin = builder.ensure(tool.clone()).tool_path; + let res = builder.ensure(tool.clone()); // We also need to symlink the `rustc-fake` binary to the corresponding directory, // because `collector` expects it in the same directory. copy_link_tool_bin(builder, tool.compiler, tool.target, tool.mode, "rustc-fake"); - collector_bin + res } } @@ -552,7 +550,7 @@ impl ErrorIndex { // for rustc_private and libLLVM.so, and `sysroot_lib` for libstd, etc. let host = builder.config.build; let compiler = builder.compiler_for(builder.top_stage, host, host); - let mut cmd = command(builder.ensure(ErrorIndex { compiler })); + let mut cmd = command(builder.ensure(ErrorIndex { compiler }).tool_path); let mut dylib_paths = builder.rustc_lib_paths(compiler); dylib_paths.push(PathBuf::from(&builder.sysroot_target_libdir(compiler, compiler.host))); add_dylib_path(dylib_paths, &mut cmd); @@ -561,16 +559,13 @@ impl ErrorIndex { } impl Step for ErrorIndex { - type Output = PathBuf; + type Output = ToolBuildResult; fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { run.path("src/tools/error_index_generator") } fn make_run(run: RunConfig<'_>) { - // Compile the error-index in the same stage as rustdoc to avoid - // recompiling rustdoc twice if we can. - // // NOTE: This `make_run` isn't used in normal situations, only if you // manually build the tool with `x.py build // src/tools/error-index-generator` which almost nobody does. @@ -580,20 +575,18 @@ impl Step for ErrorIndex { run.builder.ensure(ErrorIndex { compiler }); } - fn run(self, builder: &Builder<'_>) -> PathBuf { - builder - .ensure(ToolBuild { - compiler: self.compiler, - target: self.compiler.host, - tool: "error_index_generator", - mode: Mode::ToolRustc, - path: "src/tools/error_index_generator", - source_type: SourceType::InTree, - extra_features: Vec::new(), - allow_features: "", - cargo_args: Vec::new(), - }) - .tool_path + fn run(self, builder: &Builder<'_>) -> ToolBuildResult { + builder.ensure(ToolBuild { + compiler: self.compiler, + target: self.compiler.host, + tool: "error_index_generator", + mode: Mode::ToolRustc, + path: "src/tools/error_index_generator", + source_type: SourceType::InTree, + extra_features: Vec::new(), + allow_features: "", + cargo_args: Vec::new(), + }) } } @@ -604,7 +597,7 @@ pub struct RemoteTestServer { } impl Step for RemoteTestServer { - type Output = PathBuf; + type Output = ToolBuildResult; fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { run.path("src/tools/remote-test-server") @@ -617,20 +610,18 @@ impl Step for RemoteTestServer { }); } - fn run(self, builder: &Builder<'_>) -> PathBuf { - builder - .ensure(ToolBuild { - compiler: self.compiler, - target: self.target, - tool: "remote-test-server", - mode: Mode::ToolStd, - path: "src/tools/remote-test-server", - source_type: SourceType::InTree, - extra_features: Vec::new(), - allow_features: "", - cargo_args: Vec::new(), - }) - .tool_path + fn run(self, builder: &Builder<'_>) -> ToolBuildResult { + builder.ensure(ToolBuild { + compiler: self.compiler, + target: self.target, + tool: "remote-test-server", + mode: Mode::ToolStd, + path: "src/tools/remote-test-server", + source_type: SourceType::InTree, + extra_features: Vec::new(), + allow_features: "", + cargo_args: Vec::new(), + }) } } @@ -642,7 +633,7 @@ pub struct Rustdoc { } impl Step for Rustdoc { - type Output = PathBuf; + type Output = ToolBuildResult; const DEFAULT: bool = true; const ONLY_HOSTS: bool = true; @@ -655,15 +646,21 @@ impl Step for Rustdoc { .ensure(Rustdoc { compiler: run.builder.compiler(run.builder.top_stage, run.target) }); } - fn run(self, builder: &Builder<'_>) -> PathBuf { + fn run(self, builder: &Builder<'_>) -> ToolBuildResult { let target_compiler = self.compiler; + let target = target_compiler.host; + if target_compiler.stage == 0 { if !target_compiler.is_snapshot(builder) { panic!("rustdoc in stage 0 must be snapshot rustdoc"); } - return builder.initial_rustdoc.clone(); + + return ToolBuildResult { + tool_path: builder.initial_rustdoc.clone(), + build_compiler: target_compiler, + target_compiler, + }; } - let target = target_compiler.host; let bin_rustdoc = || { let sysroot = builder.sysroot(target_compiler); @@ -693,7 +690,12 @@ impl Step for Rustdoc { let bin_rustdoc = bin_rustdoc(); builder.copy_link(&precompiled_rustdoc, &bin_rustdoc); - return bin_rustdoc; + + return ToolBuildResult { + tool_path: bin_rustdoc, + build_compiler: target_compiler, + target_compiler, + }; } } @@ -709,7 +711,7 @@ impl Step for Rustdoc { extra_features.push("jemalloc".to_string()); } - let ToolBuildResult { tool_path, build_compiler: _build_compiler, target_compiler } = + let ToolBuildResult { tool_path, build_compiler, target_compiler } = builder.ensure(ToolBuild { compiler: target_compiler, target, @@ -734,9 +736,9 @@ impl Step for Rustdoc { } let bin_rustdoc = bin_rustdoc(); builder.copy_link(&tool_path, &bin_rustdoc); - bin_rustdoc + ToolBuildResult { tool_path: bin_rustdoc, build_compiler, target_compiler } } else { - tool_path + ToolBuildResult { tool_path, build_compiler, target_compiler } } } } @@ -748,7 +750,7 @@ pub struct Cargo { } impl Step for Cargo { - type Output = PathBuf; + type Output = ToolBuildResult; const DEFAULT: bool = true; const ONLY_HOSTS: bool = true; @@ -764,22 +766,20 @@ impl Step for Cargo { }); } - fn run(self, builder: &Builder<'_>) -> PathBuf { + fn run(self, builder: &Builder<'_>) -> ToolBuildResult { builder.build.require_submodule("src/tools/cargo", None); - builder - .ensure(ToolBuild { - compiler: self.compiler, - target: self.target, - tool: "cargo", - mode: Mode::ToolRustc, - path: "src/tools/cargo", - source_type: SourceType::Submodule, - extra_features: Vec::new(), - allow_features: "", - cargo_args: Vec::new(), - }) - .tool_path + builder.ensure(ToolBuild { + compiler: self.compiler, + target: self.target, + tool: "cargo", + mode: Mode::ToolRustc, + path: "src/tools/cargo", + source_type: SourceType::Submodule, + extra_features: Vec::new(), + allow_features: "", + cargo_args: Vec::new(), + }) } } @@ -790,7 +790,7 @@ pub struct LldWrapper { } impl Step for LldWrapper { - type Output = (); + type Output = ToolBuildResult; fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { run.never() @@ -805,26 +805,29 @@ impl Step for LldWrapper { fields(build_compiler = ?self.build_compiler, target_compiler = ?self.target_compiler), ), )] - fn run(self, builder: &Builder<'_>) { + + fn run(self, builder: &Builder<'_>) -> ToolBuildResult { if builder.config.dry_run() { - return; + return ToolBuildResult { + tool_path: Default::default(), + build_compiler: self.build_compiler, + target_compiler: self.target_compiler, + }; } let target = self.target_compiler.host; - let executable = builder - .ensure(ToolBuild { - compiler: self.build_compiler, - target, - tool: "lld-wrapper", - mode: Mode::ToolStd, - path: "src/tools/lld-wrapper", - source_type: SourceType::InTree, - extra_features: Vec::new(), - allow_features: "", - cargo_args: Vec::new(), - }) - .tool_path; + let tool_result = builder.ensure(ToolBuild { + compiler: self.build_compiler, + target, + tool: "lld-wrapper", + mode: Mode::ToolStd, + path: "src/tools/lld-wrapper", + source_type: SourceType::InTree, + extra_features: Vec::new(), + allow_features: "", + cargo_args: Vec::new(), + }); let libdir_bin = builder.sysroot_target_bindir(self.target_compiler, target); t!(fs::create_dir_all(&libdir_bin)); @@ -838,8 +841,11 @@ impl Step for LldWrapper { t!(fs::create_dir_all(&self_contained_lld_dir)); for name in crate::LLD_FILE_NAMES { - builder.copy_link(&executable, &self_contained_lld_dir.join(exe(name, target))); + builder + .copy_link(&tool_result.tool_path, &self_contained_lld_dir.join(exe(name, target))); } + + tool_result } } @@ -854,7 +860,7 @@ impl RustAnalyzer { } impl Step for RustAnalyzer { - type Output = PathBuf; + type Output = ToolBuildResult; const DEFAULT: bool = true; const ONLY_HOSTS: bool = true; @@ -870,20 +876,18 @@ impl Step for RustAnalyzer { }); } - fn run(self, builder: &Builder<'_>) -> PathBuf { - builder - .ensure(ToolBuild { - compiler: self.compiler, - target: self.target, - tool: "rust-analyzer", - mode: Mode::ToolRustc, - path: "src/tools/rust-analyzer", - extra_features: vec!["in-rust-tree".to_owned()], - source_type: SourceType::InTree, - allow_features: RustAnalyzer::ALLOW_FEATURES, - cargo_args: Vec::new(), - }) - .tool_path + fn run(self, builder: &Builder<'_>) -> ToolBuildResult { + builder.ensure(ToolBuild { + compiler: self.compiler, + target: self.target, + tool: "rust-analyzer", + mode: Mode::ToolRustc, + path: "src/tools/rust-analyzer", + extra_features: vec!["in-rust-tree".to_owned()], + source_type: SourceType::InTree, + allow_features: RustAnalyzer::ALLOW_FEATURES, + cargo_args: Vec::new(), + }) } } @@ -894,7 +898,7 @@ pub struct RustAnalyzerProcMacroSrv { } impl Step for RustAnalyzerProcMacroSrv { - type Output = Option; + type Output = Option; const DEFAULT: bool = true; const ONLY_HOSTS: bool = true; @@ -916,28 +920,27 @@ impl Step for RustAnalyzerProcMacroSrv { }); } - fn run(self, builder: &Builder<'_>) -> Option { - let path = builder - .ensure(ToolBuild { - compiler: self.compiler, - target: self.target, - tool: "rust-analyzer-proc-macro-srv", - mode: Mode::ToolRustc, - path: "src/tools/rust-analyzer/crates/proc-macro-srv-cli", - extra_features: vec!["in-rust-tree".to_owned()], - source_type: SourceType::InTree, - allow_features: RustAnalyzer::ALLOW_FEATURES, - cargo_args: Vec::new(), - }) - .tool_path; + fn run(self, builder: &Builder<'_>) -> Option { + let tool_result = builder.ensure(ToolBuild { + compiler: self.compiler, + target: self.target, + tool: "rust-analyzer-proc-macro-srv", + mode: Mode::ToolRustc, + path: "src/tools/rust-analyzer/crates/proc-macro-srv-cli", + extra_features: vec!["in-rust-tree".to_owned()], + source_type: SourceType::InTree, + allow_features: RustAnalyzer::ALLOW_FEATURES, + cargo_args: Vec::new(), + }); // Copy `rust-analyzer-proc-macro-srv` to `/libexec/` // so that r-a can use it. let libexec_path = builder.sysroot(self.compiler).join("libexec"); t!(fs::create_dir_all(&libexec_path)); - builder.copy_link(&path, &libexec_path.join("rust-analyzer-proc-macro-srv")); + builder + .copy_link(&tool_result.tool_path, &libexec_path.join("rust-analyzer-proc-macro-srv")); - Some(path) + Some(tool_result) } } @@ -949,7 +952,7 @@ pub struct LlvmBitcodeLinker { } impl Step for LlvmBitcodeLinker { - type Output = PathBuf; + type Output = ToolBuildResult; const DEFAULT: bool = true; const ONLY_HOSTS: bool = true; @@ -971,31 +974,34 @@ impl Step for LlvmBitcodeLinker { feature = "tracing", instrument(level = "debug", name = "LlvmBitcodeLinker::run", skip_all) )] - fn run(self, builder: &Builder<'_>) -> PathBuf { - let ToolBuildResult { tool_path, build_compiler: _build_compiler, target_compiler } = - builder.ensure(ToolBuild { - compiler: self.compiler, - target: self.target, - tool: "llvm-bitcode-linker", - mode: Mode::ToolRustc, - path: "src/tools/llvm-bitcode-linker", - source_type: SourceType::InTree, - extra_features: self.extra_features, - allow_features: "", - cargo_args: Vec::new(), - }); + fn run(self, builder: &Builder<'_>) -> ToolBuildResult { + let tool_result = builder.ensure(ToolBuild { + compiler: self.compiler, + target: self.target, + tool: "llvm-bitcode-linker", + mode: Mode::ToolRustc, + path: "src/tools/llvm-bitcode-linker", + source_type: SourceType::InTree, + extra_features: self.extra_features, + allow_features: "", + cargo_args: Vec::new(), + }); - if target_compiler.stage > 0 { + if tool_result.target_compiler.stage > 0 { let bindir_self_contained = builder - .sysroot(target_compiler) + .sysroot(tool_result.target_compiler) .join(format!("lib/rustlib/{}/bin/self-contained", self.target.triple)); t!(fs::create_dir_all(&bindir_self_contained)); - let bin_destination = - bindir_self_contained.join(exe("llvm-bitcode-linker", target_compiler.host)); - builder.copy_link(&tool_path, &bin_destination); - bin_destination + let bin_destination = bindir_self_contained + .join(exe("llvm-bitcode-linker", tool_result.target_compiler.host)); + builder.copy_link(&tool_result.tool_path, &bin_destination); + ToolBuildResult { + tool_path: bin_destination, + build_compiler: tool_result.build_compiler, + target_compiler: tool_result.target_compiler, + } } else { - tool_path + tool_result } } } @@ -1214,7 +1220,7 @@ pub struct TestFloatParse { } impl Step for TestFloatParse { - type Output = (); + type Output = ToolBuildResult; const ONLY_HOSTS: bool = true; const DEFAULT: bool = false; @@ -1222,7 +1228,7 @@ impl Step for TestFloatParse { run.path("src/etc/test-float-parse") } - fn run(self, builder: &Builder<'_>) { + fn run(self, builder: &Builder<'_>) -> ToolBuildResult { let bootstrap_host = builder.config.build; let compiler = builder.compiler(builder.top_stage, bootstrap_host); @@ -1236,7 +1242,7 @@ impl Step for TestFloatParse { extra_features: Vec::new(), allow_features: "", cargo_args: Vec::new(), - }); + }) } } diff --git a/src/bootstrap/src/core/builder/mod.rs b/src/bootstrap/src/core/builder/mod.rs index c94bdd0a24684..25fa10e081114 100644 --- a/src/bootstrap/src/core/builder/mod.rs +++ b/src/bootstrap/src/core/builder/mod.rs @@ -1392,7 +1392,7 @@ impl<'a> Builder<'a> { } pub fn rustdoc(&self, compiler: Compiler) -> PathBuf { - self.ensure(tool::Rustdoc { compiler }) + self.ensure(tool::Rustdoc { compiler }).tool_path } pub fn cargo_clippy_cmd(&self, run_compiler: Compiler) -> BootstrapCommand { From 60ade6c1d7c8bbac52d6d1b9ed6703582028f954 Mon Sep 17 00:00:00 2001 From: onur-ozkan Date: Tue, 18 Feb 2025 16:07:42 +0000 Subject: [PATCH 10/16] add test coverage for `tool::get_tool_rustc_compiler` Signed-off-by: onur-ozkan --- src/bootstrap/src/core/builder/tests.rs | 30 +++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/src/bootstrap/src/core/builder/tests.rs b/src/bootstrap/src/core/builder/tests.rs index a828a844f8103..b6aa9e7c8446d 100644 --- a/src/bootstrap/src/core/builder/tests.rs +++ b/src/bootstrap/src/core/builder/tests.rs @@ -1085,3 +1085,33 @@ fn test_is_builder_target() { assert!(!builder.is_builder_target(target2)); } } + +#[test] +fn test_get_tool_rustc_compiler() { + let mut config = configure("build", &[], &[]); + config.download_rustc_commit = None; + let build = Build::new(config); + let builder = Builder::new(&build); + + let target_triple_1 = TargetSelection::from_user(TEST_TRIPLE_1); + + let compiler = Compiler { stage: 2, host: target_triple_1 }; + let expected = Compiler { stage: 1, host: target_triple_1 }; + let actual = tool::get_tool_rustc_compiler(&builder, compiler); + assert_eq!(expected, actual); + + let compiler = Compiler { stage: 1, host: target_triple_1 }; + let expected = Compiler { stage: 0, host: target_triple_1 }; + let actual = tool::get_tool_rustc_compiler(&builder, compiler); + assert_eq!(expected, actual); + + let mut config = configure("build", &[], &[]); + config.download_rustc_commit = Some("".to_owned()); + let build = Build::new(config); + let builder = Builder::new(&build); + + let compiler = Compiler { stage: 1, host: target_triple_1 }; + let expected = Compiler { stage: 1, host: target_triple_1 }; + let actual = tool::get_tool_rustc_compiler(&builder, compiler); + assert_eq!(expected, actual); +} From 26f588bba4e2b2309628ecc1322b6d254253eae7 Mon Sep 17 00:00:00 2001 From: onur-ozkan Date: Tue, 18 Feb 2025 16:28:05 +0000 Subject: [PATCH 11/16] fix cargo tests Signed-off-by: onur-ozkan --- src/bootstrap/src/core/build_steps/test.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/bootstrap/src/core/build_steps/test.rs b/src/bootstrap/src/core/build_steps/test.rs index c7e9675faa2a1..db2f433bfe15e 100644 --- a/src/bootstrap/src/core/build_steps/test.rs +++ b/src/bootstrap/src/core/build_steps/test.rs @@ -300,7 +300,9 @@ impl Step for Cargo { fn run(self, builder: &Builder<'_>) { let compiler = builder.compiler(self.stage, self.host); - builder.ensure(tool::Cargo { compiler, target: self.host }); + let cargo = builder.ensure(tool::Cargo { compiler, target: self.host }); + let compiler = cargo.build_compiler; + let cargo = tool::prepare_tool_cargo( builder, compiler, From 5408aaea598793f0803af59f66dfd31376b6c661 Mon Sep 17 00:00:00 2001 From: onur-ozkan Date: Tue, 18 Feb 2025 20:21:50 +0300 Subject: [PATCH 12/16] fix rust-analyzer tests Signed-off-by: onur-ozkan --- src/bootstrap/src/core/build_steps/test.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/bootstrap/src/core/build_steps/test.rs b/src/bootstrap/src/core/build_steps/test.rs index db2f433bfe15e..8d53ebdca19ef 100644 --- a/src/bootstrap/src/core/build_steps/test.rs +++ b/src/bootstrap/src/core/build_steps/test.rs @@ -369,6 +369,7 @@ impl Step for RustAnalyzer { let stage = self.stage; let host = self.host; let compiler = builder.compiler(stage, host); + let compiler = tool::get_tool_rustc_compiler(builder, compiler); // We don't need to build the whole Rust Analyzer for the proc-macro-srv test suite, // but we do need the standard library to be present. From 26d6ce76a7b38059146a277d15c007b0eeab834e Mon Sep 17 00:00:00 2001 From: onur-ozkan Date: Wed, 19 Feb 2025 09:03:35 +0300 Subject: [PATCH 13/16] add rustc-dev doc about bootstrap tools Signed-off-by: onur-ozkan --- src/doc/rustc-dev-guide/src/SUMMARY.md | 1 + .../writing-tools-in-bootstrap.md | 23 +++++++++++++++++++ 2 files changed, 24 insertions(+) create mode 100644 src/doc/rustc-dev-guide/src/building/bootstrapping/writing-tools-in-bootstrap.md diff --git a/src/doc/rustc-dev-guide/src/SUMMARY.md b/src/doc/rustc-dev-guide/src/SUMMARY.md index 91c4aeacbd749..106db508ebbc7 100644 --- a/src/doc/rustc-dev-guide/src/SUMMARY.md +++ b/src/doc/rustc-dev-guide/src/SUMMARY.md @@ -75,6 +75,7 @@ - [Prologue](./building/bootstrapping/intro.md) - [What Bootstrapping does](./building/bootstrapping/what-bootstrapping-does.md) - [How Bootstrap does it](./building/bootstrapping/how-bootstrap-does-it.md) +- [Writing tools in Bootstrap](./building/bootstrapping/writing-tools-in-bootstrap.md) - [Debugging bootstrap](./building/bootstrapping/debugging-bootstrap.md) # High-level Compiler Architecture diff --git a/src/doc/rustc-dev-guide/src/building/bootstrapping/writing-tools-in-bootstrap.md b/src/doc/rustc-dev-guide/src/building/bootstrapping/writing-tools-in-bootstrap.md new file mode 100644 index 0000000000000..6046d5b133d7d --- /dev/null +++ b/src/doc/rustc-dev-guide/src/building/bootstrapping/writing-tools-in-bootstrap.md @@ -0,0 +1,23 @@ +# Writing tools in Bootstrap + +There are three types of tools you can write in bootstrap: + +- **`Mode::ToolBootstrap`** + Use this for tools that don’t need anything from the in-tree compiler and can run with the stage0 `rustc`. + The output is placed in the "stage0-bootstrap-tools" directory. This mode is for general-purpose tools built + entirely with the stage0 compiler, including target libraries and only works for stage 0. + +- **`Mode::ToolStd`** + Use this for tools that rely on the locally built std. The output goes into the "stageN-tools" directory. + This mode is rarely used, mainly for `compiletest` which requires `libtest`. + +- **`Mode::ToolRustc`** + Use this for tools that depend on both the locally built `rustc` and the target `std`. This is more complex than + the other modes because the tool must be built with the same compiler used for `rustc` and placed in the "stageN-tools" + directory. When you choose `Mode::ToolRustc`, `ToolBuild` implementation takes care of this automatically. + If you need to use the builder’s compiler for something specific, you can get it from `ToolBuildResult`, which is + returned by the tool's [`Step`]. + +Regardless of the tool type you must return `ToolBuildResult` from the tool’s [`Step`] implementation and use `ToolBuild` inside it. + +[`Step`]: https://doc.rust-lang.org/nightly/nightly-rustc/bootstrap/core/builder/trait.Step.html From b4a3b6484d4a6ca4d5b4561aa0113a8e2ab27695 Mon Sep 17 00:00:00 2001 From: onur-ozkan Date: Wed, 19 Feb 2025 19:16:49 +0300 Subject: [PATCH 14/16] set `build.test-stage = 2` for `tools` profile Signed-off-by: onur-ozkan --- src/bootstrap/defaults/config.tools.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/bootstrap/defaults/config.tools.toml b/src/bootstrap/defaults/config.tools.toml index 64097320caba7..57c2706f60a56 100644 --- a/src/bootstrap/defaults/config.tools.toml +++ b/src/bootstrap/defaults/config.tools.toml @@ -8,6 +8,8 @@ incremental = true download-rustc = "if-unchanged" [build] +# cargo and clippy tests don't pass on stage 1 +test-stage = 2 # Document with the in-tree rustdoc by default, since `download-rustc` makes it quick to compile. doc-stage = 2 # Contributors working on tools will probably expect compiler docs to be generated, so they can figure out how to use the API. From 4b9b5d78963154386d7a0b41b4215bc13ad4dbca Mon Sep 17 00:00:00 2001 From: onur-ozkan Date: Wed, 19 Feb 2025 19:18:54 +0300 Subject: [PATCH 15/16] add change-entry for tools profile update Signed-off-by: onur-ozkan --- src/bootstrap/src/utils/change_tracker.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/bootstrap/src/utils/change_tracker.rs b/src/bootstrap/src/utils/change_tracker.rs index f215c3f6d0b39..8dfe0d3a35ef6 100644 --- a/src/bootstrap/src/utils/change_tracker.rs +++ b/src/bootstrap/src/utils/change_tracker.rs @@ -355,4 +355,9 @@ pub const CONFIG_CHANGE_HISTORY: &[ChangeInfo] = &[ severity: ChangeSeverity::Info, summary: "It is now possible to configure `jemalloc` for each target", }, + ChangeInfo { + change_id: 137215, + severity: ChangeSeverity::Info, + summary: "Added `build.test-stage = 2` to 'tools' profile defaults", + }, ]; From 76063a683f2ef7530d07301127027488f1ecec8d Mon Sep 17 00:00:00 2001 From: onur-ozkan Date: Wed, 19 Feb 2025 19:25:17 +0300 Subject: [PATCH 16/16] print warning and help messages Signed-off-by: onur-ozkan --- src/bootstrap/src/core/build_steps/test.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/bootstrap/src/core/build_steps/test.rs b/src/bootstrap/src/core/build_steps/test.rs index 8d53ebdca19ef..40f5259133a5d 100644 --- a/src/bootstrap/src/core/build_steps/test.rs +++ b/src/bootstrap/src/core/build_steps/test.rs @@ -298,6 +298,11 @@ impl Step for Cargo { /// Runs `cargo test` for `cargo` packaged with Rust. fn run(self, builder: &Builder<'_>) { + if self.stage < 2 { + eprintln!("WARNING: cargo tests on stage {} may not behave well.", self.stage); + eprintln!("HELP: consider using stage 2"); + } + let compiler = builder.compiler(self.stage, self.host); let cargo = builder.ensure(tool::Cargo { compiler, target: self.host }); @@ -743,6 +748,11 @@ impl Step for Clippy { let host = self.host; let compiler = builder.compiler(stage, host); + if stage < 2 { + eprintln!("WARNING: clippy tests on stage {stage} may not behave well."); + eprintln!("HELP: consider using stage 2"); + } + let tool_result = builder.ensure(tool::Clippy { compiler, target: self.host }); let compiler = tool_result.build_compiler; let mut cargo = tool::prepare_tool_cargo(