no python in shell scripts

.github/workflows/ci.yml

@@ -311,7 +311,7 @@ jobs:
             os: ubuntu-20.04-8core-32gb
           - name: dist-x86_64-apple
             env:
-              SCRIPT: "./x.py dist bootstrap --include-default-paths --host=x86_64-apple-darwin --target=x86_64-apple-darwin"
+              SCRIPT: "./x.py dist bootstrap bootstrap-shim --include-default-paths --host=x86_64-apple-darwin --target=x86_64-apple-darwin"
               RUST_CONFIGURE_ARGS: "--enable-full-tools --enable-sanitizers --enable-profiler --set rust.jemalloc --set llvm.ninja=false --set rust.lto=thin"
               RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
               MACOSX_DEPLOYMENT_TARGET: 10.7
@@ -323,7 +323,7 @@ jobs:
             os: macos-latest
           - name: dist-apple-various
             env:
-              SCRIPT: "./x.py dist bootstrap --include-default-paths --host='' --target=aarch64-apple-ios,x86_64-apple-ios,aarch64-apple-ios-sim"
+              SCRIPT: "./x.py dist bootstrap bootstrap-shim --include-default-paths --host='' --target=aarch64-apple-ios,x86_64-apple-ios,aarch64-apple-ios-sim"
               RUST_CONFIGURE_ARGS: "--enable-sanitizers --enable-profiler --set rust.jemalloc --set llvm.ninja=false"
               RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
               MACOSX_DEPLOYMENT_TARGET: 10.7
@@ -334,7 +334,7 @@ jobs:
             os: macos-latest
           - name: dist-x86_64-apple-alt
             env:
-              SCRIPT: "./x.py dist bootstrap --include-default-paths"
+              SCRIPT: "./x.py dist bootstrap bootstrap-shim --include-default-paths"
               RUST_CONFIGURE_ARGS: "--enable-extended --enable-profiler --set rust.jemalloc --set llvm.ninja=false"
               RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
               MACOSX_DEPLOYMENT_TARGET: 10.7
@@ -367,7 +367,7 @@ jobs:
             os: macos-latest
           - name: dist-aarch64-apple
             env:
-              SCRIPT: "./x.py dist bootstrap --include-default-paths --stage 2"
+              SCRIPT: "./x.py dist bootstrap bootstrap-shim --include-default-paths --stage 2"
               RUST_CONFIGURE_ARGS: "--build=x86_64-apple-darwin --host=aarch64-apple-darwin --target=aarch64-apple-darwin --enable-full-tools --enable-sanitizers --enable-profiler --disable-docs --set rust.jemalloc --set llvm.ninja=false"
               RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
               SELECT_XCODE: /Applications/Xcode_13.4.1.app
@@ -442,33 +442,33 @@ jobs:
           - name: dist-x86_64-msvc
             env:
               RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-msvc --host=x86_64-pc-windows-msvc --target=x86_64-pc-windows-msvc --enable-full-tools --enable-profiler"
-              SCRIPT: PGO_HOST=x86_64-pc-windows-msvc python src/ci/stage-build.py python x.py dist bootstrap --include-default-paths
+              SCRIPT: PGO_HOST=x86_64-pc-windows-msvc python src/ci/stage-build.py python x.py dist bootstrap bootstrap-shim --include-default-paths
               DIST_REQUIRE_ALL_TOOLS: 1
             os: windows-2019-8core-32gb
           - name: dist-i686-msvc
             env:
               RUST_CONFIGURE_ARGS: "--build=i686-pc-windows-msvc --host=i686-pc-windows-msvc --target=i686-pc-windows-msvc,i586-pc-windows-msvc --enable-full-tools --enable-profiler"
-              SCRIPT: python x.py dist bootstrap --include-default-paths
+              SCRIPT: python x.py dist bootstrap bootstrap-shim --include-default-paths
               DIST_REQUIRE_ALL_TOOLS: 1
             os: windows-2019-8core-32gb
           - name: dist-aarch64-msvc
             env:
               RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-msvc --host=aarch64-pc-windows-msvc --enable-full-tools --enable-profiler"
-              SCRIPT: python x.py dist bootstrap --include-default-paths
+              SCRIPT: python x.py dist bootstrap bootstrap-shim --include-default-paths
               DIST_REQUIRE_ALL_TOOLS: 1
               WINDOWS_SDK_20348_HACK: 1
             os: windows-2019-8core-32gb
           - name: dist-i686-mingw
             env:
               RUST_CONFIGURE_ARGS: "--build=i686-pc-windows-gnu --enable-full-tools --enable-profiler"
               NO_DOWNLOAD_CI_LLVM: 1
-              SCRIPT: python x.py dist bootstrap --include-default-paths
+              SCRIPT: python x.py dist bootstrap bootstrap-shim --include-default-paths
               CUSTOM_MINGW: 1
               DIST_REQUIRE_ALL_TOOLS: 1
             os: windows-2019-8core-32gb
           - name: dist-x86_64-mingw
             env:
-              SCRIPT: python x.py dist bootstrap --include-default-paths
+              SCRIPT: python x.py dist bootstrap bootstrap-shim --include-default-paths
               RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-gnu --enable-full-tools --enable-profiler"
               NO_DOWNLOAD_CI_LLVM: 1
               CUSTOM_MINGW: 1
@@ -477,7 +477,7 @@ jobs:
           - name: dist-x86_64-msvc-alt
             env:
               RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-msvc --enable-extended --enable-profiler"
-              SCRIPT: python x.py dist bootstrap --include-default-paths
+              SCRIPT: python x.py dist bootstrap bootstrap-shim --include-default-paths
             os: windows-2019-8core-32gb
     timeout-minutes: 600
     runs-on: "${{ matrix.os }}"

src/bootstrap/Cargo.lock

@@ -49,6 +49,7 @@ dependencies = [
  "cmake",
  "fd-lock",
  "filetime",
+ "getopts",
  "hex",
  "ignore",
  "is-terminal",
@@ -317,6 +318,15 @@ dependencies = [
  "version_check",
 ]
 
+[[package]]
+name = "getopts"
+version = "0.2.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "14dbbfd5c71d70241ecf9e6f13737f7b5ce823821063188d7e46c41d371eebd5"
+dependencies = [
+ "unicode-width",
+]
+
 [[package]]
 name = "globset"
 version = "0.4.8"
@@ -778,6 +788,12 @@ version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d22af068fba1eb5edcb4aea19d382b2a3deb4c8f9d475c589b6ada9e0fd493ee"
 
+[[package]]
+name = "unicode-width"
+version = "0.1.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b"
+
 [[package]]
 name = "version_check"
 version = "0.9.4"

src/bootstrap/Cargo.toml

@@ -14,6 +14,11 @@ name = "bootstrap"
 path = "bin/main.rs"
 test = false
 
+[[bin]]
+name = "bootstrap-shim"
+path = "bin/bootstrap-shim.rs"
+test = false
+
 [[bin]]
 name = "rustc"
 path = "bin/rustc.rs"
@@ -35,6 +40,7 @@ build_helper = { path = "../tools/build_helper" }
 cmake = "0.1.38"
 filetime = "0.2"
 cc = "1.0.69"
+getopts = "0.2"
 libc = "0.2"
 hex = "0.4"
 object = { version = "0.29.0", default-features = false, features = ["archive", "coff", "read_core", "unaligned"] }

src/bootstrap/bin/bootstrap-shim.rs

@@ -0,0 +1,79 @@
+use std::{
+    env, io,
+    process::{self, Command, ExitStatus},
+};
+
+use bootstrap::{Flags, MinimalConfig};
+
+#[path = "../../../src/tools/x/src/main.rs"]
+mod x;
+
+/// We are planning to exclude python logic from x script by executing bootstrap-shim
+/// immediately. Since `find_and_run_available_bootstrap_script` executes x script first,
+/// any changes on bootstrap will not be seen. To prevent this problem, in bootstrap-shim
+/// we want to call the python script directly.
+fn find_and_run_py_bootstrap_script() {
+    #[cfg(unix)]
+    fn exec_or_status(command: &mut Command) -> io::Result<ExitStatus> {
+        use std::os::unix::process::CommandExt;
+        Err(command.exec())
+    }
+
+    #[cfg(not(unix))]
+    fn exec_or_status(command: &mut Command) -> io::Result<ExitStatus> {
+        command.status()
+    }
+
+    let current_path = match env::current_dir() {
+        Ok(dir) => dir,
+        Err(err) => {
+            eprintln!("Failed to get current directory: {err}");
+            process::exit(1);
+        }
+    };
+
+    for dir in current_path.ancestors() {
+        let candidate = dir.join("x.py");
+        if candidate.exists() {
+            let mut cmd: Command;
+            cmd = Command::new(x::python());
+            cmd.arg(&candidate).args(env::args().skip(1)).current_dir(dir);
+            let result = exec_or_status(&mut cmd);
+
+            match result {
+                Err(error) => {
+                    eprintln!("Failed to invoke `{:?}`: {}", cmd, error);
+                }
+                Ok(status) => {
+                    process::exit(status.code().unwrap_or(1));
+                }
+            }
+        }
+    }
+}
+
+fn main() {
+    let args = env::args().skip(1).collect::<Vec<_>>();
+    let flags = Flags::parse(&args);
+
+    // If there are no untracked changes to bootstrap, download it from CI.
+    // Otherwise, build it from source. Use python to build to avoid duplicating the code between python and rust.
+    let config = MinimalConfig::parse(&flags, None);
+    let bootstrap_bin = if let Some(commit) = last_modified_bootstrap_commit(&config) {
+        config.download_bootstrap(&commit)
+    } else {
+        return find_and_run_py_bootstrap_script();
+    };
+
+    let args: Vec<_> = std::env::args().skip(1).collect();
+    println!("Running pre-compiled bootstrap binary");
+    Command::new(bootstrap_bin).args(args).status().expect("failed to spawn bootstrap binairy");
+}
+
+fn last_modified_bootstrap_commit(config: &MinimalConfig) -> Option<String> {
+    config.last_modified_commit(
+        &["src/bootstrap", "src/tools/build_helper", "src/tools/x"],
+        "download-bootstrap",
+        true,
+    )
+}

src/bootstrap/bin/main.rs

@@ -13,7 +13,7 @@ use bootstrap::{Build, Config, Subcommand, VERSION};
 
 fn main() {
     let args = env::args().skip(1).collect::<Vec<_>>();
-    let config = Config::parse(&args);
+    let config = Config::parse(&args, None);
 
     #[cfg(all(any(unix, windows), not(target_os = "solaris")))]
     let mut build_lock;

src/bootstrap/builder.rs

@@ -703,7 +703,7 @@ impl<'a> Builder<'a> {
                 check::Rls,
                 check::RustAnalyzer,
                 check::Rustfmt,
-                check::Bootstrap
+                check::Bootstrap,
             ),
             Kind::Test => describe!(
                 crate::toolstate::ToolStateCheck,
@@ -808,6 +808,7 @@ impl<'a> Builder<'a> {
                 dist::LlvmTools,
                 dist::RustDev,
                 dist::Bootstrap,
+                dist::BootstrapShim,
                 dist::Extended,
                 // It seems that PlainSourceTarball somehow changes how some of the tools
                 // perceive their dependencies (see #93033) which would invalidate fingerprints

src/bootstrap/builder/tests.rs

@@ -7,7 +7,7 @@ fn configure(cmd: &str, host: &[&str], target: &[&str]) -> Config {
 }
 
 fn configure_with_args(cmd: &[String], host: &[&str], target: &[&str]) -> Config {
-    let mut config = Config::parse(cmd);
+    let mut config = Config::parse(cmd, None);
     // don't save toolstates
     config.save_toolstates = None;
     config.dry_run = DryRun::SelfCheck;
@@ -18,7 +18,7 @@ fn configure_with_args(cmd: &[String], host: &[&str], target: &[&str]) -> Config
     let submodule_build = Build::new(Config {
         // don't include LLVM, so CI doesn't require ninja/cmake to be installed
         rust_codegen_backends: vec![],
-        ..Config::parse(&["check".to_owned()])
+        ..Config::parse(&["check".to_owned()], None)
     });
     submodule_build.update_submodule(Path::new("src/doc/book"));
     submodule_build.update_submodule(Path::new("src/tools/rust-analyzer"));

src/bootstrap/check.rs

@@ -448,6 +448,7 @@ tool_check_step!(Rls, "src/tools/rls", SourceType::InTree);
 tool_check_step!(Rustfmt, "src/tools/rustfmt", SourceType::InTree);
 tool_check_step!(MiroptTestTools, "src/tools/miropt-test-tools", SourceType::InTree);
 
+// FIXME: currently  these are marked as ToolRustc, but they should be ToolBootstrap instead to avoid having to build the compiler first
 tool_check_step!(Bootstrap, "src/bootstrap", SourceType::InTree, false);
 
 /// Cargo's output path for the standard library in a given stage, compiled