From 33e7f9bc6681e32cc3b3b26ea85e21da225b89ee Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Mon, 17 Feb 2025 10:20:14 +0100 Subject: [PATCH] generate-copyright: pass the vendored sources from bootstrap --- src/bootstrap/src/core/build_steps/run.rs | 17 +++++++- src/bootstrap/src/core/build_steps/vendor.rs | 14 +++++-- src/bootstrap/src/core/builder/cargo.rs | 3 +- src/bootstrap/src/lib.rs | 7 ++++ .../generate-copyright/src/cargo_metadata.rs | 40 ++----------------- src/tools/generate-copyright/src/main.rs | 12 ++---- 6 files changed, 41 insertions(+), 52 deletions(-) diff --git a/src/bootstrap/src/core/build_steps/run.rs b/src/bootstrap/src/core/build_steps/run.rs index 3f5e701e7e17a..2b17e02cae5a4 100644 --- a/src/bootstrap/src/core/build_steps/run.rs +++ b/src/bootstrap/src/core/build_steps/run.rs @@ -9,7 +9,7 @@ use crate::Mode; use crate::core::build_steps::dist::distdir; use crate::core::build_steps::test; use crate::core::build_steps::tool::{self, SourceType, Tool}; -use crate::core::build_steps::vendor::default_paths_to_vendor; +use crate::core::build_steps::vendor::{Vendor, default_paths_to_vendor}; use crate::core::builder::{Builder, Kind, RunConfig, ShouldRun, Step}; use crate::core::config::TargetSelection; use crate::core::config::flags::get_completion; @@ -226,13 +226,26 @@ impl Step for GenerateCopyright { .collect::>() .join(","); + let vendored_sources = if let Some(path) = builder.vendored_crates_path() { + path + } else { + let cache_dir = builder.out.join("tmp").join("generate-copyright-vendor"); + builder.ensure(Vendor { + sync_args: Vec::new(), + versioned_dirs: true, + root_dir: builder.src.clone(), + output_dir: cache_dir.clone(), + }); + cache_dir + }; + let mut cmd = builder.tool_cmd(Tool::GenerateCopyright); cmd.env("CARGO_MANIFESTS", &cargo_manifests); cmd.env("LICENSE_METADATA", &license_metadata); cmd.env("DEST", &dest); cmd.env("DEST_LIBSTD", &dest_libstd); - cmd.env("OUT_DIR", &builder.out); cmd.env("SRC_DIR", &builder.src); + cmd.env("VENDOR_DIR", &vendored_sources); cmd.env("CARGO", &builder.initial_cargo); // it is important that generate-copyright runs from the root of the // source tree, because it uses relative paths diff --git a/src/bootstrap/src/core/build_steps/vendor.rs b/src/bootstrap/src/core/build_steps/vendor.rs index 26d0f100ffd52..c68b55f358941 100644 --- a/src/bootstrap/src/core/build_steps/vendor.rs +++ b/src/bootstrap/src/core/build_steps/vendor.rs @@ -4,6 +4,8 @@ use crate::core::build_steps::tool::SUBMODULES_FOR_RUSTBOOK; use crate::core::builder::{Builder, RunConfig, ShouldRun, Step}; use crate::utils::exec::command; +pub const VENDOR_DIR: &str = "vendor"; + /// Returns the cargo workspaces to vendor for `x vendor` and dist tarballs. /// /// Returns a `Vec` of `(path_to_manifest, submodules_required)` where @@ -29,9 +31,10 @@ pub fn default_paths_to_vendor(builder: &Builder<'_>) -> Vec<(PathBuf, Vec<&'sta #[derive(Debug, Clone, Hash, PartialEq, Eq)] pub(crate) struct Vendor { - sync_args: Vec, - versioned_dirs: bool, - root_dir: PathBuf, + pub(crate) sync_args: Vec, + pub(crate) versioned_dirs: bool, + pub(crate) root_dir: PathBuf, + pub(crate) output_dir: PathBuf, } impl Step for Vendor { @@ -48,10 +51,13 @@ impl Step for Vendor { sync_args: run.builder.config.cmd.vendor_sync_args(), versioned_dirs: run.builder.config.cmd.vendor_versioned_dirs(), root_dir: run.builder.src.clone(), + output_dir: run.builder.src.join(VENDOR_DIR), }); } fn run(self, builder: &Builder<'_>) -> Self::Output { + builder.info(&format!("Vendoring sources to {:?}", self.root_dir)); + let mut cmd = command(&builder.initial_cargo); cmd.arg("vendor"); @@ -81,7 +87,7 @@ impl Step for Vendor { // which uses the unstable `public-dependency` feature. cmd.env("RUSTC_BOOTSTRAP", "1"); - cmd.current_dir(self.root_dir); + cmd.current_dir(self.root_dir).arg(&self.output_dir); cmd.run(builder); } diff --git a/src/bootstrap/src/core/builder/cargo.rs b/src/bootstrap/src/core/builder/cargo.rs index 59680af00622f..1ec3e601cad19 100644 --- a/src/bootstrap/src/core/builder/cargo.rs +++ b/src/bootstrap/src/core/builder/cargo.rs @@ -924,8 +924,7 @@ impl Builder<'_> { if self.config.rust_remap_debuginfo { let mut env_var = OsString::new(); - if self.config.vendor { - let vendor = self.build.src.join("vendor"); + if let Some(vendor) = self.build.vendored_crates_path() { env_var.push(vendor); env_var.push("=/rust/deps"); } else { diff --git a/src/bootstrap/src/lib.rs b/src/bootstrap/src/lib.rs index 7cd8aacf0d6c8..e4a6e2b55e73d 100644 --- a/src/bootstrap/src/lib.rs +++ b/src/bootstrap/src/lib.rs @@ -50,6 +50,8 @@ pub use utils::change_tracker::{ CONFIG_CHANGE_HISTORY, find_recent_config_change_ids, human_readable_changes, }; +use crate::core::build_steps::vendor::VENDOR_DIR; + const LLVM_TOOLS: &[&str] = &[ "llvm-cov", // used to generate coverage report "llvm-nm", // used to inspect binaries; it shows symbol names, their sizes and visibility @@ -782,6 +784,11 @@ impl Build { self.out.join(target).join("md-doc") } + /// Path to the vendored Rust crates. + fn vendored_crates_path(&self) -> Option { + if self.config.vendor { Some(self.src.join(VENDOR_DIR)) } else { None } + } + /// Returns `true` if this is an external version of LLVM not managed by bootstrap. /// In particular, we expect llvm sources to be available when this is false. /// diff --git a/src/tools/generate-copyright/src/cargo_metadata.rs b/src/tools/generate-copyright/src/cargo_metadata.rs index 16c5b5e7104e2..b717bd53eb1a7 100644 --- a/src/tools/generate-copyright/src/cargo_metadata.rs +++ b/src/tools/generate-copyright/src/cargo_metadata.rs @@ -11,10 +11,6 @@ pub enum Error { Io(#[from] std::io::Error), #[error("Failed get output from cargo-metadata: {0:?}")] GettingMetadata(#[from] cargo_metadata::Error), - #[error("Failed to run cargo vendor: {0:?}")] - LaunchingVendor(std::io::Error), - #[error("Failed to complete cargo vendor")] - RunningVendor, #[error("Bad path {0:?} whilst scraping files")] Scraping(PathBuf), } @@ -43,13 +39,11 @@ pub struct PackageMetadata { pub is_in_libstd: Option, } -/// Use `cargo metadata` and `cargo vendor` to get a list of dependencies and their license data. +/// Use `cargo metadata` to get a list of dependencies and their license data. License files will +/// also be pulled from the vendor path (generated by bootstrap). /// -/// This will involve running `cargo vendor` into `vendor_path` so we can -/// grab the license files. -/// -/// Any dependency with a path beginning with `root_path` is ignored, as we -/// assume `reuse` has covered it already. +/// Any dependency with a path beginning with `root_path` is ignored, as we assume `reuse` has +/// covered it already. pub fn get_metadata_and_notices( cargo: &Path, vendor_path: &Path, @@ -58,10 +52,6 @@ pub fn get_metadata_and_notices( ) -> Result, Error> { let mut output = get_metadata(cargo, root_path, manifest_paths)?; - // Now do a cargo-vendor and grab everything - println!("Vendoring deps into {}...", vendor_path.display()); - run_cargo_vendor(cargo, &vendor_path, manifest_paths)?; - // Now for each dependency we found, go and grab any important looking files for (package, metadata) in output.iter_mut() { load_important_files(package, metadata, &vendor_path)?; @@ -113,28 +103,6 @@ pub fn get_metadata( Ok(output) } -/// Run cargo-vendor, fetching into the given dir -fn run_cargo_vendor(cargo: &Path, dest: &Path, manifest_paths: &[PathBuf]) -> Result<(), Error> { - let mut vendor_command = std::process::Command::new(cargo); - vendor_command.env("RUSTC_BOOTSTRAP", "1"); - vendor_command.arg("vendor"); - vendor_command.arg("--quiet"); - vendor_command.arg("--versioned-dirs"); - for manifest_path in manifest_paths { - vendor_command.arg("-s"); - vendor_command.arg(manifest_path); - } - vendor_command.arg(dest); - - let vendor_status = vendor_command.status().map_err(Error::LaunchingVendor)?; - - if !vendor_status.success() { - return Err(Error::RunningVendor); - } - - Ok(()) -} - /// Add important files off disk into this dependency. /// /// Maybe one-day Cargo.toml will contain enough information that we don't need diff --git a/src/tools/generate-copyright/src/main.rs b/src/tools/generate-copyright/src/main.rs index 7b7cf0f4b6991..79e90d88f4446 100644 --- a/src/tools/generate-copyright/src/main.rs +++ b/src/tools/generate-copyright/src/main.rs @@ -17,8 +17,8 @@ mod cargo_metadata; fn main() -> Result<(), Error> { let dest_file = env_path("DEST")?; let libstd_dest_file = env_path("DEST_LIBSTD")?; - let out_dir = env_path("OUT_DIR")?; let src_dir = env_path("SRC_DIR")?; + let vendor_dir = env_path("VENDOR_DIR")?; let cargo = env_path("CARGO")?; let license_metadata = env_path("LICENSE_METADATA")?; @@ -39,16 +39,12 @@ fn main() -> Result<(), Error> { .collect::>(); // Scan Cargo dependencies - let mut collected_cargo_metadata = cargo_metadata::get_metadata_and_notices( - &cargo, - &out_dir.join("vendor"), - &src_dir, - &cargo_manifests, - )?; + let mut collected_cargo_metadata = + cargo_metadata::get_metadata_and_notices(&cargo, &vendor_dir, &src_dir, &cargo_manifests)?; let library_collected_cargo_metadata = cargo_metadata::get_metadata_and_notices( &cargo, - &out_dir.join("library-vendor"), + &vendor_dir, &src_dir, &library_manifests, )?;