From 8a0f1957982a6042ec550147243f811f25889d9c Mon Sep 17 00:00:00 2001 From: Weihang Lo Date: Thu, 23 May 2024 16:48:52 -0400 Subject: [PATCH 1/3] docs: add README.md for build_helper --- src/tools/build_helper/README.md | 1 + src/tools/build_helper/src/lib.rs | 2 ++ 2 files changed, 3 insertions(+) create mode 100644 src/tools/build_helper/README.md diff --git a/src/tools/build_helper/README.md b/src/tools/build_helper/README.md new file mode 100644 index 0000000000000..f81b631c3fdbf --- /dev/null +++ b/src/tools/build_helper/README.md @@ -0,0 +1 @@ +Types and functions shared across tools in this workspace. diff --git a/src/tools/build_helper/src/lib.rs b/src/tools/build_helper/src/lib.rs index 2abda5d3ebf27..d831cfa0d79a6 100644 --- a/src/tools/build_helper/src/lib.rs +++ b/src/tools/build_helper/src/lib.rs @@ -1,3 +1,5 @@ +//! Types and functions shared across tools in this workspace. + pub mod ci; pub mod git; pub mod metrics; From e24be071e3bef8dfb84b082f1478e9d33a108901 Mon Sep 17 00:00:00 2001 From: Weihang Lo Date: Thu, 23 May 2024 20:36:39 +0000 Subject: [PATCH 2/3] feat: vendor crates required by opt-dist to collect profiles These are the default package set required by opt-dist to correctly work, hence for people wanting to build a production grade of rustc in a sandboxed / air-gapped environment, these need to be vendored. The size of `rustc-src-nightly.tar.xz` before and after this change: * Before: 298M * After: 323M (+8%) These crates are the default set of packages required by opt-dist to correctly work, hence for people wanting to build a production grade of rustc in an sandboxed / air-gapped environment, these need to be vendored. The size of `rustc-src-nightly.tar.xz` before and after this change: * Before: 298M * After: 323M (+8%) Size change might or might not be a concern. See the previous discussion: https://github.com/rust-lang/rust/pull/125166#issuecomment-2113626468 Previous efforts on making: * https://github.com/rust-lang/rust/pull/125125 * https://github.com/rust-lang/rust/pull/125166 --- Note that extra works still need to be done to make it fully vendored. * The current pinned rustc-perf uses `tempfile::Tempdir` as the working directory when collecting profiles from some of these packages. This "tmp" working directory usage make it impossible for Cargo to pick up the correct vendor sources setting in `.cargo/config.toml` bundled in the rustc-src tarball. [^1] * opt-dist verifies the final built rustc against a subset of rustc test suite. However it rolls out its own `config.toml` without setting `vendor = true`, and that results in `./vendor/` directory removed. [^2] [^1]: https://github.com/rust-lang/rustc-perf/blob/4f313add609f43e928e98132358e8426ed3969ae/collector/src/compile/benchmark/mod.rs#L164-L173 [^2]: https://github.com/rust-lang/rust/blob/606afbb617a2949a4e35c4b0258ff94c980b9451/src/tools/opt-dist/src/tests.rs#L62-L77 --- src/bootstrap/src/core/build_steps/dist.rs | 15 ++++++++++++++ src/tools/build_helper/src/lib.rs | 23 ++++++++++++++++++++++ src/tools/opt-dist/src/training.rs | 22 +-------------------- 3 files changed, 39 insertions(+), 21 deletions(-) diff --git a/src/bootstrap/src/core/build_steps/dist.rs b/src/bootstrap/src/core/build_steps/dist.rs index 91039d0c8dcc1..60b1ff3e4413e 100644 --- a/src/bootstrap/src/core/build_steps/dist.rs +++ b/src/bootstrap/src/core/build_steps/dist.rs @@ -1041,6 +1041,21 @@ impl Step for PlainSourceTarball { .env("RUSTC_BOOTSTRAP", "1") .current_dir(plain_dst_src); + // Vendor packages that are required by opt-dist to collect PGO profiles. + let pkgs_for_pgo_training = build_helper::LLVM_PGO_CRATES + .iter() + .chain(build_helper::RUSTC_PGO_CRATES) + .map(|pkg| { + let mut manifest_path = + builder.src.join("./src/tools/rustc-perf/collector/compile-benchmarks"); + manifest_path.push(pkg); + manifest_path.push("Cargo.toml"); + manifest_path + }); + for manifest_path in pkgs_for_pgo_training { + cmd.arg("--sync").arg(manifest_path); + } + let config = if !builder.config.dry_run() { t!(String::from_utf8(t!(cmd.output()).stdout)) } else { diff --git a/src/tools/build_helper/src/lib.rs b/src/tools/build_helper/src/lib.rs index d831cfa0d79a6..15807d1c0d8f8 100644 --- a/src/tools/build_helper/src/lib.rs +++ b/src/tools/build_helper/src/lib.rs @@ -5,3 +5,26 @@ pub mod git; pub mod metrics; pub mod stage0_parser; pub mod util; + +/// The default set of crates for opt-dist to collect LLVM profiles. +pub const LLVM_PGO_CRATES: &[&str] = &[ + "syn-1.0.89", + "cargo-0.60.0", + "serde-1.0.136", + "ripgrep-13.0.0", + "regex-1.5.5", + "clap-3.1.6", + "hyper-0.14.18", +]; + +/// The default set of crates for opt-dist to collect rustc profiles. +pub const RUSTC_PGO_CRATES: &[&str] = &[ + "externs", + "ctfe-stress-5", + "cargo-0.60.0", + "token-stream-stress", + "match-stress", + "tuple-stress", + "diesel-1.4.8", + "bitmaps-3.1.0", +]; diff --git a/src/tools/opt-dist/src/training.rs b/src/tools/opt-dist/src/training.rs index 09263dc645e2d..89f4d8957c8bf 100644 --- a/src/tools/opt-dist/src/training.rs +++ b/src/tools/opt-dist/src/training.rs @@ -3,30 +3,10 @@ use crate::exec::{cmd, CmdBuilder}; use crate::utils::io::{count_files, delete_directory}; use crate::utils::with_log_group; use anyhow::Context; +use build_helper::{LLVM_PGO_CRATES, RUSTC_PGO_CRATES}; use camino::{Utf8Path, Utf8PathBuf}; use humansize::BINARY; -const LLVM_PGO_CRATES: &[&str] = &[ - "syn-1.0.89", - "cargo-0.60.0", - "serde-1.0.136", - "ripgrep-13.0.0", - "regex-1.5.5", - "clap-3.1.6", - "hyper-0.14.18", -]; - -const RUSTC_PGO_CRATES: &[&str] = &[ - "externs", - "ctfe-stress-5", - "cargo-0.60.0", - "token-stream-stress", - "match-stress", - "tuple-stress", - "diesel-1.4.8", - "bitmaps-3.1.0", -]; - fn init_compiler_benchmarks( env: &Environment, profiles: &[&str], From 377870386f99ee811e8945fade761a3ea8944db1 Mon Sep 17 00:00:00 2001 From: Weihang Lo Date: Sat, 25 May 2024 00:06:00 -0400 Subject: [PATCH 3/3] tidy: add license exceptions for rustc-perf `ring` is included because it is an optional dependency of `hyper`, which is a training data in rustc-pref for optimized build. The license of it is generally `ISC AND MIT AND OpenSSL`, though the `package.license` field is not set. See https://github.com/briansmith/ring/issues/902 `git+https://github.com/rust-lang/team` git source is from `rust_team_data`, which is used by `site` in src/tools/rustc-perf. This doesn't really a part of the compiler, so doesn't really affect the bootstrapping process. See https://github.com/rust-lang/rustc-perf/pull/1914. --- src/tools/tidy/src/deps.rs | 36 +++++++++++++++++++++++++++++++++++ src/tools/tidy/src/extdeps.rs | 6 +++++- 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/src/tools/tidy/src/deps.rs b/src/tools/tidy/src/deps.rs index efcd2a181ffa2..7b93b09dda24b 100644 --- a/src/tools/tidy/src/deps.rs +++ b/src/tools/tidy/src/deps.rs @@ -67,6 +67,7 @@ pub(crate) const WORKSPACES: &[(&str, ExceptionList, Option<(&[&str], &[&str])>) //("src/tools/miri/test-cargo-miri", &[], None), // FIXME uncomment once all deps are vendored //("src/tools/miri/test_dependencies", &[], None), // FIXME uncomment once all deps are vendored ("src/tools/rust-analyzer", EXCEPTIONS_RUST_ANALYZER, None), + ("src/tools/rustc-perf", EXCEPTIONS_RUSTC_PERF, None), ("src/tools/x", &[], None), // tidy-alphabetical-end ]; @@ -142,6 +143,22 @@ const EXCEPTIONS_RUST_ANALYZER: ExceptionList = &[ // tidy-alphabetical-end ]; +const EXCEPTIONS_RUSTC_PERF: ExceptionList = &[ + // tidy-alphabetical-start + ("alloc-no-stdlib", "BSD-3-Clause"), + ("alloc-stdlib", "BSD-3-Clause"), + ("brotli", "BSD-3-Clause/MIT"), + ("brotli-decompressor", "BSD-3-Clause/MIT"), + ("encoding_rs", "(Apache-2.0 OR MIT) AND BSD-3-Clause"), + ("inferno", "CDDL-1.0"), + ("instant", "BSD-3-Clause"), + ("ring", NON_STANDARD_LICENSE), // see EXCEPTIONS_NON_STANDARD_LICENSE_DEPS for more. + ("ryu", "Apache-2.0 OR BSL-1.0"), + ("snap", "BSD-3-Clause"), + ("subtle", "BSD-3-Clause"), + // tidy-alphabetical-end +]; + const EXCEPTIONS_CRANELIFT: ExceptionList = &[ // tidy-alphabetical-start ("cranelift-bforest", "Apache-2.0 WITH LLVM-exception"), @@ -178,6 +195,20 @@ const EXCEPTIONS_UEFI_QEMU_TEST: ExceptionList = &[ ("r-efi", "MIT OR Apache-2.0 OR LGPL-2.1-or-later"), // LGPL is not acceptible, but we use it under MIT OR Apache-2.0 ]; +/// Placeholder for non-standard license file. +const NON_STANDARD_LICENSE: &str = "NON_STANDARD_LICENSE"; + +/// These dependencies have non-standard licenses but are genenrally permitted. +const EXCEPTIONS_NON_STANDARD_LICENSE_DEPS: &[&str] = &[ + // `ring` is included because it is an optional dependency of `hyper`, + // which is a training data in rustc-perf for optimized build. + // The license of it is generally `ISC AND MIT AND OpenSSL`, + // though the `package.license` field is not set. + // + // See https://github.com/briansmith/ring/issues/902 + "ring", +]; + /// These are the root crates that are part of the runtime. The licenses for /// these and all their dependencies *must not* be in the exception list. const RUNTIME_CRATES: &[&str] = &["std", "core", "alloc", "test", "panic_abort", "panic_unwind"]; @@ -610,6 +641,11 @@ fn check_license_exceptions(metadata: &Metadata, exceptions: &[(&str, &str)], ba for pkg in metadata.packages.iter().filter(|p| p.name == *name) { match &pkg.license { None => { + if *license == NON_STANDARD_LICENSE + && EXCEPTIONS_NON_STANDARD_LICENSE_DEPS.contains(&pkg.name.as_str()) + { + continue; + } tidy_error!( bad, "dependency exception `{}` does not declare a license expression", diff --git a/src/tools/tidy/src/extdeps.rs b/src/tools/tidy/src/extdeps.rs index ff71ca537256f..2118de5f20411 100644 --- a/src/tools/tidy/src/extdeps.rs +++ b/src/tools/tidy/src/extdeps.rs @@ -4,7 +4,11 @@ use std::fs; use std::path::Path; /// List of allowed sources for packages. -const ALLOWED_SOURCES: &[&str] = &["\"registry+https://github.com/rust-lang/crates.io-index\""]; +const ALLOWED_SOURCES: &[&str] = &[ + r#""registry+https://github.com/rust-lang/crates.io-index""#, + // This is `rust_team_data` used by `site` in src/tools/rustc-perf, + r#""git+https://github.com/rust-lang/team#a5260e76d3aa894c64c56e6ddc8545b9a98043ec""#, +]; /// Checks for external package sources. `root` is the path to the directory that contains the /// workspace `Cargo.toml`.