From de038b746e267a4270cf4ac038fbfd92ce788526 Mon Sep 17 00:00:00 2001 From: Simon Sapin Date: Thu, 31 Aug 2017 16:37:14 +0200 Subject: [PATCH 1/4] Add full git commit hash to release channel manifests The full hash is necessary to build the download URL for "alternate" compiler builds. This is a first step for https://github.com/rust-lang-nursery/rustup.rs/issues/1099 --- src/bootstrap/dist.rs | 2 ++ src/bootstrap/lib.rs | 5 +++ src/tools/build-manifest/src/main.rs | 53 +++++++++++++++++----------- 3 files changed, 40 insertions(+), 20 deletions(-) diff --git a/src/bootstrap/dist.rs b/src/bootstrap/dist.rs index 746f85a9d59d6..b7fcfc75c4f25 100644 --- a/src/bootstrap/dist.rs +++ b/src/bootstrap/dist.rs @@ -364,7 +364,9 @@ impl Step for Rustc { cp("README.md"); // tiny morsel of metadata is used by rust-packaging let version = build.rust_version(); + let sha = build.rust_sha().unwrap_or(""); t!(t!(File::create(overlay.join("version"))).write_all(version.as_bytes())); + t!(t!(File::create(overlay.join("git-commit-hash"))).write_all(sha.as_bytes())); // On MinGW we've got a few runtime DLL dependencies that we need to // include. The first argument to this script is where to put these DLLs diff --git a/src/bootstrap/lib.rs b/src/bootstrap/lib.rs index f21b382619d0a..67791e8758c0b 100644 --- a/src/bootstrap/lib.rs +++ b/src/bootstrap/lib.rs @@ -797,6 +797,11 @@ impl Build { self.rust_info.version(self, channel::CFG_RELEASE_NUM) } + /// Return the full commit hash + fn rust_sha(&self) -> Option<&str> { + self.rust_info.sha() + } + /// Returns the `a.b.c` version that the given package is at. fn release_num(&self, package: &str) -> String { let mut toml = String::new(); diff --git a/src/tools/build-manifest/src/main.rs b/src/tools/build-manifest/src/main.rs index e2be021e7cc39..39b5d275dffaa 100644 --- a/src/tools/build-manifest/src/main.rs +++ b/src/tools/build-manifest/src/main.rs @@ -107,6 +107,7 @@ static MINGW: &'static [&'static str] = &[ struct Manifest { manifest_version: String, date: String, + git_commit_hash: String, pkg: BTreeMap, } @@ -205,15 +206,10 @@ impl Builder { self.digest_and_sign(); let manifest = self.build_manifest(); - let filename = format!("channel-rust-{}.toml", self.rust_release); - self.write_manifest(&toml::to_string(&manifest).unwrap(), &filename); - - let filename = format!("channel-rust-{}-date.txt", self.rust_release); - self.write_date_stamp(&manifest.date, &filename); + self.write_channel_files(&self.rust_release, &manifest); if self.rust_release != "beta" && self.rust_release != "nightly" { - self.write_manifest(&toml::to_string(&manifest).unwrap(), "channel-rust-stable.toml"); - self.write_date_stamp(&manifest.date, "channel-rust-stable-date.txt"); + self.write_channel_files("stable", &manifest); } } @@ -230,6 +226,7 @@ impl Builder { let mut manifest = Manifest { manifest_version: "2".to_string(), date: self.date.to_string(), + git_commit_hash: self.git_commit_hash("rust", "x86_64-unknown-linux-gnu"), pkg: BTreeMap::new(), }; @@ -382,14 +379,31 @@ impl Builder { .arg(self.input.join(&filename)) .arg(format!("{}/version", filename.replace(".tar.gz", ""))) .arg("-O"); - let version = t!(cmd.output()); - if !version.status.success() { + let output = t!(cmd.output()); + if !output.status.success() { panic!("failed to learn version:\n\n{:?}\n\n{}\n\n{}", cmd, - String::from_utf8_lossy(&version.stdout), - String::from_utf8_lossy(&version.stderr)); + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr)); + } + String::from_utf8_lossy(&output.stdout).trim().to_string() + } + + fn git_commit_hash(&self, component: &str, target: &str) -> String { + let mut cmd = Command::new("tar"); + let filename = self.filename(component, target); + cmd.arg("xf") + .arg(self.input.join(&filename)) + .arg(format!("{}/git-commit-hash", filename.replace(".tar.gz", ""))) + .arg("-O"); + let output = t!(cmd.output()); + if !output.status.success() { + panic!("failed to learn git commit hash:\n\n{:?}\n\n{}\n\n{}", + cmd, + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr)); } - String::from_utf8_lossy(&version.stdout).trim().to_string() + String::from_utf8_lossy(&output.stdout).trim().to_string() } fn hash(&self, path: &Path) -> String { @@ -425,16 +439,15 @@ impl Builder { assert!(t!(child.wait()).success()); } - fn write_manifest(&self, manifest: &str, name: &str) { - let dst = self.output.join(name); - t!(t!(File::create(&dst)).write_all(manifest.as_bytes())); - self.hash(&dst); - self.sign(&dst); + fn write_channel_files(&self, channel_name: &str, manifest: &Manifest) { + self.write(&toml::to_string(&manifest).unwrap(), channel_name, ".toml"); + self.write(&manifest.date, channel_name, "-date.txt"); + self.write(&manifest.git_commit_hash, channel_name, "-git-commit-hash.txt"); } - fn write_date_stamp(&self, date: &str, name: &str) { - let dst = self.output.join(name); - t!(t!(File::create(&dst)).write_all(date.as_bytes())); + fn write(&self, contents: &str, channel_name: &str, suffix: &str) { + let dst = self.output.join(format!("channel-rust-{}{}", channel_name, suffix)); + t!(t!(File::create(&dst)).write_all(contents.as_bytes())); self.hash(&dst); self.sign(&dst); } From 9abc549aa1ce16bf4099a56a91e9dbf6af2baa24 Mon Sep 17 00:00:00 2001 From: Simon Sapin Date: Fri, 1 Sep 2017 09:08:59 +0200 Subject: [PATCH 2/4] Add git-commit-hash in source and extended tarballs too. --- src/bootstrap/dist.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/bootstrap/dist.rs b/src/bootstrap/dist.rs index b7fcfc75c4f25..07ef043072096 100644 --- a/src/bootstrap/dist.rs +++ b/src/bootstrap/dist.rs @@ -846,6 +846,8 @@ impl Step for PlainSourceTarball { // Create the version file write_file(&plain_dst_src.join("version"), build.rust_version().as_bytes()); + let sha = build.rust_sha().unwrap_or(""); + write_file(&plain_dst_src.join("git-commit-hash"), sha.as_bytes()); // If we're building from git sources, we need to vendor a complete distribution. if build.rust_info.is_git() { @@ -1158,7 +1160,9 @@ impl Step for Extended { install(&build.src.join("LICENSE-APACHE"), &overlay, 0o644); install(&build.src.join("LICENSE-MIT"), &overlay, 0o644); let version = build.rust_version(); + let sha = build.rust_sha().unwrap_or(""); t!(t!(File::create(overlay.join("version"))).write_all(version.as_bytes())); + t!(t!(File::create(overlay.join("git-commit-hash"))).write_all(sha.as_bytes())); install(&etc.join("README.md"), &overlay, 0o644); // When rust-std package split from rustc, we needed to ensure that during From 9412fd7371bfb8cc8e6b20240b84168a88ac9d21 Mon Sep 17 00:00:00 2001 From: Simon Sapin Date: Mon, 4 Sep 2017 16:29:57 +0200 Subject: [PATCH 3/4] Only include git-commit-hash in tarballs when available MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit … instead of writing an empty file. --- src/bootstrap/dist.rs | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/bootstrap/dist.rs b/src/bootstrap/dist.rs index 07ef043072096..2920b91b25530 100644 --- a/src/bootstrap/dist.rs +++ b/src/bootstrap/dist.rs @@ -364,9 +364,10 @@ impl Step for Rustc { cp("README.md"); // tiny morsel of metadata is used by rust-packaging let version = build.rust_version(); - let sha = build.rust_sha().unwrap_or(""); t!(t!(File::create(overlay.join("version"))).write_all(version.as_bytes())); - t!(t!(File::create(overlay.join("git-commit-hash"))).write_all(sha.as_bytes())); + if let Some(sha) = build.rust_sha() { + t!(t!(File::create(overlay.join("git-commit-hash"))).write_all(sha.as_bytes())); + } // On MinGW we've got a few runtime DLL dependencies that we need to // include. The first argument to this script is where to put these DLLs @@ -846,8 +847,9 @@ impl Step for PlainSourceTarball { // Create the version file write_file(&plain_dst_src.join("version"), build.rust_version().as_bytes()); - let sha = build.rust_sha().unwrap_or(""); - write_file(&plain_dst_src.join("git-commit-hash"), sha.as_bytes()); + if let Some(sha) = build.rust_sha() { + write_file(&plain_dst_src.join("git-commit-hash"), sha.as_bytes()); + } // If we're building from git sources, we need to vendor a complete distribution. if build.rust_info.is_git() { @@ -1160,9 +1162,10 @@ impl Step for Extended { install(&build.src.join("LICENSE-APACHE"), &overlay, 0o644); install(&build.src.join("LICENSE-MIT"), &overlay, 0o644); let version = build.rust_version(); - let sha = build.rust_sha().unwrap_or(""); t!(t!(File::create(overlay.join("version"))).write_all(version.as_bytes())); - t!(t!(File::create(overlay.join("git-commit-hash"))).write_all(sha.as_bytes())); + if let Some(sha) = build.rust_sha() { + t!(t!(File::create(overlay.join("git-commit-hash"))).write_all(sha.as_bytes())); + } install(&etc.join("README.md"), &overlay, 0o644); // When rust-std package split from rustc, we needed to ensure that during From f912d771badbe981d7caa89b754248eb141e14c3 Mon Sep 17 00:00:00 2001 From: Simon Sapin Date: Mon, 4 Sep 2017 17:31:29 +0200 Subject: [PATCH 4/4] Make git commit info optional and per-package in channel manifests At the moment it is always missing for Cargo and RLS. Their respective build systems need to be modified to include `git-commit-hash` files in their "dist" tarballs. --- src/tools/build-manifest/src/main.rs | 43 +++++++++++++++++++++------- 1 file changed, 33 insertions(+), 10 deletions(-) diff --git a/src/tools/build-manifest/src/main.rs b/src/tools/build-manifest/src/main.rs index 39b5d275dffaa..0e91fa9c6022b 100644 --- a/src/tools/build-manifest/src/main.rs +++ b/src/tools/build-manifest/src/main.rs @@ -107,13 +107,13 @@ static MINGW: &'static [&'static str] = &[ struct Manifest { manifest_version: String, date: String, - git_commit_hash: String, pkg: BTreeMap, } #[derive(Serialize)] struct Package { version: String, + git_commit_hash: Option, target: BTreeMap, } @@ -168,6 +168,9 @@ struct Builder { rust_version: String, cargo_version: String, rls_version: String, + rust_git_commit_hash: Option, + cargo_git_commit_hash: Option, + rls_git_commit_hash: Option, } fn main() { @@ -195,6 +198,9 @@ fn main() { rust_version: String::new(), cargo_version: String::new(), rls_version: String::new(), + rust_git_commit_hash: None, + cargo_git_commit_hash: None, + rls_git_commit_hash: None, }.build(); } @@ -203,6 +209,9 @@ impl Builder { self.rust_version = self.version("rust", "x86_64-unknown-linux-gnu"); self.cargo_version = self.version("cargo", "x86_64-unknown-linux-gnu"); self.rls_version = self.version("rls", "x86_64-unknown-linux-gnu"); + self.rust_git_commit_hash = self.git_commit_hash("rust", "x86_64-unknown-linux-gnu"); + self.cargo_git_commit_hash = self.git_commit_hash("cargo", "x86_64-unknown-linux-gnu"); + self.rls_git_commit_hash = self.git_commit_hash("rls", "x86_64-unknown-linux-gnu"); self.digest_and_sign(); let manifest = self.build_manifest(); @@ -226,7 +235,6 @@ impl Builder { let mut manifest = Manifest { manifest_version: "2".to_string(), date: self.date.to_string(), - git_commit_hash: self.git_commit_hash("rust", "x86_64-unknown-linux-gnu"), pkg: BTreeMap::new(), }; @@ -246,6 +254,7 @@ impl Builder { let mut pkg = Package { version: self.cached_version("rust").to_string(), + git_commit_hash: self.cached_git_commit_hash("rust").clone(), target: BTreeMap::new(), }; for host in HOSTS { @@ -339,6 +348,7 @@ impl Builder { dst.insert(pkgname.to_string(), Package { version: self.cached_version(pkgname).to_string(), + git_commit_hash: self.cached_git_commit_hash(pkgname).clone(), target: targets, }); } @@ -372,6 +382,16 @@ impl Builder { } } + fn cached_git_commit_hash(&self, component: &str) -> &Option { + if component == "cargo" { + &self.cargo_git_commit_hash + } else if component == "rls" || component == "rls-preview" { + &self.rls_git_commit_hash + } else { + &self.rust_git_commit_hash + } + } + fn version(&self, component: &str, target: &str) -> String { let mut cmd = Command::new("tar"); let filename = self.filename(component, target); @@ -389,7 +409,7 @@ impl Builder { String::from_utf8_lossy(&output.stdout).trim().to_string() } - fn git_commit_hash(&self, component: &str, target: &str) -> String { + fn git_commit_hash(&self, component: &str, target: &str) -> Option { let mut cmd = Command::new("tar"); let filename = self.filename(component, target); cmd.arg("xf") @@ -397,13 +417,15 @@ impl Builder { .arg(format!("{}/git-commit-hash", filename.replace(".tar.gz", ""))) .arg("-O"); let output = t!(cmd.output()); - if !output.status.success() { - panic!("failed to learn git commit hash:\n\n{:?}\n\n{}\n\n{}", - cmd, - String::from_utf8_lossy(&output.stdout), - String::from_utf8_lossy(&output.stderr)); + if output.status.success() { + Some(String::from_utf8_lossy(&output.stdout).trim().to_string()) + } else { + // This is always called after `.version()`. + // So if that didn’t fail but this does, + // that’s very probably because the tarball is valid + // but does not contain a `git-commit-hash` file. + None } - String::from_utf8_lossy(&output.stdout).trim().to_string() } fn hash(&self, path: &Path) -> String { @@ -442,7 +464,8 @@ impl Builder { fn write_channel_files(&self, channel_name: &str, manifest: &Manifest) { self.write(&toml::to_string(&manifest).unwrap(), channel_name, ".toml"); self.write(&manifest.date, channel_name, "-date.txt"); - self.write(&manifest.git_commit_hash, channel_name, "-git-commit-hash.txt"); + self.write(manifest.pkg["rust"].git_commit_hash.as_ref().unwrap(), + channel_name, "-git-commit-hash.txt"); } fn write(&self, contents: &str, channel_name: &str, suffix: &str) {