From 535eb5fa03ee86e89affbf45a407bb488217b289 Mon Sep 17 00:00:00 2001 From: Rune Soerensen Date: Tue, 23 Apr 2024 08:41:38 -0400 Subject: [PATCH] Add bin/update_node_inventory --- Cargo.lock | 1 + common/nodejs-utils/Cargo.toml | 1 + .../src/bin/update_node_inventory.rs | 169 ++++++++++++++++++ common/nodejs-utils/src/lib.rs | 2 + 4 files changed, 173 insertions(+) create mode 100644 common/nodejs-utils/src/bin/update_node_inventory.rs diff --git a/Cargo.lock b/Cargo.lock index b6793015..67a841ee 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -653,6 +653,7 @@ dependencies = [ "serde", "serde-xml-rs", "serde_json", + "sha2", "tempfile", "thiserror", "toml", diff --git a/common/nodejs-utils/Cargo.toml b/common/nodejs-utils/Cargo.toml index afa927d8..3e129739 100644 --- a/common/nodejs-utils/Cargo.toml +++ b/common/nodejs-utils/Cargo.toml @@ -20,6 +20,7 @@ regex = "1" serde = { version = "1", features = ['derive'] } serde_json = "1" serde-xml-rs = "0.6" +sha2 = "0.10.8" thiserror = "1" toml = "0.8" ureq = { version = "2", features = ["json"] } diff --git a/common/nodejs-utils/src/bin/update_node_inventory.rs b/common/nodejs-utils/src/bin/update_node_inventory.rs new file mode 100644 index 00000000..a8195a91 --- /dev/null +++ b/common/nodejs-utils/src/bin/update_node_inventory.rs @@ -0,0 +1,169 @@ +// Required due to: https://github.com/rust-lang/rust/issues/95513 +#![allow(unused_crate_dependencies)] + +use anyhow::{Context, Result}; +use heroku_inventory_utils::{ + checksum::Checksum, + inv::{read_inventory_file, Arch, Artifact, Inventory, Os}, +}; +use node_semver::Version; +use serde::Deserialize; +use sha2::Sha256; +use std::{ + collections::{HashMap, HashSet}, + env, fs, process, +}; + +/// Updates the local node.js inventory.toml with versions published on nodejs.org. +fn main() { + let inventory_path = env::args().nth(1).unwrap_or_else(|| { + eprintln!("Usage: update_inventory "); + process::exit(2); + }); + + let inventory_artifacts: HashSet> = + read_inventory_file(&inventory_path) + .unwrap_or_else(|e| { + eprintln!("Error reading inventory at '{inventory_path}': {e}"); + std::process::exit(1); + }) + .artifacts + .into_iter() + .collect(); + + // List available upstream artifacts. + let remote_artifacts = list_upstream_artifacts().unwrap_or_else(|e| { + eprintln!("Failed to fetch upstream go versions: {e}"); + process::exit(4); + }); + + let inventory = Inventory { + artifacts: remote_artifacts, + }; + + let toml = toml::to_string(&inventory).unwrap_or_else(|e| { + eprintln!("Error serializing inventory as toml: {e}"); + process::exit(6); + }); + + fs::write(inventory_path, toml).unwrap_or_else(|e| { + eprintln!("Error writing inventory to file: {e}"); + process::exit(7); + }); + + let remote_artifacts: HashSet> = + inventory.artifacts.into_iter().collect(); + + [ + ("Added", &remote_artifacts - &inventory_artifacts), + ("Removed", &inventory_artifacts - &remote_artifacts), + ] + .iter() + .filter(|(_, artifact_diff)| !artifact_diff.is_empty()) + .for_each(|(action, artifacts)| { + let mut list: Vec<&Artifact> = artifacts.iter().collect(); + list.sort_by_key(|a| &a.version); + println!( + "{} {}.", + action, + list.iter() + .map(ToString::to_string) + .collect::>() + .join(", ") + ); + }); +} + +pub(crate) fn list_upstream_artifacts() -> Result>, anyhow::Error> { + let target_version = Version::parse("0.8.6").context("Failed to parse version")?; + + list_releases()? + .into_iter() + .filter(|release| release.version >= target_version) + .map(|release| get_release_artifacts(&release)) + .collect::>>() + .map(|nested| nested.into_iter().flatten().collect()) +} + +fn get_release_artifacts(release: &NodeJSRelease) -> Result>> { + let supported_platforms = HashMap::from([ + ("linux-arm64", (Os::Linux, Arch::Arm64)), + ("linux-x64", (Os::Linux, Arch::Amd64)), + ]); + + let shasums = fetch_checksums(&release.version)?; + release + .files + .iter() + .filter(|file| supported_platforms.contains_key(&file.as_str())) + .map(|file| { + let (os, arch) = supported_platforms + .get(file.as_str()) + .ok_or_else(|| anyhow::anyhow!("Unsupported platform: {}", file))?; + + let filename = format!("node-v{}-{}.tar.gz", release.version, file); + let checksum_hex = shasums + .get(&filename) + .ok_or_else(|| anyhow::anyhow!("Checksum not found for {}", filename))?; + + Ok(Artifact:: { + url: format!( + "https://nodejs.org/download/release/v{}/{filename}", + release.version + ), + version: release.version.clone(), + checksum: Checksum::try_from(checksum_hex.to_owned())?, + arch: *arch, + os: *os, + }) + }) + .collect() +} + +fn fetch_checksums(version: &Version) -> Result> { + ureq::get(&format!( + "https://nodejs.org/download/release/v{version}/SHASUMS256.txt" + )) + .call()? + .into_string() + .map_err(anyhow::Error::from) + .map(|x| parse_shasums(&x)) +} + +// Parses a SHASUMS256.txt file into a map of filename to checksum. +// Lines are expected to be of the form ` `. +fn parse_shasums(input: &str) -> HashMap { + input + .lines() + .filter_map(|line| { + let mut parts = line.split_whitespace(); + match (parts.next(), parts.next()) { + (Some(checksum), Some(filename)) if parts.next().is_none() => { + Some(( + // Some of the checksum filenames contain a leading `./` (e.g. + // https://nodejs.org/download/release/v0.11.6/SHASUMS256.txt) + filename.trim_start_matches("./").to_string(), + checksum.to_string(), + )) + } + _ => None, + } + }) + .collect() +} + +const NODE_UPSTREAM_LIST_URL: &str = "https://nodejs.org/download/release/index.json"; + +#[derive(Deserialize, Debug)] +pub(crate) struct NodeJSRelease { + pub(crate) version: Version, + pub(crate) files: Vec, +} + +pub(crate) fn list_releases() -> Result> { + ureq::get(NODE_UPSTREAM_LIST_URL) + .call() + .context("Failed to fetch nodejs.org release list")? + .into_json::>() + .context("Failed to parse nodejs.org release list from JSON") +} diff --git a/common/nodejs-utils/src/lib.rs b/common/nodejs-utils/src/lib.rs index 8f61355f..db3b47eb 100644 --- a/common/nodejs-utils/src/lib.rs +++ b/common/nodejs-utils/src/lib.rs @@ -1,3 +1,5 @@ +use sha2 as _; + pub mod application; pub mod distribution; pub mod inv;