From 451f6cf057b3122b45683b2fb806e66bdf1dd308 Mon Sep 17 00:00:00 2001 From: Mark Rousskov Date: Fri, 22 Dec 2023 10:44:10 -0500 Subject: [PATCH 1/3] Avoid copying/allocating String for TestResult --- src/db/mod.rs | 18 ++++++++++++++++++ src/results/db.rs | 19 +++++-------------- 2 files changed, 23 insertions(+), 14 deletions(-) diff --git a/src/db/mod.rs b/src/db/mod.rs index 39c48ca6..d7c1ab6d 100644 --- a/src/db/mod.rs +++ b/src/db/mod.rs @@ -191,6 +191,24 @@ pub trait QueryUtils { }) } + fn query_row Fallible>( + &self, + sql: &str, + params: impl rusqlite::Params, + func: F, + ) -> Fallible> { + self.with_conn(|conn| { + self.trace(sql, || { + let mut prepared = conn.prepare(sql)?; + let mut rows = prepared.query(params)?; + if let Ok(Some(row)) = rows.next() { + return Ok(Some(func(row)?)); + } + Ok(None) + }) + }) + } + fn trace T>(&self, sql: &str, f: F) -> T { let start = Instant::now(); let res = f(); diff --git a/src/results/db.rs b/src/results/db.rs index 0c0f1d99..2273e434 100644 --- a/src/results/db.rs +++ b/src/results/db.rs @@ -179,22 +179,13 @@ impl<'a> ReadResults for DatabaseDB<'a> { toolchain: &Toolchain, krate: &Crate, ) -> Fallible> { - let result: Option = self - .db - .query( - "SELECT result FROM results \ + Ok(self.db.query_row( + "SELECT result FROM results \ WHERE experiment = ?1 AND toolchain = ?2 AND crate = ?3 \ LIMIT 1;", - [&ex.name, &toolchain.to_string(), &krate.id()], - |row| row.get("result"), - )? - .pop(); - - if let Some(res) = result { - Ok(Some(res.parse()?)) - } else { - Ok(None) - } + [&ex.name, &toolchain.to_string(), &krate.id()], + |row| Ok(row.get_ref("result")?.as_str()?.parse::()?), + )?) } } From 7f6bb339a47a53b8b2e4aad0ae68f0b423b58e12 Mon Sep 17 00:00:00 2001 From: Mark Rousskov Date: Fri, 22 Dec 2023 10:48:45 -0500 Subject: [PATCH 2/3] Move to latest tera/minifier Also adds a test binary for running report generation locally. --- Cargo.lock | 238 +++++++++++++++++++++++++++-------------- Cargo.toml | 8 +- src/assets.rs | 2 +- src/bin/test-report.rs | 78 ++++++++++++++ src/db/mod.rs | 6 ++ src/experiments.rs | 6 +- 6 files changed, 249 insertions(+), 89 deletions(-) create mode 100644 src/bin/test-report.rs diff --git a/Cargo.lock b/Cargo.lock index 38c9d1b4..13cf4cb8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -153,7 +153,7 @@ dependencies = [ "log", "native-tls", "openssl", - "url 2.5.2", + "url", "wildmatch", ] @@ -189,7 +189,7 @@ dependencies = [ "time", "tokio", "tracing", - "url 2.5.2", + "url", "zeroize", ] @@ -224,7 +224,7 @@ dependencies = [ "http 0.2.12", "http-body 0.4.6", "once_cell", - "percent-encoding 2.3.1", + "percent-encoding", "pin-project-lite", "tracing", "uuid", @@ -258,11 +258,11 @@ dependencies = [ "http-body 0.4.6", "lru", "once_cell", - "percent-encoding 2.3.1", + "percent-encoding", "regex-lite", "sha2", "tracing", - "url 2.5.2", + "url", ] [[package]] @@ -352,7 +352,7 @@ dependencies = [ "http 1.1.0", "once_cell", "p256", - "percent-encoding 2.3.1", + "percent-encoding", "ring", "sha2", "subtle", @@ -419,7 +419,7 @@ dependencies = [ "http 0.2.12", "http-body 0.4.6", "once_cell", - "percent-encoding 2.3.1", + "percent-encoding", "pin-project-lite", "pin-utils", "tracing", @@ -728,6 +728,28 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "chrono-tz" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93698b29de5e97ad0ae26447b344c482a7284c737d9ddc5f9e52b74a336671bb" +dependencies = [ + "chrono", + "chrono-tz-build", + "phf", +] + +[[package]] +name = "chrono-tz-build" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c088aee841df9c3041febbb73934cfc39708749bf96dc827e3359cd39ef11b1" +dependencies = [ + "parse-zoneinfo", + "phf", + "phf_codegen", +] + [[package]] name = "clap" version = "4.5.16" @@ -851,7 +873,7 @@ dependencies = [ "mime", "minifier", "nix 0.27.1", - "percent-encoding 2.3.1", + "percent-encoding", "predicates", "prometheus", "r2d2", @@ -875,7 +897,7 @@ dependencies = [ "thiserror", "tokio", "toml 0.8.19", - "url 2.5.2", + "url", "walkdir", "warp", "zstd", @@ -1226,16 +1248,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "error-chain" -version = "0.12.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d2f06b9cac1506ece98fe3231e3cc9c4410ec3d5b1f24ae1c8946f0742cdefc" -dependencies = [ - "backtrace", - "version_check", -] - [[package]] name = "failure" version = "0.1.8" @@ -1351,7 +1363,7 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" dependencies = [ - "percent-encoding 2.3.1", + "percent-encoding", ] [[package]] @@ -1465,7 +1477,7 @@ dependencies = [ "log", "openssl-probe", "openssl-sys", - "url 2.5.2", + "url", ] [[package]] @@ -2179,7 +2191,7 @@ dependencies = [ "gix-path", "home", "thiserror", - "url 2.5.2", + "url", ] [[package]] @@ -2222,10 +2234,28 @@ dependencies = [ ] [[package]] -name = "glob" -version = "0.2.11" +name = "globset" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8be18de09a56b60ed0edf84bc9df007e30040691af7acd1c41874faac5895bfb" +checksum = "57da3b9b5b85bd66f31093f8c408b90a74431672542466497dcbdfdc02034be1" +dependencies = [ + "aho-corasick", + "bstr", + "log", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "globwalk" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf760ebf69878d9fd8f110c89703d90ce35095324d1f1edcb595c63945ee757" +dependencies = [ + "bitflags 2.6.0", + "ignore", + "walkdir", +] [[package]] name = "group" @@ -2415,9 +2445,12 @@ checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" [[package]] name = "humansize" -version = "1.1.1" +version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02296996cb8796d7c6e3bc2d9211b7802812d36999a51bb754123ead7d37d026" +checksum = "6cb51c9a029ddc91b07a787f1d86b53ccfa49b0e86688c946ebe8d3555685dd7" +dependencies = [ + "libm", +] [[package]] name = "humantime" @@ -2503,23 +2536,28 @@ dependencies = [ [[package]] name = "idna" -version = "0.1.5" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38f09e0f0b1fb55fdee1f17470ad800da77af5186a1a76c026b679358b7e844e" +checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" dependencies = [ - "matches", "unicode-bidi", "unicode-normalization", ] [[package]] -name = "idna" -version = "0.5.0" +name = "ignore" +version = "0.4.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" +checksum = "b46810df39e66e925525d6e38ce1e7f6e1d208f72dc39757880fcb66e2c58af1" dependencies = [ - "unicode-bidi", - "unicode-normalization", + "crossbeam-deque", + "globset", + "log", + "memchr", + "regex-automata", + "same-file", + "walkdir", + "winapi-util", ] [[package]] @@ -2650,6 +2688,12 @@ dependencies = [ "pkg-config", ] +[[package]] +name = "libm" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ec2a862134d2a7d32d7983ddcdd1c4923530833c9f2ea1a44fc5fa473989058" + [[package]] name = "libredox" version = "0.1.3" @@ -2739,12 +2783,6 @@ dependencies = [ "hashbrown", ] -[[package]] -name = "matches" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2532096657941c2fea9c289d370a250971c689d4f143798ff67113ec042024a5" - [[package]] name = "maybe-async" version = "0.2.10" @@ -2808,10 +2846,11 @@ dependencies = [ [[package]] name = "minifier" -version = "0.2.3" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5394aa376422b4b2b6c02fd9cfcb657e4ec544ae98e43d7d5d785fd0d042fd6d" +checksum = "9aa3f302fe0f8de065d4a2d1ed64f60204623cac58b80cd3c2a83a25d5a7d437" dependencies = [ + "clap", "regex", ] @@ -3082,10 +3121,13 @@ dependencies = [ ] [[package]] -name = "percent-encoding" -version = "1.0.1" +name = "parse-zoneinfo" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31010dd2e1ac33d5b46a5b413495239882813e0369f8ed8a5e266f173602f831" +checksum = "1f2a05b18d44e2957b88f96ba460715e295bc1d7510468a2f3d3b44535d26c24" +dependencies = [ + "regex", +] [[package]] name = "percent-encoding" @@ -3138,6 +3180,44 @@ dependencies = [ "sha2", ] +[[package]] +name = "phf" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ade2d8b8f33c7333b51bcf0428d37e217e9f32192ae4772156f65063b8ce03dc" +dependencies = [ + "phf_shared", +] + +[[package]] +name = "phf_codegen" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8d39688d359e6b34654d328e262234662d16cc0f60ec8dcbe5e718709342a5a" +dependencies = [ + "phf_generator", + "phf_shared", +] + +[[package]] +name = "phf_generator" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48e4cc64c2ad9ebe670cb8fd69dd50ae301650392e81c05f9bfcb2d5bdbc24b0" +dependencies = [ + "phf_shared", + "rand", +] + +[[package]] +name = "phf_shared" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90fcb95eef784c2ac79119d1dd819e162b5da872ce6f3c3abe1e8ca1c082f72b" +dependencies = [ + "siphasher", +] + [[package]] name = "pin-project" version = "1.1.5" @@ -3427,7 +3507,7 @@ dependencies = [ "mime", "native-tls", "once_cell", - "percent-encoding 2.3.1", + "percent-encoding", "pin-project-lite", "rustls-pemfile", "serde", @@ -3438,7 +3518,7 @@ dependencies = [ "tokio", "tokio-native-tls", "tower-service", - "url 2.5.2", + "url", "wasm-bindgen", "wasm-bindgen-futures", "web-sys", @@ -3590,7 +3670,7 @@ dependencies = [ "lazy_static", "log", "nix 0.25.1", - "percent-encoding 2.3.1", + "percent-encoding", "remove_dir_all", "scopeguard", "serde", @@ -3849,6 +3929,12 @@ dependencies = [ "rand_core", ] +[[package]] +name = "siphasher" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38b58827f4464d87d377d175e90bf58eb00fd8716ff0a62f80356b5e61555d0d" + [[package]] name = "slab" version = "0.4.9" @@ -4028,23 +4114,24 @@ dependencies = [ [[package]] name = "tera" -version = "0.11.20" +version = "1.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b505279e19d8f7d24b1a9dc58327c9c36174b1a2c7ebdeac70792d017cb64f3" +checksum = "ab9d851b45e865f178319da0abdbfe6acbc4328759ff18dafc3a41c16b4cd2ee" dependencies = [ "chrono", - "error-chain", - "glob", + "chrono-tz", + "globwalk", "humansize", "lazy_static", + "percent-encoding", "pest", "pest_derive", + "rand", "regex", "serde", "serde_json", "slug", "unic-segment", - "url 1.7.2", ] [[package]] @@ -4304,7 +4391,7 @@ dependencies = [ "rand", "sha1", "thiserror", - "url 2.5.2", + "url", "utf-8", ] @@ -4331,39 +4418,39 @@ dependencies = [ [[package]] name = "unic-char-property" -version = "0.7.0" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce36d3f7ce754afdbccccf8ff0dd0134e50fb44aaae579f96218856e9e5dbd1e" +checksum = "a8c57a407d9b6fa02b4795eb81c5b6652060a15a7903ea981f3d723e6c0be221" dependencies = [ "unic-char-range", ] [[package]] name = "unic-char-range" -version = "0.7.0" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9ab85fab42ad1b26cafc03bf891f69cb4d6e15f491030e89a0122197baa8ae8" +checksum = "0398022d5f700414f6b899e10b8348231abf9173fa93144cbc1a43b9793c1fbc" [[package]] name = "unic-common" -version = "0.7.0" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff8d4a7ade929ef7d971e16ced21a8cd56a63869aa6032dfb8cb083cf7d077bf" +checksum = "80d7ff825a6a654ee85a63e80f92f054f904f21e7d12da4e22f9834a4aaa35bc" [[package]] name = "unic-segment" -version = "0.7.0" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9ca47cbb09fb5fcd066b5867d11dc528302fa465277882797d6a836e1ee6f9e" +checksum = "e4ed5d26be57f84f176157270c112ef57b86debac9cd21daaabbe56db0f88f23" dependencies = [ "unic-ucd-segment", ] [[package]] name = "unic-ucd-segment" -version = "0.7.0" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48f1a08ce0409a9e391b88d1930118eec48af12742fc538bcec55f775865776e" +checksum = "2079c122a62205b421f499da10f3ee0f7697f012f55b675e002483c73ea34700" dependencies = [ "unic-char-property", "unic-char-range", @@ -4372,9 +4459,9 @@ dependencies = [ [[package]] name = "unic-ucd-version" -version = "0.7.0" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf1f5e6c6c53c2d0ece4a5964bc55fcff8602153063cb4fab20958ff32998ff6" +checksum = "96bd2f2237fe450fcd0a1d2f5f4e91711124f7857ba2e964247776ebeeb7b0c4" dependencies = [ "unic-common", ] @@ -4427,17 +4514,6 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" -[[package]] -name = "url" -version = "1.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd4e7c0d531266369519a4aa4f399d748bd37043b00bde1e4ff1f60a120b355a" -dependencies = [ - "idna 0.1.5", - "matches", - "percent-encoding 1.0.1", -] - [[package]] name = "url" version = "2.5.2" @@ -4445,8 +4521,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "22784dbdf76fdde8af1aeda5622b546b422b6fc585325248a2bf9f5e41e94d6c" dependencies = [ "form_urlencoded", - "idna 0.5.0", - "percent-encoding 2.3.1", + "idna", + "percent-encoding", ] [[package]] @@ -4539,7 +4615,7 @@ dependencies = [ "mime", "mime_guess", "multer", - "percent-encoding 2.3.1", + "percent-encoding", "pin-project", "scoped-tls", "serde", diff --git a/Cargo.toml b/Cargo.toml index e5157d5c..013f3622 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,8 +2,8 @@ name = "crater" version = "0.1.0" edition = "2021" - build = "build.rs" +default-run = "crater" [profile.dev] opt-level = 0 @@ -24,7 +24,7 @@ http = "0.2" hyper = "0.14" lazy_static = "1.0" mime = "0.3.1" -minifier = { version = "0.2", features = ["html"] } +minifier = { version = "0.3", features = ["html"] } r2d2 = "0.8.2" r2d2_sqlite = "0.22.0" rusqlite = { version = "0.29.0", features = ["chrono", "functions", "bundled"] } @@ -38,7 +38,7 @@ serde_regex = "1.1.0" clap = { version = "4", features = ["derive"] } tar = "0.4.36" tempfile = "3.0.0" -tera = "0.11.7" +tera = "1.19.1" toml = "0.8.6" url = "2" walkdir = "2" @@ -60,7 +60,7 @@ tokio = "1.24" aws-sdk-s3 = "1.7" aws-config = { version = "1", features = ["behavior-version-latest"] } thiserror = "1.0.38" -nix = { version = "0.27.1", features = ["mman"] } +nix = { version = "0.27.1", features = ["mman", "resource"] } [dev-dependencies] assert_cmd = "2.0.4" diff --git a/src/assets.rs b/src/assets.rs index 0b7d727c..424e2215 100644 --- a/src/assets.rs +++ b/src/assets.rs @@ -162,6 +162,6 @@ pub fn render_template(name: &str, context: &C) -> Fallible = experiments + .into_iter() + .map(|record| record.into_experiment()) + .collect::>() + .unwrap(); + let ex = experiments.iter().find(|e| e.name == "pr-118920").unwrap(); + let rdb = crater::results::DatabaseDB::new(&db); + + log::info!("Getting crates..."); + + let crates = ex.get_crates(&db).unwrap(); + let writer = NullWriter; + + log::info!("Starting report generation..."); + log::info!( + "@ {:?}", + nix::sys::resource::getrusage(nix::sys::resource::UsageWho::RUSAGE_SELF) + .unwrap() + .max_rss() + ); + crater::report::gen(&rdb, ex, &crates, &writer, &config, false).unwrap(); + log::info!( + "@ {:?}", + nix::sys::resource::getrusage(nix::sys::resource::UsageWho::RUSAGE_SELF) + .unwrap() + .max_rss() + ); +} + +#[derive(Debug)] +struct NullWriter; + +impl ReportWriter for NullWriter { + fn write_bytes>( + &self, + _path: P, + _b: &[u8], + _mime: &Mime, + _encoding_type: EncodingType, + ) -> Fallible<()> { + // no-op + Ok(()) + } + fn write_string>(&self, _path: P, _s: Cow, _mime: &Mime) -> Fallible<()> { + // no-op + Ok(()) + } +} + +impl fmt::Display for NullWriter { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self) + } +} diff --git a/src/db/mod.rs b/src/db/mod.rs index d7c1ab6d..2a609a67 100644 --- a/src/db/mod.rs +++ b/src/db/mod.rs @@ -6,6 +6,7 @@ use r2d2::{CustomizeConnection, Pool}; use r2d2_sqlite::SqliteConnectionManager; use rusqlite::types::ToSql; use rusqlite::{Connection, Row, Transaction}; +use std::path::Path; use std::sync::Arc; use std::time::Instant; use tempfile::NamedTempFile; @@ -55,6 +56,11 @@ impl Database { Database::new(SqliteConnectionManager::file(path), None) } + pub fn open_at(path: &Path) -> Fallible { + std::fs::create_dir_all(&*WORK_DIR)?; + Database::new(SqliteConnectionManager::file(path), None) + } + #[cfg(test)] pub fn temp() -> Fallible { let tempfile = NamedTempFile::new()?; diff --git a/src/experiments.rs b/src/experiments.rs index 9e57e3e4..5374b989 100644 --- a/src/experiments.rs +++ b/src/experiments.rs @@ -659,7 +659,7 @@ impl Experiment { } } -struct ExperimentDBRecord { +pub struct ExperimentDBRecord { name: String, mode: String, cap_lints: String, @@ -680,7 +680,7 @@ struct ExperimentDBRecord { } impl ExperimentDBRecord { - fn from_row(row: &Row) -> rusqlite::Result { + pub fn from_row(row: &Row) -> rusqlite::Result { Ok(ExperimentDBRecord { name: row.get("name")?, mode: row.get("mode")?, @@ -702,7 +702,7 @@ impl ExperimentDBRecord { }) } - fn into_experiment(self) -> Fallible { + pub fn into_experiment(self) -> Fallible { Ok(Experiment { name: self.name, toolchains: [self.toolchain_start.parse()?, self.toolchain_end.parse()?], From 1416cf307e184fa413f93440dc913a575d9ae5f7 Mon Sep 17 00:00:00 2001 From: Mark Rousskov Date: Fri, 22 Dec 2023 11:11:34 -0500 Subject: [PATCH 3/3] Drop tera context sooner --- src/assets.rs | 5 +++-- src/report/html.rs | 9 +++++---- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/assets.rs b/src/assets.rs index 424e2215..f1ce6182 100644 --- a/src/assets.rs +++ b/src/assets.rs @@ -145,7 +145,7 @@ fn build_tera_cache() -> Fallible { } #[allow(unused_variables)] -pub fn render_template(name: &str, context: &C) -> Fallible { +pub fn render_template(name: &str, context: C) -> Fallible { // On debug builds the cache is rebuilt every time to pick up changed templates let tera_owned: Tera; let tera; @@ -161,7 +161,8 @@ pub fn render_template(name: &str, context: &C) -> Fallible( }; info!("generating {}", to); - let rendered = assets::render_template("report/results.html", &context) - .context("rendering template report/results.html")?; - let html = minifier::html::minify(&rendered); - dest.write_string(to, html.into(), &mime::TEXT_HTML)?; if output_templates { dest.write_string( @@ -269,6 +265,11 @@ fn write_report( )?; } + let rendered = assets::render_template("report/results.html", context) + .context("rendering template report/results.html")?; + let html = minifier::html::minify(&rendered); + dest.write_string(to, html.into(), &mime::TEXT_HTML)?; + Ok(()) }