From ad337d3ccb4ea01951f1b080537f79d4ba6e4c3c Mon Sep 17 00:00:00 2001 From: Serhiy Barhamon Date: Wed, 14 Dec 2022 20:52:35 +0100 Subject: [PATCH 1/6] #14385 WIP Added the `json` flag, copypasted ConsoleReporter to JsonReporter and shuffled code arround --- cli/args/flags.rs | 16 +++++ cli/tools/bench.rs | 175 ++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 188 insertions(+), 3 deletions(-) diff --git a/cli/args/flags.rs b/cli/args/flags.rs index 7c3eae7787d118..33787610c00da3 100644 --- a/cli/args/flags.rs +++ b/cli/args/flags.rs @@ -51,6 +51,7 @@ pub struct BenchFlags { pub ignore: Vec, pub include: Option>, pub filter: Option, + pub json: bool } #[derive(Clone, Debug, Eq, PartialEq, Deserialize, Serialize)] @@ -695,6 +696,12 @@ fn clap_root(version: &str) -> Command { fn bench_subcommand<'a>() -> Command<'a> { runtime_args(Command::new("bench"), true, false) .trailing_var_arg(true) + .arg( + Arg::new("json") + .long("json") + .help("Output benchmark result in JSON format") + .takes_value(false), + ) .arg( Arg::new("ignore") .long("ignore") @@ -729,6 +736,10 @@ and report results to standard output: deno bench src/fetch_bench.ts src/signal_bench.ts +Output benchmark result in JSON format: + + deno bench --json src/fetch_bench.ts src/signal_bench.ts + Directory arguments are expanded to all contained files matching the \ glob {*_,*.,}bench.{js,mjs,ts,mts,jsx,tsx}: @@ -2279,6 +2290,8 @@ fn bench_parse(flags: &mut Flags, matches: &clap::ArgMatches) { // interactive prompts, unless done by user code flags.no_prompt = true; + let json = matches.is_present("json"); + let ignore = match matches.values_of("ignore") { Some(f) => f.map(PathBuf::from).collect(), None => vec![], @@ -2314,6 +2327,7 @@ fn bench_parse(flags: &mut Flags, matches: &clap::ArgMatches) { include, ignore, filter, + json }); } @@ -6335,6 +6349,7 @@ mod tests { let r = flags_from_vec(svec![ "deno", "bench", + "--json", "--unstable", "--filter", "- foo", @@ -6354,6 +6369,7 @@ mod tests { filter: Some("- foo".to_string()), include: Some(svec!["dir1/", "dir2/"]), ignore: vec![], + json: true }), unstable: true, type_check_mode: TypeCheckMode::Local, diff --git a/cli/tools/bench.rs b/cli/tools/bench.rs index 2bebe124b1196d..c4634e84697b21 100644 --- a/cli/tools/bench.rs +++ b/cli/tools/bench.rs @@ -42,6 +42,7 @@ use tokio::sync::mpsc::UnboundedSender; #[derive(Debug, Clone, Deserialize)] struct BenchSpecifierOptions { filter: Option, + json: bool, } #[derive(Debug, Clone, Eq, PartialEq, Deserialize)] @@ -110,7 +111,13 @@ impl BenchReport { } } -fn create_reporter(show_output: bool) -> Box { +fn create_reporter( + show_output: bool, + json: bool, +) -> Box { + if json { + return Box::new(JsonReporter::new(show_output)); + } Box::new(ConsoleReporter::new(show_output)) } @@ -124,6 +131,163 @@ pub trait BenchReporter { fn report_result(&mut self, desc: &BenchDescription, result: &BenchResult); } +struct JsonReporter { + name: String, + show_output: bool, + has_ungrouped: bool, + group: Option, + baseline: bool, + group_measurements: Vec<(BenchDescription, BenchStats)>, + options: Option, +} + +impl JsonReporter { + fn new(show_output: bool) -> Self { + Self { + show_output, + group: None, + options: None, + baseline: false, + name: String::new(), + has_ungrouped: false, + group_measurements: Vec::new(), + } + } +} + +impl BenchReporter for JsonReporter { + #[cold] + fn report_plan(&mut self, plan: &BenchPlan) { + self.report_group_summary(); + self.group = None; + self.baseline = false; + self.name = String::new(); + self.group_measurements.clear(); + self.options = Some(mitata::reporter::Options::new( + &plan.names.iter().map(|x| x.as_str()).collect::>(), + )); + + let options = self.options.as_mut().unwrap(); + + options.percentiles = true; + } + fn report_register(&mut self, desc: &BenchDescription) { + todo!() + } + + fn report_wait(&mut self, desc: &BenchDescription) { + self.name = desc.name.clone(); + + match &desc.group { + None => { + self.has_ungrouped = true; + } + + Some(group) => { + if self.group.is_none() + && self.has_ungrouped + && self.group_measurements.is_empty() + { + println!(); + } + + if self.group.is_none() || group != self.group.as_ref().unwrap() { + self.report_group_summary(); + } + + if (self.group.is_none() && self.has_ungrouped) + || (self.group.is_some() && self.group_measurements.is_empty()) + { + println!(); + } + + self.group = Some(group.clone()); + } + } + } + + fn report_result(&mut self, desc: &BenchDescription, result: &BenchResult) { + let options = self.options.as_ref().unwrap(); + + match result { + BenchResult::Ok(stats) => { + let mut desc = desc.clone(); + + if desc.baseline && !self.baseline { + self.baseline = true; + } else { + desc.baseline = false; + } + + self.group_measurements.push((desc, stats.clone())); + } + + BenchResult::Failed(js_error) => { + println!( + "{}", + mitata::reporter::benchmark_error( + &desc.name, + &mitata::reporter::Error { + stack: None, + message: format_test_error(js_error), + }, + options + ) + ) + } + }; + } + + fn report_output(&mut self, output: &str) { + if self.show_output { + println!("{}", output); + } + } + + fn report_group_summary(&mut self) { + let options = match self.options.as_ref() { + None => return, + Some(options) => options, + }; + + if 2 <= self.group_measurements.len() + && (self.group.is_some() || (self.group.is_none() && self.baseline)) + { + println!( + "\n{}", + mitata::reporter::summary( + &self + .group_measurements + .iter() + .map(|(d, s)| mitata::reporter::GroupBenchmark { + name: d.name.clone(), + baseline: d.baseline, + group: d.group.as_deref().unwrap_or("").to_owned(), + + stats: mitata::reporter::BenchmarkStats { + avg: s.avg, + min: s.min, + max: s.max, + p75: s.p75, + p99: s.p99, + p995: s.p995, + }, + }) + .collect::>(), + options + ) + ); + } + + self.baseline = false; + self.group_measurements.clear(); + } + + fn report_end(&mut self, report: &BenchReport) { + self.report_group_summary(); + } +} + struct ConsoleReporter { name: String, show_output: bool, @@ -378,12 +542,14 @@ async fn bench_specifiers( let (sender, mut receiver) = unbounded_channel::(); + let option_for_handles = options.clone(); + let join_handles = specifiers.iter().map(move |specifier| { let ps = ps.clone(); let permissions = permissions.clone(); let specifier = specifier.clone(); let sender = sender.clone(); - let options = options.clone(); + let options = option_for_handles.clone(); tokio::task::spawn_blocking(move || { let future = bench_specifier(ps, permissions, specifier, sender, options); @@ -400,7 +566,8 @@ async fn bench_specifiers( tokio::task::spawn(async move { let mut used_only = false; let mut report = BenchReport::new(); - let mut reporter = create_reporter(log_level != Some(Level::Error)); + let mut reporter = + create_reporter(log_level != Some(Level::Error), options.json); let mut benches = IndexMap::new(); while let Some(event) = receiver.recv().await { @@ -514,6 +681,7 @@ pub async fn run_benchmarks( specifiers, BenchSpecifierOptions { filter: bench_flags.filter, + json: bench_flags.json, }, ) .await?; @@ -675,6 +843,7 @@ pub async fn run_benchmarks_with_watch( let specifier_options = BenchSpecifierOptions { filter: filter.clone(), + json: bench_flags.json, }; bench_specifiers(ps, permissions.clone(), specifiers, specifier_options) .await?; From cc3a05b1ca2500aeb7413bff35208456b790d533 Mon Sep 17 00:00:00 2001 From: Serhiy Barhamon Date: Mon, 30 Jan 2023 16:36:30 +0100 Subject: [PATCH 2/6] #14385 benchmark json output output format: ``` [ { "runtime": "Deno/1.28.3 x86_64-apple-darwin", "cpu": "Intel(R) Core(TM) i7-9750H CPU @ 2.60GHz", "origin": "file:///Users/serhiy.barhamon/tmp/deno/test_ffi/tests/bench.js", "group": null, "name": "Deno.UnsafePointerView#getUint32", "baseline": false, "result": { "ok": { "n": 49, "min": 1251.9348, "max": 1441.2696, "avg": 1308.7523755102038, "p75": 1324.1055, "p99": 1441.2696, "p995": 1441.2696, "p999": 1441.2696 } } } ] ``` time unit: nanoseconds --- cli/tools/bench.rs | 189 ++++++++++++--------------------------------- 1 file changed, 51 insertions(+), 138 deletions(-) diff --git a/cli/tools/bench.rs b/cli/tools/bench.rs index c4634e84697b21..cd52b0a736ad2e 100644 --- a/cli/tools/bench.rs +++ b/cli/tools/bench.rs @@ -11,11 +11,13 @@ use crate::ops; use crate::proc_state::ProcState; use crate::tools::test::format_test_error; use crate::tools::test::TestFilter; +use crate::util::display::write_json_to_stdout; use crate::util::file_watcher; use crate::util::file_watcher::ResolutionResult; use crate::util::fs::collect_specifiers; use crate::util::path::is_supported_ext; use crate::util::path::specifier_to_file_path; +use crate::version::get_user_agent; use crate::worker::create_main_worker_for_test_or_bench; use deno_core::error::generic_error; @@ -64,7 +66,7 @@ pub enum BenchEvent { Result(usize, BenchResult), } -#[derive(Debug, Clone, Deserialize)] +#[derive(Debug, Clone, Deserialize, Serialize)] #[serde(rename_all = "camelCase")] pub enum BenchResult { Ok(BenchStats), @@ -116,7 +118,7 @@ fn create_reporter( json: bool, ) -> Box { if json { - return Box::new(JsonReporter::new(show_output)); + return Box::new(JsonReporter::new()); } Box::new(ConsoleReporter::new(show_output)) } @@ -131,160 +133,71 @@ pub trait BenchReporter { fn report_result(&mut self, desc: &BenchDescription, result: &BenchResult); } -struct JsonReporter { - name: String, - show_output: bool, - has_ungrouped: bool, +#[derive(Debug, Serialize)] +struct JsonReporterResult { + runtime: String, + cpu: String, + origin: String, group: Option, + name: String, baseline: bool, - group_measurements: Vec<(BenchDescription, BenchStats)>, - options: Option, + result: BenchResult, } -impl JsonReporter { - fn new(show_output: bool) -> Self { +impl JsonReporterResult { + fn new( + origin: String, + group: Option, + name: String, + baseline: bool, + result: BenchResult, + ) -> Self { Self { - show_output, - group: None, - options: None, - baseline: false, - name: String::new(), - has_ungrouped: false, - group_measurements: Vec::new(), + runtime: format!("{} {}", get_user_agent(), env!("TARGET")), + cpu: mitata::cpu::name(), + origin, + group, + name, + baseline, + result, } } } -impl BenchReporter for JsonReporter { - #[cold] - fn report_plan(&mut self, plan: &BenchPlan) { - self.report_group_summary(); - self.group = None; - self.baseline = false; - self.name = String::new(); - self.group_measurements.clear(); - self.options = Some(mitata::reporter::Options::new( - &plan.names.iter().map(|x| x.as_str()).collect::>(), - )); - - let options = self.options.as_mut().unwrap(); - - options.percentiles = true; - } - fn report_register(&mut self, desc: &BenchDescription) { - todo!() - } - - fn report_wait(&mut self, desc: &BenchDescription) { - self.name = desc.name.clone(); - - match &desc.group { - None => { - self.has_ungrouped = true; - } - - Some(group) => { - if self.group.is_none() - && self.has_ungrouped - && self.group_measurements.is_empty() - { - println!(); - } - - if self.group.is_none() || group != self.group.as_ref().unwrap() { - self.report_group_summary(); - } - - if (self.group.is_none() && self.has_ungrouped) - || (self.group.is_some() && self.group_measurements.is_empty()) - { - println!(); - } - - self.group = Some(group.clone()); - } - } +#[derive(Debug, Serialize)] +struct JsonReporter(Vec); +impl JsonReporter { + fn new() -> Self { + Self(vec![]) } +} - fn report_result(&mut self, desc: &BenchDescription, result: &BenchResult) { - let options = self.options.as_ref().unwrap(); - - match result { - BenchResult::Ok(stats) => { - let mut desc = desc.clone(); - - if desc.baseline && !self.baseline { - self.baseline = true; - } else { - desc.baseline = false; - } - - self.group_measurements.push((desc, stats.clone())); - } - - BenchResult::Failed(js_error) => { - println!( - "{}", - mitata::reporter::benchmark_error( - &desc.name, - &mitata::reporter::Error { - stack: None, - message: format_test_error(js_error), - }, - options - ) - ) - } - }; - } +impl BenchReporter for JsonReporter { + fn report_group_summary(&mut self) {} + #[cold] + fn report_plan(&mut self, _plan: &BenchPlan) {} - fn report_output(&mut self, output: &str) { - if self.show_output { - println!("{}", output); + fn report_end(&mut self, _report: &BenchReport) { + match write_json_to_stdout(self) { + Ok(_) => (), + Err(e) => println!("{}", e), } } - fn report_group_summary(&mut self) { - let options = match self.options.as_ref() { - None => return, - Some(options) => options, - }; - - if 2 <= self.group_measurements.len() - && (self.group.is_some() || (self.group.is_none() && self.baseline)) - { - println!( - "\n{}", - mitata::reporter::summary( - &self - .group_measurements - .iter() - .map(|(d, s)| mitata::reporter::GroupBenchmark { - name: d.name.clone(), - baseline: d.baseline, - group: d.group.as_deref().unwrap_or("").to_owned(), + fn report_register(&mut self, _desc: &BenchDescription) {} - stats: mitata::reporter::BenchmarkStats { - avg: s.avg, - min: s.min, - max: s.max, - p75: s.p75, - p99: s.p99, - p995: s.p995, - }, - }) - .collect::>(), - options - ) - ); - } + fn report_wait(&mut self, _desc: &BenchDescription) {} - self.baseline = false; - self.group_measurements.clear(); - } + fn report_output(&mut self, _output: &str) {} - fn report_end(&mut self, report: &BenchReport) { - self.report_group_summary(); + fn report_result(&mut self, desc: &BenchDescription, result: &BenchResult) { + self.0.push(JsonReporterResult::new( + desc.origin.clone(), + desc.group.clone(), + desc.name.clone(), + desc.baseline, + result.clone(), + )); } } From 1d7be61534730e34a1aabf469a03c89ac5f2a7cf Mon Sep 17 00:00:00 2001 From: Serhiy Barhamon Date: Mon, 30 Jan 2023 18:20:59 +0100 Subject: [PATCH 3/6] #14385 fix merge conflict mess --- cli/args/mod.rs | 2 ++ cli/tools/bench.rs | 11 ++++------- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/cli/args/mod.rs b/cli/args/mod.rs index d75f25d525831a..f49852a6f31409 100644 --- a/cli/args/mod.rs +++ b/cli/args/mod.rs @@ -105,6 +105,7 @@ impl CacheSetting { pub struct BenchOptions { pub files: FilesConfig, pub filter: Option, + pub json: bool, } impl BenchOptions { @@ -119,6 +120,7 @@ impl BenchOptions { Some(bench_flags.files), ), filter: bench_flags.filter, + json: bench_flags.json, }) } } diff --git a/cli/tools/bench.rs b/cli/tools/bench.rs index d49d401c5afc29..b27e08f88c9a06 100644 --- a/cli/tools/bench.rs +++ b/cli/tools/bench.rs @@ -4,17 +4,16 @@ use crate::args::BenchOptions; use crate::args::CliOptions; use crate::args::TypeCheckMode; use crate::colors; +use crate::display::write_json_to_stdout; use crate::graph_util::graph_valid; use crate::ops; use crate::proc_state::ProcState; use crate::tools::test::format_test_error; use crate::tools::test::TestFilter; -use crate::util::display::write_json_to_stdout; use crate::util::file_watcher; use crate::util::file_watcher::ResolutionResult; use crate::util::fs::collect_specifiers; use crate::util::path::is_supported_ext; -use crate::util::path::specifier_to_file_path; use crate::version::get_user_agent; use crate::worker::create_main_worker_for_test_or_bench; @@ -42,9 +41,8 @@ use tokio::sync::mpsc::UnboundedSender; #[derive(Debug, Clone)] struct BenchSpecifierOptions { - filter: Option, - json: bool, filter: TestFilter, + json: bool, } #[derive(Debug, Clone, Eq, PartialEq, Deserialize)] @@ -590,9 +588,8 @@ pub async fn run_benchmarks( &permissions, specifiers, BenchSpecifierOptions { - filter: bench_flags.filter, - json: bench_flags.json, filter: TestFilter::from_flag(&bench_options.filter), + json: bench_options.json, }, ) .await?; @@ -739,7 +736,7 @@ pub async fn run_benchmarks_with_watch( specifiers, BenchSpecifierOptions { filter: TestFilter::from_flag(&bench_options.filter), - json: bench_flags.json, + json: bench_options.json, }, ) .await?; From a06b5e1d7ca77135afcc849286809c864df8cecc Mon Sep 17 00:00:00 2001 From: Serhiy Barhamon Date: Fri, 3 Feb 2023 11:37:47 +0100 Subject: [PATCH 4/6] Fix tests --- cli/args/flags.rs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/cli/args/flags.rs b/cli/args/flags.rs index ac140161212ac6..57a4318fb82836 100644 --- a/cli/args/flags.rs +++ b/cli/args/flags.rs @@ -54,7 +54,7 @@ pub struct FileFlags { pub struct BenchFlags { pub files: FileFlags, pub filter: Option, - pub json: bool + pub json: bool, } #[derive(Clone, Debug, Eq, PartialEq)] @@ -2368,7 +2368,7 @@ fn bench_parse(flags: &mut Flags, matches: &clap::ArgMatches) { flags.subcommand = DenoSubcommand::Bench(BenchFlags { files: FileFlags { include, ignore }, filter, - json + json, }); } @@ -6571,8 +6571,6 @@ mod tests { Flags { subcommand: DenoSubcommand::Bench(BenchFlags { filter: Some("- foo".to_string()), - include: Some(svec!["dir1/", "dir2/"]), - ignore: vec![], json: true, files: FileFlags { include: vec![PathBuf::from("dir1/"), PathBuf::from("dir2/")], @@ -6598,6 +6596,7 @@ mod tests { Flags { subcommand: DenoSubcommand::Bench(BenchFlags { filter: None, + json: false, files: FileFlags { include: vec![], ignore: vec![], From e8499b6d933ebd0cef8cd4eeb8714dc1d41c6503 Mon Sep 17 00:00:00 2001 From: Serhiy Barhamon Date: Fri, 3 Feb 2023 14:26:45 +0100 Subject: [PATCH 5/6] make clippy happy --- cli/tools/bench.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/tools/bench.rs b/cli/tools/bench.rs index b27e08f88c9a06..7c94d7b9214ca3 100644 --- a/cli/tools/bench.rs +++ b/cli/tools/bench.rs @@ -178,7 +178,7 @@ impl BenchReporter for JsonReporter { fn report_end(&mut self, _report: &BenchReport) { match write_json_to_stdout(self) { Ok(_) => (), - Err(e) => println!("{}", e), + Err(e) => println!("{e}"), } } From c7e9729c0e447fbfc9c702df7fd02396bbd1983d Mon Sep 17 00:00:00 2001 From: Serhiy Barhamon Date: Sat, 11 Feb 2023 15:42:18 +0100 Subject: [PATCH 6/6] fix doc args --- cli/args/flags.rs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/cli/args/flags.rs b/cli/args/flags.rs index 57a4318fb82836..255bc447904009 100644 --- a/cli/args/flags.rs +++ b/cli/args/flags.rs @@ -721,7 +721,7 @@ fn bench_subcommand<'a>() -> Command<'a> { .arg( Arg::new("json") .long("json") - .help("Output benchmark result in JSON format") + .help("UNSTABLE: Output benchmark result in JSON format") .takes_value(false), ) .arg( @@ -758,10 +758,6 @@ and report results to standard output: deno bench src/fetch_bench.ts src/signal_bench.ts -Output benchmark result in JSON format: - - deno bench --json src/fetch_bench.ts src/signal_bench.ts - Directory arguments are expanded to all contained files matching the \ glob {*_,*.,}bench.{js,mjs,ts,mts,jsx,tsx}: