diff --git a/Cargo.lock b/Cargo.lock index 65a00cd..84d7637 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -139,6 +139,7 @@ dependencies = [ "indoc", "itertools 0.12.0", "nonblock", + "num-format", "once_cell", "regex", "serde", diff --git a/crates/bencher/Cargo.toml b/crates/bencher/Cargo.toml index bdcdaf6..5fd5631 100644 --- a/crates/bencher/Cargo.toml +++ b/crates/bencher/Cargo.toml @@ -27,6 +27,7 @@ clap = { version = "4", features = ["derive"] } indexmap = "2" indoc = "2" itertools = "0.12" +num-format = "0.4" nonblock = "0.2" once_cell = "1" regex = "1" diff --git a/crates/bencher/src/html.rs b/crates/bencher/src/html.rs index e848741..e87802a 100644 --- a/crates/bencher/src/html.rs +++ b/crates/bencher/src/html.rs @@ -1,7 +1,9 @@ use indexmap::{IndexMap, IndexSet}; +use indoc::formatdoc; use itertools::Itertools; +use num_format::{Locale, ToFormattedString}; -use crate::set::TrackerCoreCountResults; +use crate::set::{LoadTestRunResults, TrackerCoreCountResults}; pub fn html_best_results(results: &[TrackerCoreCountResults]) -> String { let mut all_implementation_names = IndexSet::new(); @@ -45,7 +47,7 @@ pub fn html_best_results(results: &[TrackerCoreCountResults]) -> String { r#"{}"#, r.tracker_info, r.tracker_process_stats.avg_cpu_utilization, - r.average_responses, + r.average_responses.to_formatted_string(&Locale::en), ) } else { "-".to_string() @@ -59,6 +61,7 @@ pub fn html_best_results(results: &[TrackerCoreCountResults]) -> String { format!( " +

Best results

@@ -78,3 +81,129 @@ pub fn html_best_results(results: &[TrackerCoreCountResults]) -> String { data_rows.join("\n") ) } + +pub fn html_all_runs(all_results: &[TrackerCoreCountResults]) -> String { + let mut all_implementation_names = IndexSet::new(); + + for core_count_results in all_results { + all_implementation_names.extend( + core_count_results + .implementations + .iter() + .map(|r| r.name.clone()), + ); + } + + struct R { + core_count: usize, + avg_responses: Option, + tracker_keys: IndexMap, + tracker_vcpus: String, + load_test_keys: IndexMap, + load_test_vcpus: String, + } + + let mut output = String::new(); + + let mut results_by_implementation: IndexMap> = Default::default(); + + for implementation_name in all_implementation_names { + let results = results_by_implementation + .entry(implementation_name.clone()) + .or_default(); + + let mut tracker_key_names: IndexSet = Default::default(); + let mut load_test_key_names: IndexSet = Default::default(); + + for r in all_results { + for i in r + .implementations + .iter() + .filter(|i| i.name == implementation_name) + { + for c in i.configurations.iter() { + for l in c.load_tests.iter() { + match l { + LoadTestRunResults::Success(l) => { + tracker_key_names.extend(l.tracker_keys.keys().cloned()); + load_test_key_names.extend(l.load_test_keys.keys().cloned()); + + results.push(R { + core_count: r.core_count, + avg_responses: Some(l.average_responses), + tracker_keys: l.tracker_keys.clone(), + tracker_vcpus: l.tracker_vcpus.as_cpu_list(), + load_test_keys: l.load_test_keys.clone(), + load_test_vcpus: l.load_test_vcpus.as_cpu_list(), + }) + } + LoadTestRunResults::Failure(l) => { + tracker_key_names.extend(l.tracker_keys.keys().cloned()); + load_test_key_names.extend(l.load_test_keys.keys().cloned()); + + results.push(R { + core_count: r.core_count, + avg_responses: None, + tracker_keys: l.tracker_keys.clone(), + tracker_vcpus: l.tracker_vcpus.as_cpu_list(), + load_test_keys: l.load_test_keys.clone(), + load_test_vcpus: l.load_test_vcpus.as_cpu_list(), + }) + } + } + } + } + } + } + + output.push_str(&formatdoc! { + " +

Results for {}

+
+ + + + + {} + {} + + + + + + {} + +
CoresResponsesTracker vCPUsLoad test vCPUs
+ ", + implementation_name, + tracker_key_names.iter().map(|name| format!("Tracker {}", name)).join("\n"), + load_test_key_names.iter().map(|name| format!("Load test {}", name)).join("\n"), + results.into_iter().map(|r| { + formatdoc! { + " + + {} + {} + {} + {} + {} + {} + + ", + r.core_count, + r.avg_responses.map(|v| v.to_formatted_string(&Locale::en)).unwrap_or_else(|| "-".to_string()), + tracker_key_names.iter().map(|name| { + format!("{}", r.tracker_keys.get(name).cloned().unwrap_or_else(|| "-".to_string())) + }).join("\n"), + load_test_key_names.iter().map(|name| { + format!("{}", r.load_test_keys.get(name).cloned().unwrap_or_else(|| "-".to_string())) + }).join("\n"), + r.tracker_vcpus, + r.load_test_vcpus, + } + }).join("\n") + }); + } + + output +} diff --git a/crates/bencher/src/run.rs b/crates/bencher/src/run.rs index 09573e0..293eb8c 100644 --- a/crates/bencher/src/run.rs +++ b/crates/bencher/src/run.rs @@ -163,9 +163,8 @@ impl RunConfig { }; let avg_responses = { - static RE: Lazy = Lazy::new(|| { - Regex::new(r"Average responses per second: ([0-9]+\.?[0-9]+)").unwrap() - }); + static RE: Lazy = + Lazy::new(|| Regex::new(r"Average responses per second: ([0-9]+\.?)").unwrap()); let opt_avg_responses = RE .captures_iter(&load_test_stdout) @@ -175,7 +174,7 @@ impl RunConfig { avg_responses.to_string() }) - .and_then(|v| v.parse::().ok()); + .and_then(|v| v.parse::().ok()); if let Some(avg_responses) = opt_avg_responses { avg_responses @@ -199,7 +198,7 @@ impl RunConfig { pub struct RunSuccessResults { pub tracker_process_stats: ProcessStats, - pub avg_responses: f32, + pub avg_responses: u64, } #[derive(Debug)] diff --git a/crates/bencher/src/set.rs b/crates/bencher/src/set.rs index 837aae2..3aee777 100644 --- a/crates/bencher/src/set.rs +++ b/crates/bencher/src/set.rs @@ -1,10 +1,11 @@ use std::rc::Rc; use indexmap::IndexMap; +use num_format::{Locale, ToFormattedString}; use crate::{ common::{CpuDirection, CpuMode, TaskSetCpuList}, - html::html_best_results, + html::{html_all_runs, html_best_results}, run::{ProcessRunner, ProcessStats, RunConfig}, }; @@ -106,7 +107,8 @@ pub fn run_sets( }) .collect::>(); - html_best_results(&results); + println!("{}", html_all_runs(&results)); + println!("{}", html_best_results(&results)); } pub struct TrackerCoreCountResults { @@ -185,18 +187,21 @@ impl LoadTestRunResults { ); let load_test_runner = load_test_gen(workers); - // let load_test_keys = load_test_runner.keys(); + let load_test_keys = load_test_runner.keys(); let run_config = RunConfig { tracker_runner: tracker_process.clone(), tracker_vcpus: tracker_vcpus.clone(), load_test_runner, - load_test_vcpus, + load_test_vcpus: load_test_vcpus.clone(), }; match run_config.run(command) { Ok(r) => { - println!("- Average responses per second: {}", r.avg_responses); + println!( + "- Average responses per second: {}", + r.avg_responses.to_formatted_string(&Locale::en) + ); println!( "- Average tracker CPU utilization: {}%", r.tracker_process_stats.avg_cpu_utilization, @@ -208,17 +213,22 @@ impl LoadTestRunResults { LoadTestRunResults::Success(LoadTestRunResultsSuccess { average_responses: r.avg_responses, - // tracker_keys: tracker_process.keys(), + tracker_keys: tracker_process.keys(), tracker_info: tracker_process.info(), tracker_process_stats: r.tracker_process_stats, - // load_test_keys, + tracker_vcpus, + load_test_keys, + load_test_vcpus, }) } Err(results) => { println!("\nRun failed:\n{:#}\n", results); LoadTestRunResults::Failure(LoadTestRunResultsFailure { - // load_test_keys + tracker_keys: tracker_process.keys(), + tracker_vcpus, + load_test_keys, + load_test_vcpus, }) } } @@ -227,13 +237,18 @@ impl LoadTestRunResults { #[derive(Clone)] pub struct LoadTestRunResultsSuccess { - pub average_responses: f32, - // tracker_keys: IndexMap, + pub average_responses: u64, + pub tracker_keys: IndexMap, pub tracker_info: String, pub tracker_process_stats: ProcessStats, - // load_test_keys: IndexMap, + pub tracker_vcpus: TaskSetCpuList, + pub load_test_keys: IndexMap, + pub load_test_vcpus: TaskSetCpuList, } pub struct LoadTestRunResultsFailure { - // load_test_keys: IndexMap, + pub tracker_keys: IndexMap, + pub tracker_vcpus: TaskSetCpuList, + pub load_test_keys: IndexMap, + pub load_test_vcpus: TaskSetCpuList, }