bencher: print html table with full results too, use num_format

Cargo.lock

@@ -139,6 +139,7 @@ dependencies = [
  "indoc",
  "itertools 0.12.0",
  "nonblock",
+ "num-format",
  "once_cell",
  "regex",
  "serde",

crates/bencher/Cargo.toml

@@ -27,6 +27,7 @@ clap = { version = "4", features = ["derive"] }
 indexmap = "2"
 indoc = "2"
 itertools = "0.12"
+num-format = "0.4"
 nonblock = "0.2"
 once_cell = "1"
 regex = "1"

crates/bencher/src/html.rs

@@ -1,7 +1,9 @@
 use indexmap::{IndexMap, IndexSet};
+use indoc::formatdoc;
 use itertools::Itertools;
+use num_format::{Locale, ToFormattedString};
 
-use crate::set::TrackerCoreCountResults;
+use crate::set::{LoadTestRunResults, TrackerCoreCountResults};
 
 pub fn html_best_results(results: &[TrackerCoreCountResults]) -> String {
     let mut all_implementation_names = IndexSet::new();
@@ -45,7 +47,7 @@ pub fn html_best_results(results: &[TrackerCoreCountResults]) -> String {
                             r#"<td><span title="{}, avg cpu utilization: {}%">{}</span></td>"#,
                             r.tracker_info,
                             r.tracker_process_stats.avg_cpu_utilization,
-                            r.average_responses,
+                            r.average_responses.to_formatted_string(&Locale::en),
                         )
                     } else {
                         "<td>-</td>".to_string()
@@ -59,6 +61,7 @@ pub fn html_best_results(results: &[TrackerCoreCountResults]) -> String {
 
     format!(
         "
+        <h2>Best results</h2>
         <table>
             <thead>
                 <tr>
@@ -78,3 +81,129 @@ pub fn html_best_results(results: &[TrackerCoreCountResults]) -> String {
         data_rows.join("\n")
     )
 }
+
+pub fn html_all_runs(all_results: &[TrackerCoreCountResults]) -> String {
+    let mut all_implementation_names = IndexSet::new();
+
+    for core_count_results in all_results {
+        all_implementation_names.extend(
+            core_count_results
+                .implementations
+                .iter()
+                .map(|r| r.name.clone()),
+        );
+    }
+
+    struct R {
+        core_count: usize,
+        avg_responses: Option<u64>,
+        tracker_keys: IndexMap<String, String>,
+        tracker_vcpus: String,
+        load_test_keys: IndexMap<String, String>,
+        load_test_vcpus: String,
+    }
+
+    let mut output = String::new();
+
+    let mut results_by_implementation: IndexMap<String, Vec<R>> = Default::default();
+
+    for implementation_name in all_implementation_names {
+        let results = results_by_implementation
+            .entry(implementation_name.clone())
+            .or_default();
+
+        let mut tracker_key_names: IndexSet<String> = Default::default();
+        let mut load_test_key_names: IndexSet<String> = Default::default();
+
+        for r in all_results {
+            for i in r
+                .implementations
+                .iter()
+                .filter(|i| i.name == implementation_name)
+            {
+                for c in i.configurations.iter() {
+                    for l in c.load_tests.iter() {
+                        match l {
+                            LoadTestRunResults::Success(l) => {
+                                tracker_key_names.extend(l.tracker_keys.keys().cloned());
+                                load_test_key_names.extend(l.load_test_keys.keys().cloned());
+
+                                results.push(R {
+                                    core_count: r.core_count,
+                                    avg_responses: Some(l.average_responses),
+                                    tracker_keys: l.tracker_keys.clone(),
+                                    tracker_vcpus: l.tracker_vcpus.as_cpu_list(),
+                                    load_test_keys: l.load_test_keys.clone(),
+                                    load_test_vcpus: l.load_test_vcpus.as_cpu_list(),
+                                })
+                            }
+                            LoadTestRunResults::Failure(l) => {
+                                tracker_key_names.extend(l.tracker_keys.keys().cloned());
+                                load_test_key_names.extend(l.load_test_keys.keys().cloned());
+
+                                results.push(R {
+                                    core_count: r.core_count,
+                                    avg_responses: None,
+                                    tracker_keys: l.tracker_keys.clone(),
+                                    tracker_vcpus: l.tracker_vcpus.as_cpu_list(),
+                                    load_test_keys: l.load_test_keys.clone(),
+                                    load_test_vcpus: l.load_test_vcpus.as_cpu_list(),
+                                })
+                            }
+                        }
+                    }
+                }
+            }
+        }
+
+        output.push_str(&formatdoc! {
+            "
+            <h2>Results for {}</h2>
+            <table>
+                <thead>
+                    <tr>
+                        <th>Cores</th>
+                        <th>Responses</th>
+                        {}
+                        {}
+                        <th>Tracker vCPUs</th>
+                        <th>Load test vCPUs</th>
+                    </tr>
+                </thead>
+                <tbody>
+                {}
+                </tbody>
+            </table>
+            ",
+            implementation_name,
+            tracker_key_names.iter().map(|name| format!("<th>Tracker {}</th>", name)).join("\n"),
+            load_test_key_names.iter().map(|name| format!("<th>Load test {}</th>", name)).join("\n"),
+            results.into_iter().map(|r| {
+                formatdoc! {
+                    "
+                    <tr>
+                        <td>{}</td>
+                        <td>{}</td>
+                        {}
+                        {}
+                        <td>{}</td>
+                        <td>{}</td>
+                    </tr>
+                    ",
+                    r.core_count,
+                    r.avg_responses.map(|v| v.to_formatted_string(&Locale::en)).unwrap_or_else(|| "-".to_string()),
+                    tracker_key_names.iter().map(|name| {
+                        format!("<th>{}</th>", r.tracker_keys.get(name).cloned().unwrap_or_else(|| "-".to_string()))
+                    }).join("\n"),
+                    load_test_key_names.iter().map(|name| {
+                        format!("<th>{}</th>", r.load_test_keys.get(name).cloned().unwrap_or_else(|| "-".to_string()))
+                    }).join("\n"),
+                    r.tracker_vcpus,
+                    r.load_test_vcpus,
+                }
+            }).join("\n")
+        });
+    }
+
+    output
+}

crates/bencher/src/run.rs

@@ -163,9 +163,8 @@ impl<C> RunConfig<C> {
         };
 
         let avg_responses = {
-            static RE: Lazy<Regex> = Lazy::new(|| {
+            static RE: Lazy<Regex> =
-                Regex::new(r"Average responses per second: ([0-9]+\.?[0-9]+)").unwrap()
+                Lazy::new(|| Regex::new(r"Average responses per second: ([0-9]+\.?)").unwrap());
-            });
 
             let opt_avg_responses = RE
                 .captures_iter(&load_test_stdout)
@@ -175,7 +174,7 @@ impl<C> RunConfig<C> {
 
                     avg_responses.to_string()
                 })
-                .and_then(|v| v.parse::<f32>().ok());
+                .and_then(|v| v.parse::<u64>().ok());
 
             if let Some(avg_responses) = opt_avg_responses {
                 avg_responses
@@ -199,7 +198,7 @@ impl<C> RunConfig<C> {
 
 pub struct RunSuccessResults {
     pub tracker_process_stats: ProcessStats,
-    pub avg_responses: f32,
+    pub avg_responses: u64,
 }
 
 #[derive(Debug)]

crates/bencher/src/set.rs

@@ -1,10 +1,11 @@
 use std::rc::Rc;
 
 use indexmap::IndexMap;
+use num_format::{Locale, ToFormattedString};
 
 use crate::{
     common::{CpuDirection, CpuMode, TaskSetCpuList},
-    html::html_best_results,
+    html::{html_all_runs, html_best_results},
     run::{ProcessRunner, ProcessStats, RunConfig},
 };
 
@@ -106,7 +107,8 @@ pub fn run_sets<C, F, I>(
         })
         .collect::<Vec<_>>();
 
-    html_best_results(&results);
+    println!("{}", html_all_runs(&results));
+    println!("{}", html_best_results(&results));
 }
 
 pub struct TrackerCoreCountResults {
@@ -185,18 +187,21 @@ impl LoadTestRunResults {
         );
 
         let load_test_runner = load_test_gen(workers);
-        // let load_test_keys = load_test_runner.keys();
+        let load_test_keys = load_test_runner.keys();
 
         let run_config = RunConfig {
             tracker_runner: tracker_process.clone(),
             tracker_vcpus: tracker_vcpus.clone(),
             load_test_runner,
-            load_test_vcpus,
+            load_test_vcpus: load_test_vcpus.clone(),
         };
 
         match run_config.run(command) {
             Ok(r) => {
-                println!("- Average responses per second: {}", r.avg_responses);
+                println!(
+                    "- Average responses per second: {}",
+                    r.avg_responses.to_formatted_string(&Locale::en)
+                );
                 println!(
                     "- Average tracker CPU utilization: {}%",
                     r.tracker_process_stats.avg_cpu_utilization,
@@ -208,17 +213,22 @@ impl LoadTestRunResults {
 
                 LoadTestRunResults::Success(LoadTestRunResultsSuccess {
                     average_responses: r.avg_responses,
-                    // tracker_keys: tracker_process.keys(),
+                    tracker_keys: tracker_process.keys(),
                     tracker_info: tracker_process.info(),
                     tracker_process_stats: r.tracker_process_stats,
-                    // load_test_keys,
+                    tracker_vcpus,
+                    load_test_keys,
+                    load_test_vcpus,
                 })
             }
             Err(results) => {
                 println!("\nRun failed:\n{:#}\n", results);
 
                 LoadTestRunResults::Failure(LoadTestRunResultsFailure {
-                    // load_test_keys
+                    tracker_keys: tracker_process.keys(),
+                    tracker_vcpus,
+                    load_test_keys,
+                    load_test_vcpus,
                 })
             }
         }
@@ -227,13 +237,18 @@ impl LoadTestRunResults {
 
 #[derive(Clone)]
 pub struct LoadTestRunResultsSuccess {
-    pub average_responses: f32,
+    pub average_responses: u64,
-    // tracker_keys: IndexMap<String, String>,
+    pub tracker_keys: IndexMap<String, String>,
     pub tracker_info: String,
     pub tracker_process_stats: ProcessStats,
-    // load_test_keys: IndexMap<String, String>,
+    pub tracker_vcpus: TaskSetCpuList,
+    pub load_test_keys: IndexMap<String, String>,
+    pub load_test_vcpus: TaskSetCpuList,
 }
 
 pub struct LoadTestRunResultsFailure {
-    // load_test_keys: IndexMap<String, String>,
+    pub tracker_keys: IndexMap<String, String>,
+    pub tracker_vcpus: TaskSetCpuList,
+    pub load_test_keys: IndexMap<String, String>,
+    pub load_test_vcpus: TaskSetCpuList,
 }