bencher: add args duration and summarize_last (seconds)

crates/bencher/src/main.rs

@@ -6,6 +6,7 @@ pub mod set;
 
 use clap::{Parser, Subcommand};
 use common::{CpuMode, Priority};
+use set::run_sets;
 
 #[derive(Parser)]
 #[command(author, version, about)]
@@ -23,6 +24,17 @@ struct Args {
     /// Minimum benchmark priority
     #[arg(long, default_value_t = Priority::Medium)]
     min_priority: Priority,
+    /// How long to run each load test for
+    #[arg(long, default_value_t = 60)]
+    duration: usize,
+    /// Only include data for last N seconds of load test runs.
+    ///
+    /// Useful if the tracker/load tester combination is slow at reaching
+    /// maximum throughput
+    ///
+    /// 0 = use data for whole run
+    #[arg(long, default_value_t = 0)]
+    summarize_last: usize,
     #[command(subcommand)]
     command: Command,
 }
@@ -39,13 +51,21 @@ fn main() {
 
     match args.command {
         #[cfg(feature = "udp")]
-        Command::Udp(command) => command
+        Command::Udp(command) => {
-            .run(
+            let sets = command.sets(args.cpu_mode);
+            let load_test_gen = protocols::udp::UdpCommand::load_test_gen;
+
+            run_sets(
+                &command,
                 args.cpu_mode,
                 args.min_cores,
                 args.max_cores,
                 args.min_priority,
-            )
+                args.duration,
-            .unwrap(),
+                args.summarize_last,
+                sets,
+                load_test_gen,
+            );
+        }
     }
 }

crates/bencher/src/protocols/udp.rs

@@ -13,7 +13,7 @@ use tempfile::NamedTempFile;
 use crate::{
     common::{simple_load_test_runs, CpuMode, Priority, TaskSetCpuList},
     run::ProcessRunner,
-    set::{run_sets, SetConfig, Tracker},
+    set::{LoadTestRunnerParameters, SetConfig, Tracker},
 };
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
@@ -50,29 +50,7 @@ pub struct UdpCommand {
 }
 
 impl UdpCommand {
-    pub fn run(
+    pub fn sets(&self, cpu_mode: CpuMode) -> IndexMap<usize, SetConfig<UdpCommand, UdpTracker>> {
-        &self,
-        cpu_mode: CpuMode,
-        min_cores: Option<usize>,
-        max_cores: Option<usize>,
-        min_priority: Priority,
-    ) -> anyhow::Result<()> {
-        let sets = self.sets(cpu_mode);
-
-        run_sets(
-            self,
-            cpu_mode,
-            min_cores,
-            max_cores,
-            min_priority,
-            sets,
-            |workers| Box::new(AquaticUdpLoadTestRunner { workers }),
-        );
-
-        Ok(())
-    }
-
-    fn sets(&self, cpu_mode: CpuMode) -> IndexMap<usize, SetConfig<UdpCommand, UdpTracker>> {
         // Priorities are based on what has previously produced the best results
         indexmap::indexmap! {
             1 => SetConfig {
@@ -264,6 +242,12 @@ impl UdpCommand {
             },
         }
     }
+
+    pub fn load_test_gen(
+        parameters: LoadTestRunnerParameters,
+    ) -> Box<dyn ProcessRunner<Command = UdpCommand>> {
+        Box::new(AquaticUdpLoadTestRunner { parameters })
+    }
 }
 
 #[derive(Debug, Clone)]
@@ -434,7 +418,7 @@ impl ProcessRunner for ChihayaUdpRunner {
 
 #[derive(Debug, Clone)]
 struct AquaticUdpLoadTestRunner {
-    workers: usize,
+    parameters: LoadTestRunnerParameters,
 }
 
 impl ProcessRunner for AquaticUdpLoadTestRunner {
@@ -448,8 +432,9 @@ impl ProcessRunner for AquaticUdpLoadTestRunner {
     ) -> anyhow::Result<Child> {
         let mut c = aquatic_udp_load_test::config::Config::default();
 
-        c.workers = self.workers as u8;
+        c.workers = self.parameters.workers as u8;
-        c.duration = 60;
+        c.duration = self.parameters.duration;
+        c.summarize_last = self.parameters.summarize_last;
 
         c.requests.weight_connect = 0;
         c.requests.weight_announce = 100;
@@ -478,7 +463,7 @@ impl ProcessRunner for AquaticUdpLoadTestRunner {
 
     fn keys(&self) -> IndexMap<String, String> {
         indexmap! {
-            "workers".to_string() => self.workers.to_string(),
+            "workers".to_string() => self.parameters.workers.to_string(),
         }
     }
 }

crates/bencher/src/run.rs

@@ -45,7 +45,11 @@ pub struct RunConfig<C> {
 }
 
 impl<C> RunConfig<C> {
-    pub fn run(self, command: &C) -> Result<RunSuccessResults, RunErrorResults<C>> {
+    pub fn run(
+        self,
+        command: &C,
+        duration: usize,
+    ) -> Result<RunSuccessResults, RunErrorResults<C>> {
         let mut tracker_config_file = NamedTempFile::new().unwrap();
         let mut load_test_config_file = NamedTempFile::new().unwrap();
 
@@ -75,7 +79,7 @@ impl<C> RunConfig<C> {
             }
         };
 
-        for _ in 0..59 {
+        for _ in 0..(duration - 1) {
             if let Ok(Some(status)) = tracker.0.try_wait() {
                 return Err(RunErrorResults::new(self)
                     .set_tracker_outputs(tracker)

crates/bencher/src/set.rs

@@ -10,6 +10,13 @@ use crate::{
     run::{ProcessRunner, ProcessStats, RunConfig},
 };
 
+#[derive(Debug, Clone, Copy)]
+pub struct LoadTestRunnerParameters {
+    pub workers: usize,
+    pub duration: usize,
+    pub summarize_last: usize,
+}
+
 pub trait Tracker: ::std::fmt::Debug + Copy + Clone + ::std::hash::Hash {
     fn name(&self) -> String;
 }
@@ -25,12 +32,14 @@ pub fn run_sets<C, F, I>(
     min_cores: Option<usize>,
     max_cores: Option<usize>,
     min_priority: Priority,
+    duration: usize,
+    summarize_last: usize,
     mut set_configs: IndexMap<usize, SetConfig<C, I>>,
     load_test_gen: F,
 ) where
     C: ::std::fmt::Debug,
     I: Tracker,
-    F: Fn(usize) -> Box<dyn ProcessRunner<Command = C>>,
+    F: Fn(LoadTestRunnerParameters) -> Box<dyn ProcessRunner<Command = C>>,
 {
     if let Some(min_cores) = min_cores {
         set_configs.retain(|cores, _| *cores >= min_cores);
@@ -59,7 +68,7 @@ pub fn run_sets<C, F, I>(
         .sum::<usize>();
 
     let (estimated_hours, estimated_minutes) = {
-        let minutes = (total_num_runs * 67) / 60;
+        let minutes = (total_num_runs * (duration + 7)) / 60;
 
         (minutes / 60, minutes % 60)
     };
@@ -96,13 +105,18 @@ pub fn run_sets<C, F, I>(
                                 .clone()
                                 .into_iter()
                                 .map(|(workers, _, load_test_vcpus)| {
+                                    let load_test_parameters = LoadTestRunnerParameters {
+                                        workers,
+                                        duration,
+                                        summarize_last,
+                                    };
                                     LoadTestRunResults::produce(
                                         command,
                                         &load_test_gen,
+                                        load_test_parameters,
                                         implementation,
                                         &tracker_run,
                                         tracker_vcpus.clone(),
-                                        workers,
                                         load_test_vcpus,
                                     )
                                 })
@@ -188,26 +202,26 @@ impl LoadTestRunResults {
     pub fn produce<C, F, I>(
         command: &C,
         load_test_gen: &F,
+        load_test_parameters: LoadTestRunnerParameters,
         implementation: I,
         tracker_process: &Rc<dyn ProcessRunner<Command = C>>,
         tracker_vcpus: TaskSetCpuList,
-        workers: usize,
         load_test_vcpus: TaskSetCpuList,
     ) -> Self
     where
         C: ::std::fmt::Debug,
         I: Tracker,
-        F: Fn(usize) -> Box<dyn ProcessRunner<Command = C>>,
+        F: Fn(LoadTestRunnerParameters) -> Box<dyn ProcessRunner<Command = C>>,
     {
         println!(
             "### {} run ({}) (load test workers: {}, cpus: {})",
             implementation.name(),
             tracker_process.info(),
-            workers,
+            load_test_parameters.workers,
             load_test_vcpus.as_cpu_list()
         );
 
-        let load_test_runner = load_test_gen(workers);
+        let load_test_runner = load_test_gen(load_test_parameters);
         let load_test_keys = load_test_runner.keys();
 
         let run_config = RunConfig {
@@ -217,7 +231,7 @@ impl LoadTestRunResults {
             load_test_vcpus: load_test_vcpus.clone(),
         };
 
-        match run_config.run(command) {
+        match run_config.run(command, load_test_parameters.duration) {
             Ok(r) => {
                 println!(
                     "- Average responses per second: {}",