From 85862f161a82fb6592083d252df04025f949c224 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20Frosteg=C3=A5rd?= Date: Wed, 3 Jan 2024 17:36:04 +0100 Subject: [PATCH] bencher: add args duration and summarize_last (seconds) --- crates/bencher/src/main.rs | 28 +++++++++++++++++--- crates/bencher/src/protocols/udp.rs | 41 +++++++++-------------------- crates/bencher/src/run.rs | 8 ++++-- crates/bencher/src/set.rs | 30 +++++++++++++++------ 4 files changed, 65 insertions(+), 42 deletions(-) diff --git a/crates/bencher/src/main.rs b/crates/bencher/src/main.rs index fbbc4c6..1ced040 100644 --- a/crates/bencher/src/main.rs +++ b/crates/bencher/src/main.rs @@ -6,6 +6,7 @@ pub mod set; use clap::{Parser, Subcommand}; use common::{CpuMode, Priority}; +use set::run_sets; #[derive(Parser)] #[command(author, version, about)] @@ -23,6 +24,17 @@ struct Args { /// Minimum benchmark priority #[arg(long, default_value_t = Priority::Medium)] min_priority: Priority, + /// How long to run each load test for + #[arg(long, default_value_t = 60)] + duration: usize, + /// Only include data for last N seconds of load test runs. + /// + /// Useful if the tracker/load tester combination is slow at reaching + /// maximum throughput + /// + /// 0 = use data for whole run + #[arg(long, default_value_t = 0)] + summarize_last: usize, #[command(subcommand)] command: Command, } @@ -39,13 +51,21 @@ fn main() { match args.command { #[cfg(feature = "udp")] - Command::Udp(command) => command - .run( + Command::Udp(command) => { + let sets = command.sets(args.cpu_mode); + let load_test_gen = protocols::udp::UdpCommand::load_test_gen; + + run_sets( + &command, args.cpu_mode, args.min_cores, args.max_cores, args.min_priority, - ) - .unwrap(), + args.duration, + args.summarize_last, + sets, + load_test_gen, + ); + } } } diff --git a/crates/bencher/src/protocols/udp.rs b/crates/bencher/src/protocols/udp.rs index 4180be5..68151d8 100644 --- a/crates/bencher/src/protocols/udp.rs +++ b/crates/bencher/src/protocols/udp.rs @@ -13,7 +13,7 @@ use tempfile::NamedTempFile; use crate::{ common::{simple_load_test_runs, CpuMode, Priority, TaskSetCpuList}, run::ProcessRunner, - set::{run_sets, SetConfig, Tracker}, + set::{LoadTestRunnerParameters, SetConfig, Tracker}, }; #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] @@ -50,29 +50,7 @@ pub struct UdpCommand { } impl UdpCommand { - pub fn run( - &self, - cpu_mode: CpuMode, - min_cores: Option, - max_cores: Option, - min_priority: Priority, - ) -> anyhow::Result<()> { - let sets = self.sets(cpu_mode); - - run_sets( - self, - cpu_mode, - min_cores, - max_cores, - min_priority, - sets, - |workers| Box::new(AquaticUdpLoadTestRunner { workers }), - ); - - Ok(()) - } - - fn sets(&self, cpu_mode: CpuMode) -> IndexMap> { + pub fn sets(&self, cpu_mode: CpuMode) -> IndexMap> { // Priorities are based on what has previously produced the best results indexmap::indexmap! { 1 => SetConfig { @@ -264,6 +242,12 @@ impl UdpCommand { }, } } + + pub fn load_test_gen( + parameters: LoadTestRunnerParameters, + ) -> Box> { + Box::new(AquaticUdpLoadTestRunner { parameters }) + } } #[derive(Debug, Clone)] @@ -434,7 +418,7 @@ impl ProcessRunner for ChihayaUdpRunner { #[derive(Debug, Clone)] struct AquaticUdpLoadTestRunner { - workers: usize, + parameters: LoadTestRunnerParameters, } impl ProcessRunner for AquaticUdpLoadTestRunner { @@ -448,8 +432,9 @@ impl ProcessRunner for AquaticUdpLoadTestRunner { ) -> anyhow::Result { let mut c = aquatic_udp_load_test::config::Config::default(); - c.workers = self.workers as u8; - c.duration = 60; + c.workers = self.parameters.workers as u8; + c.duration = self.parameters.duration; + c.summarize_last = self.parameters.summarize_last; c.requests.weight_connect = 0; c.requests.weight_announce = 100; @@ -478,7 +463,7 @@ impl ProcessRunner for AquaticUdpLoadTestRunner { fn keys(&self) -> IndexMap { indexmap! { - "workers".to_string() => self.workers.to_string(), + "workers".to_string() => self.parameters.workers.to_string(), } } } diff --git a/crates/bencher/src/run.rs b/crates/bencher/src/run.rs index 0f0fc4f..2757404 100644 --- a/crates/bencher/src/run.rs +++ b/crates/bencher/src/run.rs @@ -45,7 +45,11 @@ pub struct RunConfig { } impl RunConfig { - pub fn run(self, command: &C) -> Result> { + pub fn run( + self, + command: &C, + duration: usize, + ) -> Result> { let mut tracker_config_file = NamedTempFile::new().unwrap(); let mut load_test_config_file = NamedTempFile::new().unwrap(); @@ -75,7 +79,7 @@ impl RunConfig { } }; - for _ in 0..59 { + for _ in 0..(duration - 1) { if let Ok(Some(status)) = tracker.0.try_wait() { return Err(RunErrorResults::new(self) .set_tracker_outputs(tracker) diff --git a/crates/bencher/src/set.rs b/crates/bencher/src/set.rs index a7d59c0..a34a3e7 100644 --- a/crates/bencher/src/set.rs +++ b/crates/bencher/src/set.rs @@ -10,6 +10,13 @@ use crate::{ run::{ProcessRunner, ProcessStats, RunConfig}, }; +#[derive(Debug, Clone, Copy)] +pub struct LoadTestRunnerParameters { + pub workers: usize, + pub duration: usize, + pub summarize_last: usize, +} + pub trait Tracker: ::std::fmt::Debug + Copy + Clone + ::std::hash::Hash { fn name(&self) -> String; } @@ -25,12 +32,14 @@ pub fn run_sets( min_cores: Option, max_cores: Option, min_priority: Priority, + duration: usize, + summarize_last: usize, mut set_configs: IndexMap>, load_test_gen: F, ) where C: ::std::fmt::Debug, I: Tracker, - F: Fn(usize) -> Box>, + F: Fn(LoadTestRunnerParameters) -> Box>, { if let Some(min_cores) = min_cores { set_configs.retain(|cores, _| *cores >= min_cores); @@ -59,7 +68,7 @@ pub fn run_sets( .sum::(); let (estimated_hours, estimated_minutes) = { - let minutes = (total_num_runs * 67) / 60; + let minutes = (total_num_runs * (duration + 7)) / 60; (minutes / 60, minutes % 60) }; @@ -96,13 +105,18 @@ pub fn run_sets( .clone() .into_iter() .map(|(workers, _, load_test_vcpus)| { + let load_test_parameters = LoadTestRunnerParameters { + workers, + duration, + summarize_last, + }; LoadTestRunResults::produce( command, &load_test_gen, + load_test_parameters, implementation, &tracker_run, tracker_vcpus.clone(), - workers, load_test_vcpus, ) }) @@ -188,26 +202,26 @@ impl LoadTestRunResults { pub fn produce( command: &C, load_test_gen: &F, + load_test_parameters: LoadTestRunnerParameters, implementation: I, tracker_process: &Rc>, tracker_vcpus: TaskSetCpuList, - workers: usize, load_test_vcpus: TaskSetCpuList, ) -> Self where C: ::std::fmt::Debug, I: Tracker, - F: Fn(usize) -> Box>, + F: Fn(LoadTestRunnerParameters) -> Box>, { println!( "### {} run ({}) (load test workers: {}, cpus: {})", implementation.name(), tracker_process.info(), - workers, + load_test_parameters.workers, load_test_vcpus.as_cpu_list() ); - let load_test_runner = load_test_gen(workers); + let load_test_runner = load_test_gen(load_test_parameters); let load_test_keys = load_test_runner.keys(); let run_config = RunConfig { @@ -217,7 +231,7 @@ impl LoadTestRunResults { load_test_vcpus: load_test_vcpus.clone(), }; - match run_config.run(command) { + match run_config.run(command, load_test_parameters.duration) { Ok(r) => { println!( "- Average responses per second: {}",