From e18b50227c81489f00fca32a790215687055cfcc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20Frosteg=C3=A5rd?= Date: Wed, 3 Jan 2024 16:32:10 +0100 Subject: [PATCH] bencher: implement load test run priorities --- crates/bencher/src/common.rs | 25 ++- crates/bencher/src/main.rs | 12 +- crates/bencher/src/protocols/udp.rs | 241 +++++++++++++++++----------- crates/bencher/src/run.rs | 4 +- crates/bencher/src/set.rs | 28 +++- 5 files changed, 211 insertions(+), 99 deletions(-) diff --git a/crates/bencher/src/common.rs b/crates/bencher/src/common.rs index 3cb4e11..6d17664 100644 --- a/crates/bencher/src/common.rs +++ b/crates/bencher/src/common.rs @@ -2,6 +2,23 @@ use std::{fmt::Display, ops::Range, thread::available_parallelism}; use itertools::Itertools; +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, clap::ValueEnum)] +pub enum Priority { + Low, + Medium, + High, +} + +impl Display for Priority { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Low => f.write_str("low"), + Self::Medium => f.write_str("medium"), + Self::High => f.write_str("high"), + } + } +} + #[derive(Debug, Clone)] pub struct TaskSetCpuList(pub Vec); @@ -141,13 +158,17 @@ pub enum CpuDirection { Desc, } -pub fn simple_load_test_runs(cpu_mode: CpuMode, workers: &[usize]) -> Vec<(usize, TaskSetCpuList)> { +pub fn simple_load_test_runs( + cpu_mode: CpuMode, + workers: &[(usize, Priority)], +) -> Vec<(usize, Priority, TaskSetCpuList)> { workers .into_iter() .copied() - .map(|workers| { + .map(|(workers, priority)| { ( workers, + priority, TaskSetCpuList::new(cpu_mode, CpuDirection::Desc, workers).unwrap(), ) }) diff --git a/crates/bencher/src/main.rs b/crates/bencher/src/main.rs index 5494d5c..fbbc4c6 100644 --- a/crates/bencher/src/main.rs +++ b/crates/bencher/src/main.rs @@ -5,7 +5,7 @@ pub mod run; pub mod set; use clap::{Parser, Subcommand}; -use common::CpuMode; +use common::{CpuMode, Priority}; #[derive(Parser)] #[command(author, version, about)] @@ -20,6 +20,9 @@ struct Args { /// Maximum number of tracker cpu cores to run benchmarks for #[arg(long)] max_cores: Option, + /// Minimum benchmark priority + #[arg(long, default_value_t = Priority::Medium)] + min_priority: Priority, #[command(subcommand)] command: Command, } @@ -37,7 +40,12 @@ fn main() { match args.command { #[cfg(feature = "udp")] Command::Udp(command) => command - .run(args.cpu_mode, args.min_cores, args.max_cores) + .run( + args.cpu_mode, + args.min_cores, + args.max_cores, + args.min_priority, + ) .unwrap(), } } diff --git a/crates/bencher/src/protocols/udp.rs b/crates/bencher/src/protocols/udp.rs index 735162d..4180be5 100644 --- a/crates/bencher/src/protocols/udp.rs +++ b/crates/bencher/src/protocols/udp.rs @@ -11,7 +11,7 @@ use indoc::writedoc; use tempfile::NamedTempFile; use crate::{ - common::{simple_load_test_runs, CpuMode, TaskSetCpuList}, + common::{simple_load_test_runs, CpuMode, Priority, TaskSetCpuList}, run::ProcessRunner, set::{run_sets, SetConfig, Tracker}, }; @@ -55,173 +55,212 @@ impl UdpCommand { cpu_mode: CpuMode, min_cores: Option, max_cores: Option, + min_priority: Priority, ) -> anyhow::Result<()> { - let mut sets = self.sets(cpu_mode); + let sets = self.sets(cpu_mode); - if let Some(min_cores) = min_cores { - sets = sets.into_iter().filter(|(k, _)| *k >= min_cores).collect(); - } - if let Some(max_cores) = max_cores { - sets = sets.into_iter().filter(|(k, _)| *k <= max_cores).collect(); - } - - run_sets(self, cpu_mode, sets, |workers| { - Box::new(AquaticUdpLoadTestRunner { workers }) - }); + run_sets( + self, + cpu_mode, + min_cores, + max_cores, + min_priority, + sets, + |workers| Box::new(AquaticUdpLoadTestRunner { workers }), + ); Ok(()) } fn sets(&self, cpu_mode: CpuMode) -> IndexMap> { + // Priorities are based on what has previously produced the best results indexmap::indexmap! { 1 => SetConfig { implementations: indexmap! { UdpTracker::Aquatic => vec![ - AquaticUdpRunner::new(1, 1), - AquaticUdpRunner::new(2, 1), + AquaticUdpRunner::new(1, 1, Priority::High), + AquaticUdpRunner::new(2, 1, Priority::High), ], UdpTracker::OpenTracker => vec![ - OpenTrackerUdpRunner::new(0), // Handle requests within event loop - OpenTrackerUdpRunner::new(1), - OpenTrackerUdpRunner::new(2), + OpenTrackerUdpRunner::new(0, Priority::Low), // Handle requests within event loop + OpenTrackerUdpRunner::new(1, Priority::Medium), + OpenTrackerUdpRunner::new(2, Priority::High), ], UdpTracker::Chihaya => vec![ ChihayaUdpRunner::new(), ], }, - load_test_runs: simple_load_test_runs(cpu_mode, &[1, 2, 4, 6, 8]), + load_test_runs: simple_load_test_runs(cpu_mode, &[ + (1, Priority::High), + (2, Priority::Medium), + (4, Priority::Medium), + (6, Priority::Medium), + (8, Priority::High) + ]), }, 2 => SetConfig { implementations: indexmap! { UdpTracker::Aquatic => vec![ - AquaticUdpRunner::new(1, 1), - AquaticUdpRunner::new(2, 1), - AquaticUdpRunner::new(3, 1), + AquaticUdpRunner::new(1, 1, Priority::Low), + AquaticUdpRunner::new(2, 1, Priority::Medium), + AquaticUdpRunner::new(3, 1, Priority::High), ], UdpTracker::OpenTracker => vec![ - OpenTrackerUdpRunner::new(2), - OpenTrackerUdpRunner::new(4), + OpenTrackerUdpRunner::new(2, Priority::Medium), + OpenTrackerUdpRunner::new(4, Priority::High), ], UdpTracker::Chihaya => vec![ ChihayaUdpRunner::new(), ], }, - load_test_runs: simple_load_test_runs(cpu_mode, &[1, 2, 4, 6, 8]), + load_test_runs: simple_load_test_runs(cpu_mode, &[ + (1, Priority::Medium), + (2, Priority::Medium), + (4, Priority::Medium), + (6, Priority::Medium), + (8, Priority::High) + ]), }, 4 => SetConfig { implementations: indexmap! { UdpTracker::Aquatic => vec![ - AquaticUdpRunner::new(3, 1), - AquaticUdpRunner::new(4, 1), - AquaticUdpRunner::new(5, 1), - AquaticUdpRunner::new(6, 1), - AquaticUdpRunner::new(7, 1), + AquaticUdpRunner::new(3, 1, Priority::Low), + AquaticUdpRunner::new(4, 1, Priority::Low), + AquaticUdpRunner::new(5, 1, Priority::Medium), + AquaticUdpRunner::new(6, 1, Priority::Medium), + AquaticUdpRunner::new(7, 1, Priority::High), ], UdpTracker::OpenTracker => vec![ - OpenTrackerUdpRunner::new(4), - OpenTrackerUdpRunner::new(8), + OpenTrackerUdpRunner::new(4, Priority::High), + OpenTrackerUdpRunner::new(8, Priority::Medium), ], UdpTracker::Chihaya => vec![ ChihayaUdpRunner::new(), ], }, - load_test_runs: simple_load_test_runs(cpu_mode, &[6, 8, 12, 16]), + load_test_runs: simple_load_test_runs(cpu_mode, &[ + (6, Priority::High), + (8, Priority::Medium), + (12, Priority::High), + (16, Priority::Medium) + ]), }, 6 => SetConfig { implementations: indexmap! { UdpTracker::Aquatic => vec![ - AquaticUdpRunner::new(5, 1), - AquaticUdpRunner::new(6, 1), - AquaticUdpRunner::new(10, 1), + AquaticUdpRunner::new(5, 1, Priority::Medium), + AquaticUdpRunner::new(6, 1, Priority::Medium), + AquaticUdpRunner::new(10, 1, Priority::Low), - AquaticUdpRunner::new(4, 2), - AquaticUdpRunner::new(6, 2), - AquaticUdpRunner::new(8, 2), + AquaticUdpRunner::new(4, 2, Priority::Low), + AquaticUdpRunner::new(6, 2, Priority::Medium), + AquaticUdpRunner::new(8, 2, Priority::High), ], UdpTracker::OpenTracker => vec![ - OpenTrackerUdpRunner::new(6), - OpenTrackerUdpRunner::new(12), + OpenTrackerUdpRunner::new(6, Priority::High), + OpenTrackerUdpRunner::new(12, Priority::Medium), ], }, - load_test_runs: simple_load_test_runs(cpu_mode, &[6, 8, 12, 16, 24]), + load_test_runs: simple_load_test_runs(cpu_mode, &[ + (6, Priority::Medium), + (8, Priority::Medium), + (12, Priority::High), + (16, Priority::High), + (24, Priority::Medium), + ]), }, 8 => SetConfig { implementations: indexmap! { UdpTracker::Aquatic => vec![ - AquaticUdpRunner::new(7, 1), - AquaticUdpRunner::new(8, 1), - AquaticUdpRunner::new(14, 1), - AquaticUdpRunner::new(6, 2), - AquaticUdpRunner::new(12, 2), - AquaticUdpRunner::new(5, 3), - AquaticUdpRunner::new(10, 3), + AquaticUdpRunner::new(7, 1, Priority::Medium), + AquaticUdpRunner::new(8, 1, Priority::Medium), + AquaticUdpRunner::new(14, 1, Priority::Low), + AquaticUdpRunner::new(6, 2, Priority::Low), + AquaticUdpRunner::new(12, 2, Priority::High), + AquaticUdpRunner::new(5, 3, Priority::Low), + AquaticUdpRunner::new(10, 3, Priority::Medium), ], UdpTracker::OpenTracker => vec![ - OpenTrackerUdpRunner::new(8), - OpenTrackerUdpRunner::new(16), + OpenTrackerUdpRunner::new(8, Priority::High), + OpenTrackerUdpRunner::new(16, Priority::Medium), ], }, - load_test_runs: simple_load_test_runs(cpu_mode, &[8, 12, 16, 24]), + load_test_runs: simple_load_test_runs(cpu_mode, &[ + (8, Priority::High), + (12, Priority::Medium), + (16, Priority::High), + (24, Priority::Medium) + ]), }, 12 => SetConfig { implementations: indexmap! { UdpTracker::Aquatic => vec![ - AquaticUdpRunner::new(10, 2), - AquaticUdpRunner::new(12, 2), - AquaticUdpRunner::new(20, 2), + AquaticUdpRunner::new(10, 2, Priority::Medium), + AquaticUdpRunner::new(12, 2, Priority::Medium), + AquaticUdpRunner::new(20, 2, Priority::Low), - AquaticUdpRunner::new(9, 3), - AquaticUdpRunner::new(12, 3), - AquaticUdpRunner::new(18, 3), + AquaticUdpRunner::new(9, 3, Priority::Low), + AquaticUdpRunner::new(12, 3, Priority::Medium), + AquaticUdpRunner::new(18, 3, Priority::Low), - AquaticUdpRunner::new(8, 4), - AquaticUdpRunner::new(12, 4), - AquaticUdpRunner::new(16, 4), + AquaticUdpRunner::new(8, 4, Priority::Low), + AquaticUdpRunner::new(12, 4, Priority::Medium), + AquaticUdpRunner::new(16, 4, Priority::High), - AquaticUdpRunner::new(7, 5), - AquaticUdpRunner::new(12, 5), - AquaticUdpRunner::new(14, 5), + AquaticUdpRunner::new(7, 5, Priority::Low), + AquaticUdpRunner::new(12, 5, Priority::Medium), + AquaticUdpRunner::new(14, 5, Priority::Medium), ], UdpTracker::OpenTracker => vec![ - OpenTrackerUdpRunner::new(12), - OpenTrackerUdpRunner::new(24), + OpenTrackerUdpRunner::new(12, Priority::High), + OpenTrackerUdpRunner::new(24, Priority::Medium), ], }, - load_test_runs: simple_load_test_runs(cpu_mode, &[8, 12, 16, 24]), + load_test_runs: simple_load_test_runs(cpu_mode, &[ + (8, Priority::Medium), + (12, Priority::Medium), + (16, Priority::High), + (24, Priority::High), + ]), }, 16 => SetConfig { implementations: indexmap! { UdpTracker::Aquatic => vec![ - AquaticUdpRunner::new(14, 2), - AquaticUdpRunner::new(16, 2), - AquaticUdpRunner::new(28, 2), + AquaticUdpRunner::new(14, 2, Priority::Low), + AquaticUdpRunner::new(16, 2, Priority::Low), + AquaticUdpRunner::new(28, 2, Priority::Low), - AquaticUdpRunner::new(13, 3), - AquaticUdpRunner::new(16, 3), - AquaticUdpRunner::new(26, 3), + AquaticUdpRunner::new(13, 3, Priority::Low), + AquaticUdpRunner::new(16, 3, Priority::Low), + AquaticUdpRunner::new(26, 3, Priority::Low), - AquaticUdpRunner::new(12, 4), - AquaticUdpRunner::new(16, 4), - AquaticUdpRunner::new(24, 4), + AquaticUdpRunner::new(12, 4, Priority::Medium), + AquaticUdpRunner::new(16, 4, Priority::Medium), + AquaticUdpRunner::new(24, 4, Priority::Low), - AquaticUdpRunner::new(11, 5), - AquaticUdpRunner::new(16, 5), - AquaticUdpRunner::new(22, 5), + AquaticUdpRunner::new(11, 5, Priority::Low), + AquaticUdpRunner::new(16, 5, Priority::Medium), + AquaticUdpRunner::new(22, 5, Priority::Low), - AquaticUdpRunner::new(10, 6), - AquaticUdpRunner::new(16, 6), - AquaticUdpRunner::new(20, 6), + AquaticUdpRunner::new(10, 6, Priority::Low), + AquaticUdpRunner::new(16, 6, Priority::High), + AquaticUdpRunner::new(20, 6, Priority::Medium), - AquaticUdpRunner::new(9, 7), - AquaticUdpRunner::new(16, 7), - AquaticUdpRunner::new(18, 7), + AquaticUdpRunner::new(9, 7, Priority::Low), + AquaticUdpRunner::new(16, 7, Priority::Medium), + AquaticUdpRunner::new(18, 7, Priority::Low), ], UdpTracker::OpenTracker => vec![ - OpenTrackerUdpRunner::new(16), - OpenTrackerUdpRunner::new(32), + OpenTrackerUdpRunner::new(16, Priority::High), + OpenTrackerUdpRunner::new(32, Priority::Medium), ], }, - load_test_runs: simple_load_test_runs(cpu_mode, &[8, 12, 16, 24]), + load_test_runs: simple_load_test_runs(cpu_mode, &[ + (8, Priority::High), + (12, Priority::High), + (16, Priority::High), + (24, Priority::High), + ]), }, } } @@ -231,16 +270,19 @@ impl UdpCommand { struct AquaticUdpRunner { socket_workers: usize, swarm_workers: usize, + priority: Priority, } impl AquaticUdpRunner { fn new( socket_workers: usize, swarm_workers: usize, + priority: Priority, ) -> Rc> { Rc::new(Self { socket_workers, swarm_workers, + priority, }) } } @@ -275,6 +317,10 @@ impl ProcessRunner for AquaticUdpRunner { .spawn()?) } + fn priority(&self) -> crate::common::Priority { + self.priority + } + fn keys(&self) -> IndexMap { indexmap! { "socket workers".to_string() => self.socket_workers.to_string(), @@ -286,11 +332,12 @@ impl ProcessRunner for AquaticUdpRunner { #[derive(Debug, Clone)] struct OpenTrackerUdpRunner { workers: usize, + priority: Priority, } impl OpenTrackerUdpRunner { - fn new(workers: usize) -> Rc> { - Rc::new(Self { workers }) + fn new(workers: usize, priority: Priority) -> Rc> { + Rc::new(Self { workers, priority }) } } @@ -320,6 +367,10 @@ impl ProcessRunner for OpenTrackerUdpRunner { .spawn()?) } + fn priority(&self) -> crate::common::Priority { + self.priority + } + fn keys(&self) -> IndexMap { indexmap! { "workers".to_string() => self.workers.to_string(), @@ -332,7 +383,7 @@ struct ChihayaUdpRunner; impl ChihayaUdpRunner { fn new() -> Rc> { - Rc::new(Self) + Rc::new(Self {}) } } @@ -372,6 +423,10 @@ impl ProcessRunner for ChihayaUdpRunner { .spawn()?) } + fn priority(&self) -> crate::common::Priority { + Priority::High + } + fn keys(&self) -> IndexMap { Default::default() } @@ -415,6 +470,12 @@ impl ProcessRunner for AquaticUdpLoadTestRunner { .spawn()?) } + fn priority(&self) -> crate::common::Priority { + eprintln!("load test runner priority method called"); + + Priority::High + } + fn keys(&self) -> IndexMap { indexmap! { "workers".to_string() => self.workers.to_string(), diff --git a/crates/bencher/src/run.rs b/crates/bencher/src/run.rs index d3b0210..0f0fc4f 100644 --- a/crates/bencher/src/run.rs +++ b/crates/bencher/src/run.rs @@ -12,7 +12,7 @@ use once_cell::sync::Lazy; use regex::Regex; use tempfile::NamedTempFile; -use crate::common::TaskSetCpuList; +use crate::common::{Priority, TaskSetCpuList}; pub trait ProcessRunner: ::std::fmt::Debug { type Command; @@ -26,6 +26,8 @@ pub trait ProcessRunner: ::std::fmt::Debug { fn keys(&self) -> IndexMap; + fn priority(&self) -> Priority; + fn info(&self) -> String { self.keys() .into_iter() diff --git a/crates/bencher/src/set.rs b/crates/bencher/src/set.rs index d4f7f80..a7d59c0 100644 --- a/crates/bencher/src/set.rs +++ b/crates/bencher/src/set.rs @@ -5,7 +5,7 @@ use indexmap::IndexMap; use num_format::{Locale, ToFormattedString}; use crate::{ - common::{CpuDirection, CpuMode, TaskSetCpuList}, + common::{CpuDirection, CpuMode, Priority, TaskSetCpuList}, html::{html_all_runs, html_best_results}, run::{ProcessRunner, ProcessStats, RunConfig}, }; @@ -16,19 +16,39 @@ pub trait Tracker: ::std::fmt::Debug + Copy + Clone + ::std::hash::Hash { pub struct SetConfig { pub implementations: IndexMap>>>, - pub load_test_runs: Vec<(usize, TaskSetCpuList)>, + pub load_test_runs: Vec<(usize, Priority, TaskSetCpuList)>, } pub fn run_sets( command: &C, cpu_mode: CpuMode, - set_configs: IndexMap>, + min_cores: Option, + max_cores: Option, + min_priority: Priority, + mut set_configs: IndexMap>, load_test_gen: F, ) where C: ::std::fmt::Debug, I: Tracker, F: Fn(usize) -> Box>, { + if let Some(min_cores) = min_cores { + set_configs.retain(|cores, _| *cores >= min_cores); + } + if let Some(max_cores) = max_cores { + set_configs.retain(|cores, _| *cores <= max_cores); + } + + for set_config in set_configs.values_mut() { + for runners in set_config.implementations.values_mut() { + runners.retain(|r| r.priority() >= min_priority); + } + + set_config + .load_test_runs + .retain(|(_, priority, _)| *priority >= min_priority); + } + println!("# Benchmark report"); let total_num_runs = set_configs @@ -75,7 +95,7 @@ pub fn run_sets( .load_test_runs .clone() .into_iter() - .map(|(workers, load_test_vcpus)| { + .map(|(workers, _, load_test_vcpus)| { LoadTestRunResults::produce( command, &load_test_gen,