YGGverse/aquatic
1
0
Fork 0
mirror of https://github.com/YGGverse/aquatic.git synced 2026-03-31 17:55:36 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

bencher: implement load test run priorities

Browse source
This commit is contained in:
Joakim Frostegård 2024-01-03 16:32:10 +01:00
parent 74155b4d79
commit e18b50227c
5 changed files with 211 additions and 99 deletions
Download patch file Download diff file Expand all files Collapse all files

25
crates/bencher/src/common.rs
View file

@ -2,6 +2,23 @@ use std::{fmt::Display, ops::Range, thread::available_parallelism};
use itertools::Itertools; use itertools::Itertools;
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, clap::ValueEnum)]
pub enum Priority {
Low,
Medium,
High,
}
impl Display for Priority {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Self::Low => f.write_str("low"),
Self::Medium => f.write_str("medium"),
Self::High => f.write_str("high"),
}
}
}
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
pub struct TaskSetCpuList(pub Vec<TaskSetCpuIndicator>); pub struct TaskSetCpuList(pub Vec<TaskSetCpuIndicator>);
@ -141,13 +158,17 @@ pub enum CpuDirection {
Desc, Desc,
} }
pub fn simple_load_test_runs(cpu_mode: CpuMode, workers: &[usize]) -> Vec<(usize, TaskSetCpuList)> { pub fn simple_load_test_runs(
cpu_mode: CpuMode,
workers: &[(usize, Priority)],
) -> Vec<(usize, Priority, TaskSetCpuList)> {
workers workers
.into_iter() .into_iter()
.copied() .copied()
.map(|workers| { .map(|(workers, priority)| {
( (
workers, workers,
priority,
TaskSetCpuList::new(cpu_mode, CpuDirection::Desc, workers).unwrap(), TaskSetCpuList::new(cpu_mode, CpuDirection::Desc, workers).unwrap(),
) )
}) })

12
crates/bencher/src/main.rs
View file

@ -5,7 +5,7 @@ pub mod run;
pub mod set; pub mod set;
use clap::{Parser, Subcommand}; use clap::{Parser, Subcommand};
use common::CpuMode; use common::{CpuMode, Priority};
#[derive(Parser)] #[derive(Parser)]
#[command(author, version, about)] #[command(author, version, about)]
@ -20,6 +20,9 @@ struct Args {
/// Maximum number of tracker cpu cores to run benchmarks for /// Maximum number of tracker cpu cores to run benchmarks for
#[arg(long)] #[arg(long)]
max_cores: Option<usize>, max_cores: Option<usize>,
/// Minimum benchmark priority
#[arg(long, default_value_t = Priority::Medium)]
min_priority: Priority,
#[command(subcommand)] #[command(subcommand)]
command: Command, command: Command,
} }
@ -37,7 +40,12 @@ fn main() {
match args.command { match args.command {
#[cfg(feature = "udp")] #[cfg(feature = "udp")]
Command::Udp(command) => command Command::Udp(command) => command
.run(args.cpu_mode, args.min_cores, args.max_cores) .run(
args.cpu_mode,
args.min_cores,
args.max_cores,
args.min_priority,
)
.unwrap(), .unwrap(),
} }
} }

241
crates/bencher/src/protocols/udp.rs
View file

@ -11,7 +11,7 @@ use indoc::writedoc;
use tempfile::NamedTempFile; use tempfile::NamedTempFile;
use crate::{ use crate::{
common::{simple_load_test_runs, CpuMode, TaskSetCpuList}, common::{simple_load_test_runs, CpuMode, Priority, TaskSetCpuList},
run::ProcessRunner, run::ProcessRunner,
set::{run_sets, SetConfig, Tracker}, set::{run_sets, SetConfig, Tracker},
}; };
@ -55,173 +55,212 @@ impl UdpCommand {
cpu_mode: CpuMode, cpu_mode: CpuMode,
min_cores: Option<usize>, min_cores: Option<usize>,
max_cores: Option<usize>, max_cores: Option<usize>,
min_priority: Priority,
) -> anyhow::Result<()> { ) -> anyhow::Result<()> {
let mut sets = self.sets(cpu_mode); let sets = self.sets(cpu_mode);
if let Some(min_cores) = min_cores { run_sets(
sets = sets.into_iter().filter(|(k, _)| *k >= min_cores).collect(); self,
} cpu_mode,
if let Some(max_cores) = max_cores { min_cores,
sets = sets.into_iter().filter(|(k, _)| *k <= max_cores).collect(); max_cores,
} min_priority,
sets,
run_sets(self, cpu_mode, sets, |workers| { |workers| Box::new(AquaticUdpLoadTestRunner { workers }),
Box::new(AquaticUdpLoadTestRunner { workers }) );
});
Ok(()) Ok(())
} }
fn sets(&self, cpu_mode: CpuMode) -> IndexMap<usize, SetConfig<UdpCommand, UdpTracker>> { fn sets(&self, cpu_mode: CpuMode) -> IndexMap<usize, SetConfig<UdpCommand, UdpTracker>> {
// Priorities are based on what has previously produced the best results
indexmap::indexmap! { indexmap::indexmap! {
1 => SetConfig { 1 => SetConfig {
implementations: indexmap! { implementations: indexmap! {
UdpTracker::Aquatic => vec![ UdpTracker::Aquatic => vec![
AquaticUdpRunner::new(1, 1), AquaticUdpRunner::new(1, 1, Priority::High),
AquaticUdpRunner::new(2, 1), AquaticUdpRunner::new(2, 1, Priority::High),
], ],
UdpTracker::OpenTracker => vec![ UdpTracker::OpenTracker => vec![
OpenTrackerUdpRunner::new(0), // Handle requests within event loop OpenTrackerUdpRunner::new(0, Priority::Low), // Handle requests within event loop
OpenTrackerUdpRunner::new(1), OpenTrackerUdpRunner::new(1, Priority::Medium),
OpenTrackerUdpRunner::new(2), OpenTrackerUdpRunner::new(2, Priority::High),
], ],
UdpTracker::Chihaya => vec![ UdpTracker::Chihaya => vec![
ChihayaUdpRunner::new(), ChihayaUdpRunner::new(),
], ],
}, },
load_test_runs: simple_load_test_runs(cpu_mode, &[1, 2, 4, 6, 8]), load_test_runs: simple_load_test_runs(cpu_mode, &[
(1, Priority::High),
(2, Priority::Medium),
(4, Priority::Medium),
(6, Priority::Medium),
(8, Priority::High)
]),
}, },
2 => SetConfig { 2 => SetConfig {
implementations: indexmap! { implementations: indexmap! {
UdpTracker::Aquatic => vec![ UdpTracker::Aquatic => vec![
AquaticUdpRunner::new(1, 1), AquaticUdpRunner::new(1, 1, Priority::Low),
AquaticUdpRunner::new(2, 1), AquaticUdpRunner::new(2, 1, Priority::Medium),
AquaticUdpRunner::new(3, 1), AquaticUdpRunner::new(3, 1, Priority::High),
], ],
UdpTracker::OpenTracker => vec![ UdpTracker::OpenTracker => vec![
OpenTrackerUdpRunner::new(2), OpenTrackerUdpRunner::new(2, Priority::Medium),
OpenTrackerUdpRunner::new(4), OpenTrackerUdpRunner::new(4, Priority::High),
], ],
UdpTracker::Chihaya => vec![ UdpTracker::Chihaya => vec![
ChihayaUdpRunner::new(), ChihayaUdpRunner::new(),
], ],
}, },
load_test_runs: simple_load_test_runs(cpu_mode, &[1, 2, 4, 6, 8]), load_test_runs: simple_load_test_runs(cpu_mode, &[
(1, Priority::Medium),
(2, Priority::Medium),
(4, Priority::Medium),
(6, Priority::Medium),
(8, Priority::High)
]),
}, },
4 => SetConfig { 4 => SetConfig {
implementations: indexmap! { implementations: indexmap! {
UdpTracker::Aquatic => vec![ UdpTracker::Aquatic => vec![
AquaticUdpRunner::new(3, 1), AquaticUdpRunner::new(3, 1, Priority::Low),
AquaticUdpRunner::new(4, 1), AquaticUdpRunner::new(4, 1, Priority::Low),
AquaticUdpRunner::new(5, 1), AquaticUdpRunner::new(5, 1, Priority::Medium),
AquaticUdpRunner::new(6, 1), AquaticUdpRunner::new(6, 1, Priority::Medium),
AquaticUdpRunner::new(7, 1), AquaticUdpRunner::new(7, 1, Priority::High),
], ],
UdpTracker::OpenTracker => vec![ UdpTracker::OpenTracker => vec![
OpenTrackerUdpRunner::new(4), OpenTrackerUdpRunner::new(4, Priority::High),
OpenTrackerUdpRunner::new(8), OpenTrackerUdpRunner::new(8, Priority::Medium),
], ],
UdpTracker::Chihaya => vec![ UdpTracker::Chihaya => vec![
ChihayaUdpRunner::new(), ChihayaUdpRunner::new(),
], ],
}, },
load_test_runs: simple_load_test_runs(cpu_mode, &[6, 8, 12, 16]), load_test_runs: simple_load_test_runs(cpu_mode, &[
(6, Priority::High),
(8, Priority::Medium),
(12, Priority::High),
(16, Priority::Medium)
]),
}, },
6 => SetConfig { 6 => SetConfig {
implementations: indexmap! { implementations: indexmap! {
UdpTracker::Aquatic => vec![ UdpTracker::Aquatic => vec![
AquaticUdpRunner::new(5, 1), AquaticUdpRunner::new(5, 1, Priority::Medium),
AquaticUdpRunner::new(6, 1), AquaticUdpRunner::new(6, 1, Priority::Medium),
AquaticUdpRunner::new(10, 1), AquaticUdpRunner::new(10, 1, Priority::Low),
AquaticUdpRunner::new(4, 2), AquaticUdpRunner::new(4, 2, Priority::Low),
AquaticUdpRunner::new(6, 2), AquaticUdpRunner::new(6, 2, Priority::Medium),
AquaticUdpRunner::new(8, 2), AquaticUdpRunner::new(8, 2, Priority::High),
], ],
UdpTracker::OpenTracker => vec![ UdpTracker::OpenTracker => vec![
OpenTrackerUdpRunner::new(6), OpenTrackerUdpRunner::new(6, Priority::High),
OpenTrackerUdpRunner::new(12), OpenTrackerUdpRunner::new(12, Priority::Medium),
], ],
}, },
load_test_runs: simple_load_test_runs(cpu_mode, &[6, 8, 12, 16, 24]), load_test_runs: simple_load_test_runs(cpu_mode, &[
(6, Priority::Medium),
(8, Priority::Medium),
(12, Priority::High),
(16, Priority::High),
(24, Priority::Medium),
]),
}, },
8 => SetConfig { 8 => SetConfig {
implementations: indexmap! { implementations: indexmap! {
UdpTracker::Aquatic => vec![ UdpTracker::Aquatic => vec![
AquaticUdpRunner::new(7, 1), AquaticUdpRunner::new(7, 1, Priority::Medium),
AquaticUdpRunner::new(8, 1), AquaticUdpRunner::new(8, 1, Priority::Medium),
AquaticUdpRunner::new(14, 1), AquaticUdpRunner::new(14, 1, Priority::Low),
AquaticUdpRunner::new(6, 2), AquaticUdpRunner::new(6, 2, Priority::Low),
AquaticUdpRunner::new(12, 2), AquaticUdpRunner::new(12, 2, Priority::High),
AquaticUdpRunner::new(5, 3), AquaticUdpRunner::new(5, 3, Priority::Low),
AquaticUdpRunner::new(10, 3), AquaticUdpRunner::new(10, 3, Priority::Medium),
], ],
UdpTracker::OpenTracker => vec![ UdpTracker::OpenTracker => vec![
OpenTrackerUdpRunner::new(8), OpenTrackerUdpRunner::new(8, Priority::High),
OpenTrackerUdpRunner::new(16), OpenTrackerUdpRunner::new(16, Priority::Medium),
], ],
}, },
load_test_runs: simple_load_test_runs(cpu_mode, &[8, 12, 16, 24]), load_test_runs: simple_load_test_runs(cpu_mode, &[
(8, Priority::High),
(12, Priority::Medium),
(16, Priority::High),
(24, Priority::Medium)
]),
}, },
12 => SetConfig { 12 => SetConfig {
implementations: indexmap! { implementations: indexmap! {
UdpTracker::Aquatic => vec![ UdpTracker::Aquatic => vec![
AquaticUdpRunner::new(10, 2), AquaticUdpRunner::new(10, 2, Priority::Medium),
AquaticUdpRunner::new(12, 2), AquaticUdpRunner::new(12, 2, Priority::Medium),
AquaticUdpRunner::new(20, 2), AquaticUdpRunner::new(20, 2, Priority::Low),
AquaticUdpRunner::new(9, 3), AquaticUdpRunner::new(9, 3, Priority::Low),
AquaticUdpRunner::new(12, 3), AquaticUdpRunner::new(12, 3, Priority::Medium),
AquaticUdpRunner::new(18, 3), AquaticUdpRunner::new(18, 3, Priority::Low),
AquaticUdpRunner::new(8, 4), AquaticUdpRunner::new(8, 4, Priority::Low),
AquaticUdpRunner::new(12, 4), AquaticUdpRunner::new(12, 4, Priority::Medium),
AquaticUdpRunner::new(16, 4), AquaticUdpRunner::new(16, 4, Priority::High),
AquaticUdpRunner::new(7, 5), AquaticUdpRunner::new(7, 5, Priority::Low),
AquaticUdpRunner::new(12, 5), AquaticUdpRunner::new(12, 5, Priority::Medium),
AquaticUdpRunner::new(14, 5), AquaticUdpRunner::new(14, 5, Priority::Medium),
], ],
UdpTracker::OpenTracker => vec![ UdpTracker::OpenTracker => vec![
OpenTrackerUdpRunner::new(12), OpenTrackerUdpRunner::new(12, Priority::High),
OpenTrackerUdpRunner::new(24), OpenTrackerUdpRunner::new(24, Priority::Medium),
], ],
}, },
load_test_runs: simple_load_test_runs(cpu_mode, &[8, 12, 16, 24]), load_test_runs: simple_load_test_runs(cpu_mode, &[
(8, Priority::Medium),
(12, Priority::Medium),
(16, Priority::High),
(24, Priority::High),
]),
}, },
16 => SetConfig { 16 => SetConfig {
implementations: indexmap! { implementations: indexmap! {
UdpTracker::Aquatic => vec![ UdpTracker::Aquatic => vec![
AquaticUdpRunner::new(14, 2), AquaticUdpRunner::new(14, 2, Priority::Low),
AquaticUdpRunner::new(16, 2), AquaticUdpRunner::new(16, 2, Priority::Low),
AquaticUdpRunner::new(28, 2), AquaticUdpRunner::new(28, 2, Priority::Low),
AquaticUdpRunner::new(13, 3), AquaticUdpRunner::new(13, 3, Priority::Low),
AquaticUdpRunner::new(16, 3), AquaticUdpRunner::new(16, 3, Priority::Low),
AquaticUdpRunner::new(26, 3), AquaticUdpRunner::new(26, 3, Priority::Low),
AquaticUdpRunner::new(12, 4), AquaticUdpRunner::new(12, 4, Priority::Medium),
AquaticUdpRunner::new(16, 4), AquaticUdpRunner::new(16, 4, Priority::Medium),
AquaticUdpRunner::new(24, 4), AquaticUdpRunner::new(24, 4, Priority::Low),
AquaticUdpRunner::new(11, 5), AquaticUdpRunner::new(11, 5, Priority::Low),
AquaticUdpRunner::new(16, 5), AquaticUdpRunner::new(16, 5, Priority::Medium),
AquaticUdpRunner::new(22, 5), AquaticUdpRunner::new(22, 5, Priority::Low),
AquaticUdpRunner::new(10, 6), AquaticUdpRunner::new(10, 6, Priority::Low),
AquaticUdpRunner::new(16, 6), AquaticUdpRunner::new(16, 6, Priority::High),
AquaticUdpRunner::new(20, 6), AquaticUdpRunner::new(20, 6, Priority::Medium),
AquaticUdpRunner::new(9, 7), AquaticUdpRunner::new(9, 7, Priority::Low),
AquaticUdpRunner::new(16, 7), AquaticUdpRunner::new(16, 7, Priority::Medium),
AquaticUdpRunner::new(18, 7), AquaticUdpRunner::new(18, 7, Priority::Low),
], ],
UdpTracker::OpenTracker => vec![ UdpTracker::OpenTracker => vec![
OpenTrackerUdpRunner::new(16), OpenTrackerUdpRunner::new(16, Priority::High),
OpenTrackerUdpRunner::new(32), OpenTrackerUdpRunner::new(32, Priority::Medium),
], ],
}, },
load_test_runs: simple_load_test_runs(cpu_mode, &[8, 12, 16, 24]), load_test_runs: simple_load_test_runs(cpu_mode, &[
(8, Priority::High),
(12, Priority::High),
(16, Priority::High),
(24, Priority::High),
]),
}, },
} }
} }
@ -231,16 +270,19 @@ impl UdpCommand {
struct AquaticUdpRunner { struct AquaticUdpRunner {
socket_workers: usize, socket_workers: usize,
swarm_workers: usize, swarm_workers: usize,
priority: Priority,
} }
impl AquaticUdpRunner { impl AquaticUdpRunner {
fn new( fn new(
socket_workers: usize, socket_workers: usize,
swarm_workers: usize, swarm_workers: usize,
priority: Priority,
) -> Rc<dyn ProcessRunner<Command = UdpCommand>> { ) -> Rc<dyn ProcessRunner<Command = UdpCommand>> {
Rc::new(Self { Rc::new(Self {
socket_workers, socket_workers,
swarm_workers, swarm_workers,
priority,
}) })
} }
} }
@ -275,6 +317,10 @@ impl ProcessRunner for AquaticUdpRunner {
.spawn()?) .spawn()?)
} }
fn priority(&self) -> crate::common::Priority {
self.priority
}
fn keys(&self) -> IndexMap<String, String> { fn keys(&self) -> IndexMap<String, String> {
indexmap! { indexmap! {
"socket workers".to_string() => self.socket_workers.to_string(), "socket workers".to_string() => self.socket_workers.to_string(),
@ -286,11 +332,12 @@ impl ProcessRunner for AquaticUdpRunner {
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
struct OpenTrackerUdpRunner { struct OpenTrackerUdpRunner {
workers: usize, workers: usize,
priority: Priority,
} }
impl OpenTrackerUdpRunner { impl OpenTrackerUdpRunner {
fn new(workers: usize) -> Rc<dyn ProcessRunner<Command = UdpCommand>> { fn new(workers: usize, priority: Priority) -> Rc<dyn ProcessRunner<Command = UdpCommand>> {
Rc::new(Self { workers }) Rc::new(Self { workers, priority })
} }
} }
@ -320,6 +367,10 @@ impl ProcessRunner for OpenTrackerUdpRunner {
.spawn()?) .spawn()?)
} }
fn priority(&self) -> crate::common::Priority {
self.priority
}
fn keys(&self) -> IndexMap<String, String> { fn keys(&self) -> IndexMap<String, String> {
indexmap! { indexmap! {
"workers".to_string() => self.workers.to_string(), "workers".to_string() => self.workers.to_string(),
@ -332,7 +383,7 @@ struct ChihayaUdpRunner;
impl ChihayaUdpRunner { impl ChihayaUdpRunner {
fn new() -> Rc<dyn ProcessRunner<Command = UdpCommand>> { fn new() -> Rc<dyn ProcessRunner<Command = UdpCommand>> {
Rc::new(Self) Rc::new(Self {})
} }
} }
@ -372,6 +423,10 @@ impl ProcessRunner for ChihayaUdpRunner {
.spawn()?) .spawn()?)
} }
fn priority(&self) -> crate::common::Priority {
Priority::High
}
fn keys(&self) -> IndexMap<String, String> { fn keys(&self) -> IndexMap<String, String> {
Default::default() Default::default()
} }
@ -415,6 +470,12 @@ impl ProcessRunner for AquaticUdpLoadTestRunner {
.spawn()?) .spawn()?)
} }
fn priority(&self) -> crate::common::Priority {
eprintln!("load test runner priority method called");
Priority::High
}
fn keys(&self) -> IndexMap<String, String> { fn keys(&self) -> IndexMap<String, String> {
indexmap! { indexmap! {
"workers".to_string() => self.workers.to_string(), "workers".to_string() => self.workers.to_string(),

4
crates/bencher/src/run.rs
View file

@ -12,7 +12,7 @@ use once_cell::sync::Lazy;
use regex::Regex; use regex::Regex;
use tempfile::NamedTempFile; use tempfile::NamedTempFile;
use crate::common::TaskSetCpuList; use crate::common::{Priority, TaskSetCpuList};
pub trait ProcessRunner: ::std::fmt::Debug { pub trait ProcessRunner: ::std::fmt::Debug {
type Command; type Command;
@ -26,6 +26,8 @@ pub trait ProcessRunner: ::std::fmt::Debug {
fn keys(&self) -> IndexMap<String, String>; fn keys(&self) -> IndexMap<String, String>;
fn priority(&self) -> Priority;
fn info(&self) -> String { fn info(&self) -> String {
self.keys() self.keys()
.into_iter() .into_iter()

28
crates/bencher/src/set.rs
View file

@ -5,7 +5,7 @@ use indexmap::IndexMap;
use num_format::{Locale, ToFormattedString}; use num_format::{Locale, ToFormattedString};
use crate::{ use crate::{
common::{CpuDirection, CpuMode, TaskSetCpuList}, common::{CpuDirection, CpuMode, Priority, TaskSetCpuList},
html::{html_all_runs, html_best_results}, html::{html_all_runs, html_best_results},
run::{ProcessRunner, ProcessStats, RunConfig}, run::{ProcessRunner, ProcessStats, RunConfig},
}; };
@ -16,19 +16,39 @@ pub trait Tracker: ::std::fmt::Debug + Copy + Clone + ::std::hash::Hash {
pub struct SetConfig<C, I> { pub struct SetConfig<C, I> {
pub implementations: IndexMap<I, Vec<Rc<dyn ProcessRunner<Command = C>>>>, pub implementations: IndexMap<I, Vec<Rc<dyn ProcessRunner<Command = C>>>>,
pub load_test_runs: Vec<(usize, TaskSetCpuList)>, pub load_test_runs: Vec<(usize, Priority, TaskSetCpuList)>,
} }
pub fn run_sets<C, F, I>( pub fn run_sets<C, F, I>(
command: &C, command: &C,
cpu_mode: CpuMode, cpu_mode: CpuMode,
set_configs: IndexMap<usize, SetConfig<C, I>>, min_cores: Option<usize>,
max_cores: Option<usize>,
min_priority: Priority,
mut set_configs: IndexMap<usize, SetConfig<C, I>>,
load_test_gen: F, load_test_gen: F,
) where ) where
C: ::std::fmt::Debug, C: ::std::fmt::Debug,
I: Tracker, I: Tracker,
F: Fn(usize) -> Box<dyn ProcessRunner<Command = C>>, F: Fn(usize) -> Box<dyn ProcessRunner<Command = C>>,
{ {
if let Some(min_cores) = min_cores {
set_configs.retain(|cores, _| *cores >= min_cores);
}
if let Some(max_cores) = max_cores {
set_configs.retain(|cores, _| *cores <= max_cores);
}
for set_config in set_configs.values_mut() {
for runners in set_config.implementations.values_mut() {
runners.retain(|r| r.priority() >= min_priority);
}
set_config
.load_test_runs
.retain(|(_, priority, _)| *priority >= min_priority);
}
println!("# Benchmark report"); println!("# Benchmark report");
let total_num_runs = set_configs let total_num_runs = set_configs
@ -75,7 +95,7 @@ pub fn run_sets<C, F, I>(
.load_test_runs .load_test_runs
.clone() .clone()
.into_iter() .into_iter()
.map(|(workers, load_test_vcpus)| { .map(|(workers, _, load_test_vcpus)| {
LoadTestRunResults::produce( LoadTestRunResults::produce(
command, command,
&load_test_gen, &load_test_gen,