YGGverse/aquatic
1
0
Fork 0
mirror of https://github.com/YGGverse/aquatic.git synced 2026-04-02 10:45:30 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Run rustfmt, clean up aquatic_http_protocol/Cargo.toml

Browse source
This commit is contained in:
Joakim Frostegård 2021-08-15 22:26:11 +02:00
parent 0cc312a78d
commit d0e716f80b
65 changed files with 1754 additions and 2590 deletions
Download patch file Download diff file Expand all files Collapse all files

12
aquatic_http_load_test/src/common.rs
View file

@ -1,11 +1,10 @@
use std::sync::{Arc, atomic::AtomicUsize};
use std::sync::{atomic::AtomicUsize, Arc};
use rand_distr::Pareto;
pub use aquatic_http_protocol::common::*;
pub use aquatic_http_protocol::response::*;
pub use aquatic_http_protocol::request::*;
pub use aquatic_http_protocol::response::*;
#[derive(PartialEq, Eq, Clone)]
pub struct TorrentPeer {
@ -15,7 +14,6 @@ pub struct TorrentPeer {
pub port: u16,
}
#[derive(Default)]
pub struct Statistics {
pub requests: AtomicUsize,
@ -27,7 +25,6 @@ pub struct Statistics {
pub bytes_received: AtomicUsize,
}
#[derive(Clone)]
pub struct LoadTestState {
pub info_hashes: Arc<Vec<InfoHash>>,
@ -35,9 +32,8 @@ pub struct LoadTestState {
pub pareto: Arc<Pareto<f64>>,
}
#[derive(PartialEq, Eq, Clone, Copy)]
pub enum RequestType {
Announce,
Scrape
}
Scrape,
}

10
aquatic_http_load_test/src/config.rs
View file

@ -1,7 +1,6 @@
use std::net::SocketAddr;
use serde::{Serialize, Deserialize};
use serde::{Deserialize, Serialize};
#[derive(Clone, Debug, Serialize, Deserialize)]
#[serde(default)]
@ -14,10 +13,8 @@ pub struct Config {
pub torrents: TorrentConfig,
}
impl aquatic_cli_helpers::Config for Config {}
#[derive(Clone, Debug, Serialize, Deserialize)]
#[serde(default)]
pub struct NetworkConfig {
@ -26,13 +23,12 @@ pub struct NetworkConfig {
pub poll_event_capacity: usize,
}
#[derive(Clone, Debug, Serialize, Deserialize)]
#[serde(default)]
pub struct TorrentConfig {
pub number_of_torrents: usize,
/// Pareto shape
///
///
/// Fake peers choose torrents according to Pareto distribution.
pub torrent_selection_pareto_shape: f64,
/// Probability that a generated peer is a seeder
@ -45,7 +41,6 @@ pub struct TorrentConfig {
pub weight_scrape: usize,
}
impl Default for Config {
fn default() -> Self {
Self {
@ -69,7 +64,6 @@ impl Default for NetworkConfig {
}
}
impl Default for TorrentConfig {
fn default() -> Self {
Self {

92
aquatic_http_load_test/src/main.rs
View file

@ -1,5 +1,5 @@
use std::sync::{atomic::Ordering, Arc};
use std::thread;
use std::sync::{Arc, atomic::Ordering};
use std::time::{Duration, Instant};
use rand::prelude::*;
@ -14,31 +14,27 @@ use common::*;
use config::*;
use network::*;
#[global_allocator]
static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc;
/// Multiply bytes during a second with this to get Mbit/s
const MBITS_FACTOR: f64 = 1.0 / ((1024.0 * 1024.0) / 8.0);
pub fn main(){
pub fn main() {
aquatic_cli_helpers::run_app_with_cli_and_config::<Config>(
"aquatic_http_load_test: BitTorrent load tester",
run,
None
None,
)
}
fn run(config: Config) -> ::anyhow::Result<()> {
if config.torrents.weight_announce + config.torrents.weight_scrape == 0 {
panic!("Error: at least one weight must be larger than zero.");
}
println!("Starting client with config: {:#?}", config);
let mut info_hashes = Vec::with_capacity(config.torrents.number_of_torrents);
let mut rng = SmallRng::from_entropy();
@ -47,10 +43,7 @@ fn run(config: Config) -> ::anyhow::Result<()> {
info_hashes.push(InfoHash(rng.gen()));
}
let pareto = Pareto::new(
1.0,
config.torrents.torrent_selection_pareto_shape
).unwrap();
let pareto = Pareto::new(1.0, config.torrents.torrent_selection_pareto_shape).unwrap();
let state = LoadTestState {
info_hashes: Arc::new(info_hashes),
@ -61,30 +54,18 @@ fn run(config: Config) -> ::anyhow::Result<()> {
// Start socket workers
for _ in 0..config.num_workers {
let config = config.clone();
let state = state.clone();
thread::spawn(move || run_socket_thread(
&config,
state,
1
));
thread::spawn(move || run_socket_thread(&config, state, 1));
}
monitor_statistics(
state,
&config
);
monitor_statistics(state, &config);
Ok(())
}
fn monitor_statistics(
state: LoadTestState,
config: &Config,
){
fn monitor_statistics(state: LoadTestState, config: &Config) {
let start_time = Instant::now();
let mut report_avg_response_vec: Vec<f64> = Vec::new();
@ -96,42 +77,53 @@ fn monitor_statistics(
let statistics = state.statistics.as_ref();
let responses_announce = statistics.responses_announce
.fetch_and(0, Ordering::SeqCst) as f64;
let responses_announce =
statistics.responses_announce.fetch_and(0, Ordering::SeqCst) as f64;
// let response_peers = statistics.response_peers
// .fetch_and(0, Ordering::SeqCst) as f64;
let requests_per_second = statistics.requests
.fetch_and(0, Ordering::SeqCst) as f64 / interval_f64;
let responses_scrape_per_second = statistics.responses_scrape
.fetch_and(0, Ordering::SeqCst) as f64 / interval_f64;
let responses_failure_per_second = statistics.responses_failure
.fetch_and(0, Ordering::SeqCst) as f64 / interval_f64;
let requests_per_second =
statistics.requests.fetch_and(0, Ordering::SeqCst) as f64 / interval_f64;
let responses_scrape_per_second =
statistics.responses_scrape.fetch_and(0, Ordering::SeqCst) as f64 / interval_f64;
let responses_failure_per_second =
statistics.responses_failure.fetch_and(0, Ordering::SeqCst) as f64 / interval_f64;
let bytes_sent_per_second = statistics.bytes_sent
.fetch_and(0, Ordering::SeqCst) as f64 / interval_f64;
let bytes_received_per_second = statistics.bytes_received
.fetch_and(0, Ordering::SeqCst) as f64 / interval_f64;
let bytes_sent_per_second =
statistics.bytes_sent.fetch_and(0, Ordering::SeqCst) as f64 / interval_f64;
let bytes_received_per_second =
statistics.bytes_received.fetch_and(0, Ordering::SeqCst) as f64 / interval_f64;
let responses_announce_per_second = responses_announce / interval_f64;
let responses_announce_per_second = responses_announce / interval_f64;
let responses_per_second =
responses_announce_per_second +
responses_scrape_per_second +
responses_failure_per_second;
let responses_per_second = responses_announce_per_second
+ responses_scrape_per_second
+ responses_failure_per_second;
report_avg_response_vec.push(responses_per_second);
println!();
println!("Requests out: {:.2}/second", requests_per_second);
println!("Responses in: {:.2}/second", responses_per_second);
println!(" - Announce responses: {:.2}", responses_announce_per_second);
println!(
" - Announce responses: {:.2}",
responses_announce_per_second
);
println!(" - Scrape responses: {:.2}", responses_scrape_per_second);
println!(" - Failure responses: {:.2}", responses_failure_per_second);
println!(
" - Failure responses: {:.2}",
responses_failure_per_second
);
//println!("Peers per announce response: {:.2}", response_peers / responses_announce);
println!("Bandwidth out: {:.2}Mbit/s", bytes_sent_per_second * MBITS_FACTOR);
println!("Bandwidth in: {:.2}Mbit/s", bytes_received_per_second * MBITS_FACTOR);
println!(
"Bandwidth out: {:.2}Mbit/s",
bytes_sent_per_second * MBITS_FACTOR
);
println!(
"Bandwidth in: {:.2}Mbit/s",
bytes_received_per_second * MBITS_FACTOR
);
let time_elapsed = start_time.elapsed();
let duration = Duration::from_secs(config.duration as u64);
@ -151,7 +143,7 @@ fn monitor_statistics(
config
);
break
break;
}
}
}

134
aquatic_http_load_test/src/network.rs
View file

@ -1,16 +1,15 @@
use std::io::{Cursor, ErrorKind, Read, Write};
use std::sync::atomic::Ordering;
use std::time::Duration;
use std::io::{Read, Write, ErrorKind, Cursor};
use hashbrown::HashMap;
use mio::{net::TcpStream, Events, Poll, Interest, Token};
use rand::{rngs::SmallRng, prelude::*};
use mio::{net::TcpStream, Events, Interest, Poll, Token};
use rand::{prelude::*, rngs::SmallRng};
use crate::common::*;
use crate::config::*;
use crate::utils::create_random_request;
pub struct Connection {
stream: TcpStream,
read_buffer: [u8; 4096],
@ -18,7 +17,6 @@ pub struct Connection {
can_send: bool,
}
impl Connection {
pub fn create_and_register(
config: &Config,
@ -31,29 +29,27 @@ impl Connection {
poll.registry()
.register(&mut stream, Token(*token_counter), Interest::READABLE)
.unwrap();
let connection = Connection {
let connection = Connection {
stream,
read_buffer: [0; 4096],
bytes_read: 0,
can_send: true,
};
connections.insert(*token_counter, connection);
*token_counter = token_counter.wrapping_add(1);
Ok(())
}
pub fn read_response(
&mut self,
state: &LoadTestState,
) -> bool { // bool = remove connection
pub fn read_response(&mut self, state: &LoadTestState) -> bool {
// bool = remove connection
loop {
match self.stream.read(&mut self.read_buffer[self.bytes_read..]){
match self.stream.read(&mut self.read_buffer[self.bytes_read..]) {
Ok(0) => {
if self.bytes_read == self.read_buffer.len(){
if self.bytes_read == self.read_buffer.len() {
eprintln!("read buffer is full");
}
@ -66,7 +62,7 @@ impl Connection {
let mut opt_body_start_index = None;
for (i, chunk) in interesting_bytes.windows(4).enumerate(){
for (i, chunk) in interesting_bytes.windows(4).enumerate() {
if chunk == b"\r\n\r\n" {
opt_body_start_index = Some(i + 4);
@ -77,30 +73,41 @@ impl Connection {
if let Some(body_start_index) = opt_body_start_index {
let interesting_bytes = &interesting_bytes[body_start_index..];
match Response::from_bytes(interesting_bytes){
match Response::from_bytes(interesting_bytes) {
Ok(response) => {
state.statistics.bytes_received
state
.statistics
.bytes_received
.fetch_add(self.bytes_read, Ordering::SeqCst);
match response {
Response::Announce(_) => {
state.statistics.responses_announce
state
.statistics
.responses_announce
.fetch_add(1, Ordering::SeqCst);
},
}
Response::Scrape(_) => {
state.statistics.responses_scrape
state
.statistics
.responses_scrape
.fetch_add(1, Ordering::SeqCst);
},
}
Response::Failure(response) => {
state.statistics.responses_failure
state
.statistics
.responses_failure
.fetch_add(1, Ordering::SeqCst);
println!("failure response: reason: {}", response.failure_reason);
},
println!(
"failure response: reason: {}",
response.failure_reason
);
}
}
self.bytes_read = 0;
self.can_send = true;
},
}
Err(err) => {
eprintln!(
"deserialize response error with {} bytes read: {:?}, text: {}",
@ -111,10 +118,10 @@ impl Connection {
}
}
}
},
}
Err(err) if err.kind() == ErrorKind::WouldBlock => {
break false;
},
}
Err(_) => {
self.bytes_read = 0;
@ -130,43 +137,40 @@ impl Connection {
state: &LoadTestState,
rng: &mut impl Rng,
request_buffer: &mut Cursor<&mut [u8]>,
) -> bool { // bool = remove connection
) -> bool {
// bool = remove connection
if !self.can_send {
return false;
}
let request = create_random_request(
&config,
&state,
rng
);
let request = create_random_request(&config, &state, rng);
request_buffer.set_position(0);
request.write(request_buffer).unwrap();
let position = request_buffer.position() as usize;
match self.send_request_inner(state, &request_buffer.get_mut()[..position]){
match self.send_request_inner(state, &request_buffer.get_mut()[..position]) {
Ok(()) => {
state.statistics.requests.fetch_add(1, Ordering::SeqCst);
self.can_send = false;
false
},
Err(_) => {
true
}
Err(_) => true,
}
}
fn send_request_inner(
&mut self,
state: &LoadTestState,
request: &[u8]
request: &[u8],
) -> ::std::io::Result<()> {
let bytes_sent = self.stream.write(request)?;
state.statistics.bytes_sent
state
.statistics
.bytes_sent
.fetch_add(bytes_sent, Ordering::SeqCst);
self.stream.flush()?;
@ -179,15 +183,9 @@ impl Connection {
}
}
pub type ConnectionMap = HashMap<usize, Connection>;
pub fn run_socket_thread(
config: &Config,
state: LoadTestState,
num_initial_requests: usize,
) {
pub fn run_socket_thread(config: &Config, state: LoadTestState, num_initial_requests: usize) {
let timeout = Duration::from_micros(config.network.poll_timeout_microseconds);
let create_conn_interval = 2 ^ config.network.connection_creation_interval;
@ -201,12 +199,8 @@ pub fn run_socket_thread(
let mut token_counter = 0usize;
for _ in 0..num_initial_requests {
Connection::create_and_register(
config,
&mut connections,
&mut poll,
&mut token_counter,
).unwrap();
Connection::create_and_register(config, &mut connections, &mut poll, &mut token_counter)
.unwrap();
}
let mut iter_counter = 0usize;
@ -218,14 +212,14 @@ pub fn run_socket_thread(
poll.poll(&mut events, Some(timeout))
.expect("failed polling");
for event in events.iter(){
if event.is_readable(){
for event in events.iter() {
if event.is_readable() {
let token = event.token();
if let Some(connection) = connections.get_mut(&token.0){
if let Some(connection) = connections.get_mut(&token.0) {
// Note that this does not indicate successfully reading
// response
if connection.read_response(&state){
if connection.read_response(&state) {
remove_connection(&mut poll, &mut connections, token.0);
num_to_create += 1;
@ -236,13 +230,9 @@ pub fn run_socket_thread(
}
}
for (k, connection) in connections.iter_mut(){
let remove_connection = connection.send_request(
config,
&state,
&mut rng,
&mut request_buffer
);
for (k, connection) in connections.iter_mut() {
let remove_connection =
connection.send_request(config, &state, &mut rng, &mut request_buffer);
if remove_connection {
drop_connections.push(*k);
@ -269,7 +259,8 @@ pub fn run_socket_thread(
&mut connections,
&mut poll,
&mut token_counter,
).is_ok();
)
.is_ok();
if ok {
num_to_create -= 1;
@ -280,15 +271,10 @@ pub fn run_socket_thread(
}
}
fn remove_connection(
poll: &mut Poll,
connections: &mut ConnectionMap,
connection_id: usize,
){
if let Some(mut connection) = connections.remove(&connection_id){
if let Err(err) = connection.deregister(poll){
fn remove_connection(poll: &mut Poll, connections: &mut ConnectionMap, connection_id: usize) {
if let Some(mut connection) = connections.remove(&connection_id) {
if let Err(err) = connection.deregister(poll) {
eprintln!("couldn't deregister connection: {}", err);
}
}
}
}

57
aquatic_http_load_test/src/utils.rs
View file

@ -1,13 +1,12 @@
use std::sync::Arc;
use rand::distributions::WeightedIndex;
use rand_distr::Pareto;
use rand::prelude::*;
use rand_distr::Pareto;
use crate::common::*;
use crate::config::*;
pub fn create_random_request(
config: &Config,
state: &LoadTestState,
@ -15,38 +14,21 @@ pub fn create_random_request(
) -> Request {
let weights = [
config.torrents.weight_announce as u32,
config.torrents.weight_scrape as u32,
config.torrents.weight_scrape as u32,
];
let items = [
RequestType::Announce,
RequestType::Scrape,
];
let items = [RequestType::Announce, RequestType::Scrape];
let dist = WeightedIndex::new(&weights)
.expect("random request weighted index");
let dist = WeightedIndex::new(&weights).expect("random request weighted index");
match items[dist.sample(rng)] {
RequestType::Announce => create_announce_request(
config,
state,
rng,
),
RequestType::Scrape => create_scrape_request(
config,
state,
rng,
)
RequestType::Announce => create_announce_request(config, state, rng),
RequestType::Scrape => create_scrape_request(config, state, rng),
}
}
#[inline]
fn create_announce_request(
config: &Config,
state: &LoadTestState,
rng: &mut impl Rng,
) -> Request {
fn create_announce_request(config: &Config, state: &LoadTestState, rng: &mut impl Rng) -> Request {
let (event, bytes_left) = {
if rng.gen_bool(config.torrents.peer_seeder_probability) {
(AnnounceEvent::Completed, 0)
@ -65,17 +47,12 @@ fn create_announce_request(
key: None,
numwant: None,
compact: true,
port: rng.gen()
port: rng.gen(),
})
}
#[inline]
fn create_scrape_request(
config: &Config,
state: &LoadTestState,
rng: &mut impl Rng,
) -> Request {
fn create_scrape_request(config: &Config, state: &LoadTestState, rng: &mut impl Rng) -> Request {
let mut scrape_hashes = Vec::with_capacity(5);
for _ in 0..5 {
@ -89,25 +66,15 @@ fn create_scrape_request(
})
}
#[inline]
fn select_info_hash_index(
config: &Config,
state: &LoadTestState,
rng: &mut impl Rng,
) -> usize {
fn select_info_hash_index(config: &Config, state: &LoadTestState, rng: &mut impl Rng) -> usize {
pareto_usize(rng, &state.pareto, config.torrents.number_of_torrents - 1)
}
#[inline]
fn pareto_usize(
rng: &mut impl Rng,
pareto: &Arc<Pareto<f64>>,
max: usize,
) -> usize {
fn pareto_usize(rng: &mut impl Rng, pareto: &Arc<Pareto<f64>>, max: usize) -> usize {
let p: f64 = pareto.sample(rng);
let p = (p.min(101.0f64) - 1.0) / 100.0;
(p * max as f64) as usize
}
}