Run rustfmt, clean up aquatic_http_protocol/Cargo.toml

This commit is contained in:
Joakim Frostegård 2021-08-15 22:26:11 +02:00
parent 0cc312a78d
commit d0e716f80b
65 changed files with 1754 additions and 2590 deletions

View file

@ -1,11 +1,10 @@
#[global_allocator]
static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc;
fn main(){
fn main() {
aquatic_cli_helpers::run_app_with_cli_and_config::<aquatic_udp::config::Config>(
aquatic_udp::APP_NAME,
aquatic_udp::run,
None
None,
)
}
}

View file

@ -1,6 +1,6 @@
use std::net::{SocketAddr, IpAddr, Ipv4Addr, Ipv6Addr};
use std::sync::{Arc, atomic::AtomicUsize};
use std::hash::Hash;
use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr};
use std::sync::{atomic::AtomicUsize, Arc};
use hashbrown::HashMap;
use indexmap::IndexMap;
@ -9,56 +9,45 @@ use parking_lot::Mutex;
pub use aquatic_common::ValidUntil;
pub use aquatic_udp_protocol::*;
pub const MAX_PACKET_SIZE: usize = 4096;
pub trait Ip: Hash + PartialEq + Eq + Clone + Copy {
fn ip_addr(self) -> IpAddr;
}
impl Ip for Ipv4Addr {
fn ip_addr(self) -> IpAddr {
IpAddr::V4(self)
}
}
impl Ip for Ipv6Addr {
fn ip_addr(self) -> IpAddr {
IpAddr::V6(self)
}
}
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub struct ConnectionKey {
pub connection_id: ConnectionId,
pub socket_addr: SocketAddr
pub socket_addr: SocketAddr,
}
pub type ConnectionMap = HashMap<ConnectionKey, ValidUntil>;
#[derive(PartialEq, Eq, Hash, Clone, Copy, Debug)]
pub enum PeerStatus {
Seeding,
Leeching,
Stopped
Stopped,
}
impl PeerStatus {
/// Determine peer status from announce event and number of bytes left.
///
///
/// Likely, the last branch will be taken most of the time.
#[inline]
pub fn from_event_and_bytes_left(
event: AnnounceEvent,
bytes_left: NumberOfBytes
) -> Self {
pub fn from_event_and_bytes_left(event: AnnounceEvent, bytes_left: NumberOfBytes) -> Self {
if event == AnnounceEvent::Stopped {
Self::Stopped
} else if bytes_left.0 == 0 {
@ -69,45 +58,39 @@ impl PeerStatus {
}
}
#[derive(Clone, Debug)]
pub struct Peer<I: Ip> {
pub ip_address: I,
pub port: Port,
pub status: PeerStatus,
pub valid_until: ValidUntil
pub valid_until: ValidUntil,
}
impl <I: Ip>Peer<I> {
impl<I: Ip> Peer<I> {
#[inline(always)]
pub fn to_response_peer(&self) -> ResponsePeer {
ResponsePeer {
ip_address: self.ip_address.ip_addr(),
port: self.port
port: self.port,
}
}
}
#[derive(PartialEq, Eq, Hash, Clone, Copy)]
pub struct PeerMapKey<I: Ip> {
pub ip: I,
pub peer_id: PeerId
pub peer_id: PeerId,
}
pub type PeerMap<I> = IndexMap<PeerMapKey<I>, Peer<I>>;
pub struct TorrentData<I: Ip> {
pub peers: PeerMap<I>,
pub num_seeders: usize,
pub num_leechers: usize,
}
impl <I: Ip>Default for TorrentData<I> {
impl<I: Ip> Default for TorrentData<I> {
fn default() -> Self {
Self {
peers: IndexMap::new(),
@ -117,17 +100,14 @@ impl <I: Ip>Default for TorrentData<I> {
}
}
pub type TorrentMap<I> = HashMap<InfoHash, TorrentData<I>>;
#[derive(Default)]
pub struct TorrentMaps {
pub ipv4: TorrentMap<Ipv4Addr>,
pub ipv6: TorrentMap<Ipv6Addr>,
}
#[derive(Default)]
pub struct Statistics {
pub requests_received: AtomicUsize,
@ -137,7 +117,6 @@ pub struct Statistics {
pub bytes_sent: AtomicUsize,
}
#[derive(Clone)]
pub struct State {
pub connections: Arc<Mutex<ConnectionMap>>,
@ -145,7 +124,6 @@ pub struct State {
pub statistics: Arc<Statistics>,
}
impl Default for State {
fn default() -> Self {
Self {
@ -156,11 +134,10 @@ impl Default for State {
}
}
#[cfg(test)]
mod tests {
#[test]
fn test_peer_status_from_event_and_bytes_left(){
fn test_peer_status_from_event_and_bytes_left() {
use crate::common::*;
use PeerStatus::*;
@ -179,4 +156,4 @@ mod tests {
assert_eq!(Seeding, f(AnnounceEvent::None, NumberOfBytes(0)));
assert_eq!(Leeching, f(AnnounceEvent::None, NumberOfBytes(1)));
}
}
}

View file

@ -1,10 +1,9 @@
use std::net::SocketAddr;
use serde::{Serialize, Deserialize};
use serde::{Deserialize, Serialize};
use aquatic_cli_helpers::LogLevel;
#[derive(Clone, Debug, Serialize, Deserialize)]
#[serde(default)]
pub struct Config {
@ -24,21 +23,19 @@ pub struct Config {
pub privileges: PrivilegeConfig,
}
impl aquatic_cli_helpers::Config for Config {
fn get_log_level(&self) -> Option<LogLevel> {
Some(self.log_level)
}
}
#[derive(Clone, Debug, Serialize, Deserialize)]
#[serde(default)]
pub struct NetworkConfig {
/// Bind to this address
pub address: SocketAddr,
/// Size of socket recv buffer. Use 0 for OS default.
///
///
/// This setting can have a big impact on dropped packages. It might
/// require changing system defaults. Some examples of commands to set
/// recommended values for different operating systems:
@ -55,7 +52,6 @@ pub struct NetworkConfig {
pub poll_event_capacity: usize,
}
#[derive(Clone, Debug, Serialize, Deserialize)]
#[serde(default)]
pub struct ProtocolConfig {
@ -67,7 +63,6 @@ pub struct ProtocolConfig {
pub peer_announce_interval: i32,
}
#[derive(Clone, Debug, Serialize, Deserialize)]
#[serde(default)]
pub struct HandlerConfig {
@ -77,7 +72,6 @@ pub struct HandlerConfig {
pub channel_recv_timeout_microseconds: u64,
}
#[derive(Clone, Debug, Serialize, Deserialize)]
#[serde(default)]
pub struct StatisticsConfig {
@ -85,7 +79,6 @@ pub struct StatisticsConfig {
pub interval: u64,
}
#[derive(Clone, Debug, Serialize, Deserialize)]
#[serde(default)]
pub struct CleaningConfig {
@ -97,7 +90,6 @@ pub struct CleaningConfig {
pub max_connection_age: u64,
}
#[derive(Clone, Debug, Serialize, Deserialize)]
#[serde(default)]
pub struct PrivilegeConfig {
@ -109,7 +101,6 @@ pub struct PrivilegeConfig {
pub user: String,
}
impl Default for Config {
fn default() -> Self {
Self {
@ -126,7 +117,6 @@ impl Default for Config {
}
}
impl Default for NetworkConfig {
fn default() -> Self {
Self {
@ -137,7 +127,6 @@ impl Default for NetworkConfig {
}
}
impl Default for ProtocolConfig {
fn default() -> Self {
Self {
@ -148,7 +137,6 @@ impl Default for ProtocolConfig {
}
}
impl Default for HandlerConfig {
fn default() -> Self {
Self {
@ -158,16 +146,12 @@ impl Default for HandlerConfig {
}
}
impl Default for StatisticsConfig {
fn default() -> Self {
Self {
interval: 0,
}
Self { interval: 0 }
}
}
impl Default for CleaningConfig {
fn default() -> Self {
Self {
@ -178,7 +162,6 @@ impl Default for CleaningConfig {
}
}
impl Default for PrivilegeConfig {
fn default() -> Self {
Self {
@ -187,4 +170,4 @@ impl Default for PrivilegeConfig {
user: "nobody".to_string(),
}
}
}
}

View file

@ -1,10 +1,13 @@
use std::net::{SocketAddr, IpAddr};
use std::net::{IpAddr, SocketAddr};
use std::time::Duration;
use std::vec::Drain;
use crossbeam_channel::{Sender, Receiver};
use crossbeam_channel::{Receiver, Sender};
use parking_lot::MutexGuard;
use rand::{SeedableRng, Rng, rngs::{SmallRng, StdRng}};
use rand::{
rngs::{SmallRng, StdRng},
Rng, SeedableRng,
};
use aquatic_common::{convert_ipv4_mapped_ipv6, extract_response_peers};
use aquatic_udp_protocol::*;
@ -12,13 +15,12 @@ use aquatic_udp_protocol::*;
use crate::common::*;
use crate::config::Config;
pub fn run_request_worker(
state: State,
config: Config,
request_receiver: Receiver<(Request, SocketAddr)>,
response_sender: Sender<(Response, SocketAddr)>,
){
) {
let mut connect_requests: Vec<(ConnectRequest, SocketAddr)> = Vec::new();
let mut announce_requests: Vec<(AnnounceRequest, SocketAddr)> = Vec::new();
let mut scrape_requests: Vec<(ScrapeRequest, SocketAddr)> = Vec::new();
@ -28,9 +30,7 @@ pub fn run_request_worker(
let mut std_rng = StdRng::from_entropy();
let mut small_rng = SmallRng::from_rng(&mut std_rng).unwrap();
let timeout = Duration::from_micros(
config.handlers.channel_recv_timeout_microseconds
);
let timeout = Duration::from_micros(config.handlers.channel_recv_timeout_microseconds);
loop {
let mut opt_connections = None;
@ -42,48 +42,41 @@ pub fn run_request_worker(
// only if ConnectionMap mutex isn't locked.
for i in 0..config.handlers.max_requests_per_iter {
let (request, src): (Request, SocketAddr) = if i == 0 {
match request_receiver.recv(){
match request_receiver.recv() {
Ok(r) => r,
Err(_) => break, // Really shouldn't happen
}
} else {
match request_receiver.recv_timeout(timeout){
match request_receiver.recv_timeout(timeout) {
Ok(r) => r,
Err(_) => {
if let Some(guard) = state.connections.try_lock(){
if let Some(guard) = state.connections.try_lock() {
opt_connections = Some(guard);
break
break;
} else {
continue
continue;
}
},
}
}
};
match request {
Request::Connect(r) => {
connect_requests.push((r, src))
},
Request::Announce(r) => {
announce_requests.push((r, src))
},
Request::Scrape(r) => {
scrape_requests.push((r, src))
},
Request::Connect(r) => connect_requests.push((r, src)),
Request::Announce(r) => announce_requests.push((r, src)),
Request::Scrape(r) => scrape_requests.push((r, src)),
}
}
let mut connections: MutexGuard<ConnectionMap> = opt_connections.unwrap_or_else(||
state.connections.lock()
);
let mut connections: MutexGuard<ConnectionMap> =
opt_connections.unwrap_or_else(|| state.connections.lock());
handle_connect_requests(
&config,
&mut connections,
&mut std_rng,
connect_requests.drain(..),
&mut responses
&mut responses,
);
announce_requests.retain(|(request, src)| {
@ -91,15 +84,15 @@ pub fn run_request_worker(
connection_id: request.connection_id,
socket_addr: *src,
};
if connections.contains_key(&connection_key){
if connections.contains_key(&connection_key) {
true
} else {
let response = ErrorResponse {
transaction_id: request.transaction_id,
message: "Connection invalid or expired".to_string()
message: "Connection invalid or expired".to_string(),
};
responses.push((response.into(), *src));
false
@ -111,15 +104,15 @@ pub fn run_request_worker(
connection_id: request.connection_id,
socket_addr: *src,
};
if connections.contains_key(&connection_key){
if connections.contains_key(&connection_key) {
true
} else {
let response = ErrorResponse {
transaction_id: request.transaction_id,
message: "Connection invalid or expired".to_string()
message: "Connection invalid or expired".to_string(),
};
responses.push((response.into(), *src));
false
@ -128,32 +121,27 @@ pub fn run_request_worker(
::std::mem::drop(connections);
if !(announce_requests.is_empty() && scrape_requests.is_empty()){
let mut torrents= state.torrents.lock();
if !(announce_requests.is_empty() && scrape_requests.is_empty()) {
let mut torrents = state.torrents.lock();
handle_announce_requests(
&config,
&mut torrents,
&mut small_rng,
announce_requests.drain(..),
&mut responses
);
handle_scrape_requests(
&mut torrents,
scrape_requests.drain(..),
&mut responses
&mut responses,
);
handle_scrape_requests(&mut torrents, scrape_requests.drain(..), &mut responses);
}
for r in responses.drain(..){
if let Err(err) = response_sender.send(r){
for r in responses.drain(..) {
if let Err(err) = response_sender.send(r) {
::log::error!("error sending response to channel: {}", err);
}
}
}
}
#[inline]
pub fn handle_connect_requests(
config: &Config,
@ -161,7 +149,7 @@ pub fn handle_connect_requests(
rng: &mut StdRng,
requests: Drain<(ConnectRequest, SocketAddr)>,
responses: &mut Vec<(Response, SocketAddr)>,
){
) {
let valid_until = ValidUntil::new(config.cleaning.max_connection_age);
responses.extend(requests.map(|(request, src)| {
@ -174,18 +162,15 @@ pub fn handle_connect_requests(
connections.insert(key, valid_until);
let response = Response::Connect(
ConnectResponse {
connection_id,
transaction_id: request.transaction_id,
}
);
let response = Response::Connect(ConnectResponse {
connection_id,
transaction_id: request.transaction_id,
});
(response, src)
}));
}
#[inline]
pub fn handle_announce_requests(
config: &Config,
@ -193,40 +178,35 @@ pub fn handle_announce_requests(
rng: &mut SmallRng,
requests: Drain<(AnnounceRequest, SocketAddr)>,
responses: &mut Vec<(Response, SocketAddr)>,
){
) {
let peer_valid_until = ValidUntil::new(config.cleaning.max_peer_age);
responses.extend(requests.map(|(request, src)| {
let peer_ip = convert_ipv4_mapped_ipv6(src.ip());
let response = match peer_ip {
IpAddr::V4(ip) => {
handle_announce_request(
config,
rng,
&mut torrents.ipv4,
request,
ip,
peer_valid_until,
)
},
IpAddr::V6(ip) => {
handle_announce_request(
config,
rng,
&mut torrents.ipv6,
request,
ip,
peer_valid_until,
)
}
IpAddr::V4(ip) => handle_announce_request(
config,
rng,
&mut torrents.ipv4,
request,
ip,
peer_valid_until,
),
IpAddr::V6(ip) => handle_announce_request(
config,
rng,
&mut torrents.ipv6,
request,
ip,
peer_valid_until,
),
};
(response.into(), src)
}));
}
fn handle_announce_request<I: Ip>(
config: &Config,
rng: &mut SmallRng,
@ -240,10 +220,7 @@ fn handle_announce_request<I: Ip>(
peer_id: request.peer_id,
};
let peer_status = PeerStatus::from_event_and_bytes_left(
request.event,
request.bytes_left
);
let peer_status = PeerStatus::from_event_and_bytes_left(request.event, request.bytes_left);
let peer = Peer {
ip_address: peer_ip,
@ -252,47 +229,40 @@ fn handle_announce_request<I: Ip>(
valid_until: peer_valid_until,
};
let torrent_data = torrents
.entry(request.info_hash)
.or_default();
let torrent_data = torrents.entry(request.info_hash).or_default();
let opt_removed_peer = match peer_status {
PeerStatus::Leeching => {
torrent_data.num_leechers += 1;
torrent_data.peers.insert(peer_key, peer)
},
}
PeerStatus::Seeding => {
torrent_data.num_seeders += 1;
torrent_data.peers.insert(peer_key, peer)
},
PeerStatus::Stopped => {
torrent_data.peers.remove(&peer_key)
}
PeerStatus::Stopped => torrent_data.peers.remove(&peer_key),
};
match opt_removed_peer.map(|peer| peer.status){
match opt_removed_peer.map(|peer| peer.status) {
Some(PeerStatus::Leeching) => {
torrent_data.num_leechers -= 1;
},
}
Some(PeerStatus::Seeding) => {
torrent_data.num_seeders -= 1;
},
}
_ => {}
}
let max_num_peers_to_take = calc_max_num_peers_to_take(
config,
request.peers_wanted.0
);
let max_num_peers_to_take = calc_max_num_peers_to_take(config, request.peers_wanted.0);
let response_peers = extract_response_peers(
rng,
&torrent_data.peers,
max_num_peers_to_take,
peer_key,
Peer::to_response_peer
Peer::to_response_peer,
);
AnnounceResponse {
@ -300,29 +270,26 @@ fn handle_announce_request<I: Ip>(
announce_interval: AnnounceInterval(config.protocol.peer_announce_interval),
leechers: NumberOfPeers(torrent_data.num_leechers as i32),
seeders: NumberOfPeers(torrent_data.num_seeders as i32),
peers: response_peers
peers: response_peers,
}
}
#[inline]
pub fn handle_scrape_requests(
torrents: &mut MutexGuard<TorrentMaps>,
requests: Drain<(ScrapeRequest, SocketAddr)>,
responses: &mut Vec<(Response, SocketAddr)>,
){
) {
let empty_stats = create_torrent_scrape_statistics(0, 0);
responses.extend(requests.map(|(request, src)|{
let mut stats: Vec<TorrentScrapeStatistics> = Vec::with_capacity(
request.info_hashes.len()
);
responses.extend(requests.map(|(request, src)| {
let mut stats: Vec<TorrentScrapeStatistics> = Vec::with_capacity(request.info_hashes.len());
let peer_ip = convert_ipv4_mapped_ipv6(src.ip());
if peer_ip.is_ipv4(){
if peer_ip.is_ipv4() {
for info_hash in request.info_hashes.iter() {
if let Some(torrent_data) = torrents.ipv4.get(info_hash){
if let Some(torrent_data) = torrents.ipv4.get(info_hash) {
stats.push(create_torrent_scrape_statistics(
torrent_data.num_seeders as i32,
torrent_data.num_leechers as i32,
@ -333,7 +300,7 @@ pub fn handle_scrape_requests(
}
} else {
for info_hash in request.info_hashes.iter() {
if let Some(torrent_data) = torrents.ipv6.get(info_hash){
if let Some(torrent_data) = torrents.ipv6.get(info_hash) {
stats.push(create_torrent_scrape_statistics(
torrent_data.num_seeders as i32,
torrent_data.num_leechers as i32,
@ -353,44 +320,35 @@ pub fn handle_scrape_requests(
}));
}
#[inline]
fn calc_max_num_peers_to_take(
config: &Config,
peers_wanted: i32,
) -> usize {
fn calc_max_num_peers_to_take(config: &Config, peers_wanted: i32) -> usize {
if peers_wanted <= 0 {
config.protocol.max_response_peers as usize
} else {
::std::cmp::min(
config.protocol.max_response_peers as usize,
peers_wanted as usize
peers_wanted as usize,
)
}
}
#[inline(always)]
pub fn create_torrent_scrape_statistics(
seeders: i32,
leechers: i32
) -> TorrentScrapeStatistics {
pub fn create_torrent_scrape_statistics(seeders: i32, leechers: i32) -> TorrentScrapeStatistics {
TorrentScrapeStatistics {
seeders: NumberOfPeers(seeders),
completed: NumberOfDownloads(0), // No implementation planned
leechers: NumberOfPeers(leechers)
leechers: NumberOfPeers(leechers),
}
}
#[cfg(test)]
mod tests {
use std::net::Ipv4Addr;
use std::collections::HashSet;
use std::net::Ipv4Addr;
use indexmap::IndexMap;
use quickcheck::{quickcheck, TestResult};
use rand::thread_rng;
use quickcheck::{TestResult, quickcheck};
use super::*;
@ -399,7 +357,7 @@ mod tests {
let peer_id = PeerId([0; 20]);
let key = PeerMapKey {
ip: ip_address,
ip: ip_address,
peer_id,
};
let value = Peer {
@ -413,14 +371,12 @@ mod tests {
}
#[test]
fn test_extract_response_peers(){
fn test_extract_response_peers() {
fn prop(data: (u16, u16)) -> TestResult {
let gen_num_peers = data.0 as u32;
let req_num_peers = data.1 as usize;
let mut peer_map: PeerMap<Ipv4Addr> = IndexMap::with_capacity(
gen_num_peers as usize
);
let mut peer_map: PeerMap<Ipv4Addr> = IndexMap::with_capacity(gen_num_peers as usize);
let mut opt_sender_key = None;
let mut opt_sender_peer = None;
@ -443,7 +399,7 @@ mod tests {
&peer_map,
req_num_peers,
opt_sender_key.unwrap_or_else(|| gen_peer_map_key_and_value(1).0),
Peer::to_response_peer
Peer::to_response_peer,
);
// Check that number of returned peers is correct
@ -451,8 +407,8 @@ mod tests {
let mut success = peers.len() <= req_num_peers;
if req_num_peers >= gen_num_peers as usize {
success &= peers.len() == gen_num_peers as usize ||
peers.len() + 1 == gen_num_peers as usize;
success &= peers.len() == gen_num_peers as usize
|| peers.len() + 1 == gen_num_peers as usize;
}
// Check that returned peers are unique (no overlap) and that sender
@ -461,7 +417,9 @@ mod tests {
let mut ip_addresses = HashSet::with_capacity(peers.len());
for peer in peers {
if peer == opt_sender_peer.clone().unwrap() || ip_addresses.contains(&peer.ip_address){
if peer == opt_sender_peer.clone().unwrap()
|| ip_addresses.contains(&peer.ip_address)
{
success = false;
break;
@ -471,8 +429,8 @@ mod tests {
}
TestResult::from_bool(success)
}
}
quickcheck(prop as fn((u16, u16)) -> TestResult);
}
}
}

View file

@ -1,6 +1,9 @@
use std::sync::{Arc, atomic::{AtomicUsize, Ordering}};
use std::time::Duration;
use std::sync::{
atomic::{AtomicUsize, Ordering},
Arc,
};
use std::thread::Builder;
use std::time::Duration;
use anyhow::Context;
use crossbeam_channel::unbounded;
@ -12,13 +15,11 @@ pub mod handlers;
pub mod network;
pub mod tasks;
use config::Config;
use common::State;
use config::Config;
pub const APP_NAME: &str = "aquatic_udp: UDP BitTorrent tracker";
pub fn run(config: Config) -> ::anyhow::Result<()> {
let state = State::default();
@ -48,11 +49,7 @@ pub fn run(config: Config) -> ::anyhow::Result<()> {
}
}
pub fn start_workers(
config: Config,
state: State
) -> ::anyhow::Result<Arc<AtomicUsize>> {
pub fn start_workers(config: Config, state: State) -> ::anyhow::Result<Arc<AtomicUsize>> {
let (request_sender, request_receiver) = unbounded();
let (response_sender, response_receiver) = unbounded();
@ -62,14 +59,12 @@ pub fn start_workers(
let request_receiver = request_receiver.clone();
let response_sender = response_sender.clone();
Builder::new().name(format!("request-{:02}", i + 1)).spawn(move ||
handlers::run_request_worker(
state,
config,
request_receiver,
response_sender
)
).with_context(|| "spawn request worker")?;
Builder::new()
.name(format!("request-{:02}", i + 1))
.spawn(move || {
handlers::run_request_worker(state, config, request_receiver, response_sender)
})
.with_context(|| "spawn request worker")?;
}
let num_bound_sockets = Arc::new(AtomicUsize::new(0));
@ -81,31 +76,33 @@ pub fn start_workers(
let response_receiver = response_receiver.clone();
let num_bound_sockets = num_bound_sockets.clone();
Builder::new().name(format!("socket-{:02}", i + 1)).spawn(move ||
network::run_socket_worker(
state,
config,
i,
request_sender,
response_receiver,
num_bound_sockets,
)
).with_context(|| "spawn socket worker")?;
Builder::new()
.name(format!("socket-{:02}", i + 1))
.spawn(move || {
network::run_socket_worker(
state,
config,
i,
request_sender,
response_receiver,
num_bound_sockets,
)
})
.with_context(|| "spawn socket worker")?;
}
if config.statistics.interval != 0 {
let state = state.clone();
let config = config.clone();
Builder::new().name("statistics-collector".to_string()).spawn(move ||
loop {
::std::thread::sleep(Duration::from_secs(
config.statistics.interval
));
Builder::new()
.name("statistics-collector".to_string())
.spawn(move || loop {
::std::thread::sleep(Duration::from_secs(config.statistics.interval));
tasks::gather_and_print_statistics(&state, &config);
}
).with_context(|| "spawn statistics worker")?;
})
.with_context(|| "spawn statistics worker")?;
}
Ok(num_bound_sockets)

View file

@ -1,20 +1,22 @@
use std::sync::{Arc, atomic::{AtomicUsize, Ordering}};
use std::io::{Cursor, ErrorKind};
use std::net::{SocketAddr, IpAddr};
use std::net::{IpAddr, SocketAddr};
use std::sync::{
atomic::{AtomicUsize, Ordering},
Arc,
};
use std::time::Duration;
use std::vec::Drain;
use crossbeam_channel::{Sender, Receiver};
use mio::{Events, Poll, Interest, Token};
use crossbeam_channel::{Receiver, Sender};
use mio::net::UdpSocket;
use socket2::{Socket, Domain, Type, Protocol};
use mio::{Events, Interest, Poll, Token};
use socket2::{Domain, Protocol, Socket, Type};
use aquatic_udp_protocol::{Request, Response, IpVersion};
use aquatic_udp_protocol::{IpVersion, Request, Response};
use crate::common::*;
use crate::config::Config;
pub fn run_socket_worker(
state: State,
config: Config,
@ -22,7 +24,7 @@ pub fn run_socket_worker(
request_sender: Sender<(Request, SocketAddr)>,
response_receiver: Receiver<(Response, SocketAddr)>,
num_bound_sockets: Arc<AtomicUsize>,
){
) {
let mut buffer = [0u8; MAX_PACKET_SIZE];
let mut socket = UdpSocket::from_std(create_socket(&config));
@ -33,7 +35,7 @@ pub fn run_socket_worker(
poll.registry()
.register(&mut socket, Token(token_num), interests)
.unwrap();
num_bound_sockets.fetch_add(1, Ordering::SeqCst);
let mut events = Events::with_capacity(config.network.poll_event_capacity);
@ -47,10 +49,10 @@ pub fn run_socket_worker(
poll.poll(&mut events, Some(timeout))
.expect("failed polling");
for event in events.iter(){
for event in events.iter() {
let token = event.token();
if (token.0 == token_num) & event.is_readable(){
if (token.0 == token_num) & event.is_readable() {
read_requests(
&state,
&config,
@ -60,13 +62,16 @@ pub fn run_socket_worker(
&mut local_responses,
);
for r in requests.drain(..){
if let Err(err) = request_sender.send(r){
for r in requests.drain(..) {
if let Err(err) = request_sender.send(r) {
::log::error!("error sending to request_sender: {}", err);
}
}
state.statistics.readable_events.fetch_add(1, Ordering::SeqCst);
state
.statistics
.readable_events
.fetch_add(1, Ordering::SeqCst);
}
}
@ -76,33 +81,33 @@ pub fn run_socket_worker(
&mut socket,
&mut buffer,
&response_receiver,
local_responses.drain(..)
local_responses.drain(..),
);
}
}
fn create_socket(config: &Config) -> ::std::net::UdpSocket {
let socket = if config.network.address.is_ipv4(){
let socket = if config.network.address.is_ipv4() {
Socket::new(Domain::ipv4(), Type::dgram(), Some(Protocol::udp()))
} else {
Socket::new(Domain::ipv6(), Type::dgram(), Some(Protocol::udp()))
}.expect("create socket");
}
.expect("create socket");
socket.set_reuse_port(true)
.expect("socket: set reuse port");
socket.set_reuse_port(true).expect("socket: set reuse port");
socket.set_nonblocking(true)
socket
.set_nonblocking(true)
.expect("socket: set nonblocking");
socket.bind(&config.network.address.into()).unwrap_or_else(|err|
panic!("socket: bind to {}: {:?}", config.network.address, err)
);
socket
.bind(&config.network.address.into())
.unwrap_or_else(|err| panic!("socket: bind to {}: {:?}", config.network.address, err));
let recv_buffer_size = config.network.socket_recv_buffer_size;
if recv_buffer_size != 0 {
if let Err(err) = socket.set_recv_buffer_size(recv_buffer_size){
if let Err(err) = socket.set_recv_buffer_size(recv_buffer_size) {
::log::error!(
"socket: failed setting recv buffer to {}: {:?}",
recv_buffer_size,
@ -114,7 +119,6 @@ fn create_socket(config: &Config) -> ::std::net::UdpSocket {
socket.into_udp_socket()
}
#[inline]
fn read_requests(
state: &State,
@ -123,28 +127,26 @@ fn read_requests(
buffer: &mut [u8],
requests: &mut Vec<(Request, SocketAddr)>,
local_responses: &mut Vec<(Response, SocketAddr)>,
){
) {
let mut requests_received: usize = 0;
let mut bytes_received: usize = 0;
loop {
match socket.recv_from(&mut buffer[..]) {
Ok((amt, src)) => {
let request = Request::from_bytes(
&buffer[..amt],
config.protocol.max_scrape_torrents
);
let request =
Request::from_bytes(&buffer[..amt], config.protocol.max_scrape_torrents);
bytes_received += amt;
if request.is_ok(){
if request.is_ok() {
requests_received += 1;
}
match request {
Ok(request) => {
requests.push((request, src));
},
}
Err(err) => {
::log::debug!("request_from_bytes error: {:?}", err);
@ -166,9 +168,9 @@ fn read_requests(
local_responses.push((response.into(), src));
}
}
},
}
}
},
}
Err(err) => {
if err.kind() == ErrorKind::WouldBlock {
break;
@ -180,14 +182,17 @@ fn read_requests(
}
if config.statistics.interval != 0 {
state.statistics.requests_received
state
.statistics
.requests_received
.fetch_add(requests_received, Ordering::SeqCst);
state.statistics.bytes_received
state
.statistics
.bytes_received
.fetch_add(bytes_received, Ordering::SeqCst);
}
}
#[inline]
fn send_responses(
state: &State,
@ -196,15 +201,15 @@ fn send_responses(
buffer: &mut [u8],
response_receiver: &Receiver<(Response, SocketAddr)>,
local_responses: Drain<(Response, SocketAddr)>,
){
) {
let mut responses_sent: usize = 0;
let mut bytes_sent: usize = 0;
let mut cursor = Cursor::new(buffer);
let response_iterator = local_responses.into_iter().chain(
response_receiver.try_iter()
);
let response_iterator = local_responses
.into_iter()
.chain(response_receiver.try_iter());
for (response, src) in response_iterator {
cursor.set_position(0);
@ -215,11 +220,11 @@ fn send_responses(
let amt = cursor.position() as usize;
match socket.send_to(&cursor.get_ref()[..amt], src){
match socket.send_to(&cursor.get_ref()[..amt], src) {
Ok(amt) => {
responses_sent += 1;
bytes_sent += amt;
},
}
Err(err) => {
if err.kind() == ErrorKind::WouldBlock {
break;
@ -231,23 +236,26 @@ fn send_responses(
}
if config.statistics.interval != 0 {
state.statistics.responses_sent
state
.statistics
.responses_sent
.fetch_add(responses_sent, Ordering::SeqCst);
state.statistics.bytes_sent
state
.statistics
.bytes_sent
.fetch_add(bytes_sent, Ordering::SeqCst);
}
}
fn ip_version_from_ip(ip: IpAddr) -> IpVersion {
match ip {
IpAddr::V4(_) => IpVersion::IPv4,
IpAddr::V6(ip) => {
if let [0, 0, 0, 0, 0, 0xffff, ..] = ip.segments(){
if let [0, 0, 0, 0, 0, 0xffff, ..] = ip.segments() {
IpVersion::IPv4
} else {
IpVersion::IPv6
}
}
}
}
}

View file

@ -6,8 +6,7 @@ use histogram::Histogram;
use crate::common::*;
use crate::config::Config;
pub fn clean_connections_and_torrents(state: &State){
pub fn clean_connections_and_torrents(state: &State) {
let now = Instant::now();
{
@ -18,17 +17,13 @@ pub fn clean_connections_and_torrents(state: &State){
}
let mut torrents = state.torrents.lock();
clean_torrent_map(&mut torrents.ipv4, now);
clean_torrent_map(&mut torrents.ipv6, now);
}
#[inline]
fn clean_torrent_map<I: Ip>(
torrents: &mut TorrentMap<I>,
now: Instant,
){
fn clean_torrent_map<I: Ip>(torrents: &mut TorrentMap<I>, now: Instant) {
torrents.retain(|_, torrent| {
let num_seeders = &mut torrent.num_seeders;
let num_leechers = &mut torrent.num_leechers;
@ -40,10 +35,10 @@ fn clean_torrent_map<I: Ip>(
match peer.status {
PeerStatus::Seeding => {
*num_seeders -= 1;
},
}
PeerStatus::Leeching => {
*num_leechers -= 1;
},
}
_ => (),
};
}
@ -57,28 +52,31 @@ fn clean_torrent_map<I: Ip>(
torrents.shrink_to_fit();
}
pub fn gather_and_print_statistics(
state: &State,
config: &Config,
){
pub fn gather_and_print_statistics(state: &State, config: &Config) {
let interval = config.statistics.interval;
let requests_received: f64 = state.statistics.requests_received
let requests_received: f64 = state
.statistics
.requests_received
.fetch_and(0, Ordering::SeqCst) as f64;
let responses_sent: f64 = state.statistics.responses_sent
let responses_sent: f64 = state
.statistics
.responses_sent
.fetch_and(0, Ordering::SeqCst) as f64;
let bytes_received: f64 = state.statistics.bytes_received
.fetch_and(0, Ordering::SeqCst) as f64;
let bytes_sent: f64 = state.statistics.bytes_sent
let bytes_received: f64 = state
.statistics
.bytes_received
.fetch_and(0, Ordering::SeqCst) as f64;
let bytes_sent: f64 = state.statistics.bytes_sent.fetch_and(0, Ordering::SeqCst) as f64;
let requests_per_second = requests_received / interval as f64;
let responses_per_second: f64 = responses_sent / interval as f64;
let bytes_received_per_second: f64 = bytes_received / interval as f64;
let bytes_sent_per_second: f64 = bytes_sent / interval as f64;
let readable_events: f64 = state.statistics.readable_events
let readable_events: f64 = state
.statistics
.readable_events
.fetch_and(0, Ordering::SeqCst) as f64;
let requests_per_readable_event = if readable_events == 0.0 {
0.0
@ -88,9 +86,7 @@ pub fn gather_and_print_statistics(
println!(
"stats: {:.2} requests/second, {:.2} responses/second, {:.2} requests/readable event",
requests_per_second,
responses_per_second,
requests_per_readable_event
requests_per_second, responses_per_second, requests_per_readable_event
);
println!(
@ -104,17 +100,17 @@ pub fn gather_and_print_statistics(
{
let torrents = &mut state.torrents.lock();
for torrent in torrents.ipv4.values(){
for torrent in torrents.ipv4.values() {
let num_peers = (torrent.num_seeders + torrent.num_leechers) as u64;
if let Err(err) = peers_per_torrent.increment(num_peers){
if let Err(err) = peers_per_torrent.increment(num_peers) {
::log::error!("error incrementing peers_per_torrent histogram: {}", err)
}
}
for torrent in torrents.ipv6.values(){
for torrent in torrents.ipv6.values() {
let num_peers = (torrent.num_seeders + torrent.num_leechers) as u64;
if let Err(err) = peers_per_torrent.increment(num_peers){
if let Err(err) = peers_per_torrent.increment(num_peers) {
::log::error!("error incrementing peers_per_torrent histogram: {}", err)
}
}
@ -134,4 +130,4 @@ pub fn gather_and_print_statistics(
}
println!();
}
}