YGGverse/aquatic
1
0
Fork 0
mirror of https://github.com/YGGverse/aquatic.git synced 2026-03-31 17:55:36 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

udp: initial support for listing peer clients

Browse source
This commit is contained in:
Joakim Frostegård 2023-06-06 01:04:37 +02:00
parent 977349ec03
commit a74d6aa458
14 changed files with 213 additions and 68 deletions
Download patch file Download diff file Expand all files Collapse all files

4
Cargo.lock generated
View file

@ -184,6 +184,8 @@ name = "aquatic_peer_id"
version = "0.8.0"
dependencies = [
"compact_str",
"hex",
"quickcheck",
"regex",
"serde",
]
@ -218,6 +220,7 @@ dependencies = [
"aquatic_udp_protocol",
"blake3",
"cfg-if",
"compact_str",
"constant_time_eq",
"crossbeam-channel",
"getrandom",
@ -284,6 +287,7 @@ dependencies = [
name = "aquatic_udp_protocol"
version = "0.8.0"
dependencies = [
"aquatic_peer_id",
"byteorder",
"either",
"quickcheck",

2
aquatic_peer_id/Cargo.toml
View file

@ -14,5 +14,7 @@ name = "aquatic_peer_id"
[dependencies]
compact_str = "0.7"
hex = "0.4"
regex = "1"
serde = { version = "1", features = ["derive"] }
quickcheck = "1"

48
aquatic_peer_id/src/lib.rs
View file

@ -7,6 +7,20 @@ use serde::{Deserialize, Serialize};
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
pub struct PeerId(pub [u8; 20]);
impl PeerId {
pub fn client(&self) -> PeerClient {
PeerClient::from_peer_id(self)
}
pub fn first_8_bytes_hex(&self) -> CompactString {
let mut buf = [0u8; 16];
hex::encode_to_slice(&self.0[..8], &mut buf)
.expect("PeerId.first_8_bytes_hex buffer too small");
CompactString::from_utf8_lossy(&buf)
}
}
#[non_exhaustive]
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub enum PeerClient {
@ -112,7 +126,7 @@ impl PeerClient {
}
}
pub fn from_peer_id(peer_id: PeerId) -> Self {
pub fn from_peer_id(peer_id: &PeerId) -> Self {
static AZ_RE: OnceLock<Regex> = OnceLock::new();
if let Some(caps) = AZ_RE
@ -178,6 +192,18 @@ impl Display for PeerClient {
}
}
impl quickcheck::Arbitrary for PeerId {
fn arbitrary(g: &mut quickcheck::Gen) -> Self {
let mut bytes = [0u8; 20];
for byte in bytes.iter_mut() {
*byte = u8::arbitrary(g);
}
Self(bytes)
}
}
#[cfg(test)]
mod tests {
use super::*;
@ -195,43 +221,43 @@ mod tests {
#[test]
fn test_client_from_peer_id() {
assert_eq!(
PeerClient::from_peer_id(create_peer_id(b"-lt1234-k/asdh3")),
PeerClient::from_peer_id(&create_peer_id(b"-lt1234-k/asdh3")),
PeerClient::LibTorrentRakshasa("1.23.4".into())
);
assert_eq!(
PeerClient::from_peer_id(create_peer_id(b"-UT123A-k/asdh3")),
PeerClient::from_peer_id(&create_peer_id(b"-UT123A-k/asdh3")),
PeerClient::UTorrent("1.2.3 [Alpha]".into())
);
assert_eq!(
PeerClient::from_peer_id(create_peer_id(b"-TR0012-k/asdh3")),
PeerClient::from_peer_id(&create_peer_id(b"-TR0012-k/asdh3")),
PeerClient::Transmission("0.12".into())
);
assert_eq!(
PeerClient::from_peer_id(create_peer_id(b"-TR1212-k/asdh3")),
PeerClient::from_peer_id(&create_peer_id(b"-TR1212-k/asdh3")),
PeerClient::Transmission("1.21".into())
);
assert_eq!(
PeerClient::from_peer_id(create_peer_id(b"-WW0102-k/asdh3")),
PeerClient::from_peer_id(&create_peer_id(b"-WW0102-k/asdh3")),
PeerClient::WebTorrent("1.2".into())
);
assert_eq!(
PeerClient::from_peer_id(create_peer_id(b"-WW1302-k/asdh3")),
PeerClient::from_peer_id(&create_peer_id(b"-WW1302-k/asdh3")),
PeerClient::WebTorrent("13.2".into())
);
assert_eq!(
PeerClient::from_peer_id(create_peer_id(b"-WW1324-k/asdh3")),
PeerClient::from_peer_id(&create_peer_id(b"-WW1324-k/asdh3")),
PeerClient::WebTorrent("13.24".into())
);
assert_eq!(
PeerClient::from_peer_id(create_peer_id(b"M1-2-3--k/asdh3")),
PeerClient::from_peer_id(&create_peer_id(b"M1-2-3--k/asdh3")),
PeerClient::Mainline("1.2.3".into())
);
assert_eq!(
PeerClient::from_peer_id(create_peer_id(b"M1-23-4-k/asdh3")),
PeerClient::from_peer_id(&create_peer_id(b"M1-23-4-k/asdh3")),
PeerClient::Mainline("1.23.4".into())
);
assert_eq!(
PeerClient::from_peer_id(create_peer_id(b"S3-k/asdh3")),
PeerClient::from_peer_id(&create_peer_id(b"S3-k/asdh3")),
PeerClient::OtherWithPrefix("S3".into())
);
}

1
aquatic_udp/Cargo.toml
View file

@ -30,6 +30,7 @@ aquatic_udp_protocol.workspace = true
anyhow = "1"
blake3 = "1"
cfg-if = "1"
compact_str = { version = "0.7", features = ["serde"] }
constant_time_eq = "0.2"
crossbeam-channel = "0.5"
getrandom = "0.2"

2
aquatic_udp/src/common.rs
View file

@ -136,6 +136,8 @@ impl PeerStatus {
pub enum StatisticsMessage {
Ipv4PeerHistogram(Histogram<u64>),
Ipv6PeerHistogram(Histogram<u64>),
PeerAdded(PeerId),
PeerRemoved(PeerId),
}
pub struct Statistics {

5
aquatic_udp/src/config.rs
View file

@ -161,10 +161,9 @@ impl Default for ProtocolConfig {
pub struct StatisticsConfig {
/// Collect and print/write statistics this often (seconds)
pub interval: u64,
/// Enable extended statistics (on peers per torrent)
/// Enable extended statistics (on peers per torrent and on peer clients)
///
/// Will increase time taken for torrent cleaning, since that's when
/// these statistics are collected.
/// Will increase time taken for request handling and torrent cleaning
pub extended: bool,
/// Print statistics to standard output
pub print_to_stdout: bool,

1
aquatic_udp/src/workers/statistics/collector.rs
View file

@ -2,6 +2,7 @@ use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::Arc;
use std::time::Instant;
use crossbeam_channel::Receiver;
use hdrhistogram::Histogram;
use num_format::{Locale, ToFormattedString};
use serde::Serialize;

66
aquatic_udp/src/workers/statistics/mod.rs
View file

@ -2,10 +2,12 @@ mod collector;
use std::fs::File;
use std::io::Write;
use std::time::Duration;
use std::time::{Duration, Instant};
use anyhow::Context;
use aquatic_common::PanicSentinel;
use aquatic_common::{IndexMap, PanicSentinel};
use aquatic_udp_protocol::PeerClient;
use compact_str::{CompactString, ToCompactString};
use crossbeam_channel::Receiver;
use serde::Serialize;
use time::format_description::well_known::Rfc2822;
@ -35,6 +37,7 @@ struct TemplateData {
ipv6: CollectedStatistics,
last_updated: String,
peer_update_interval: String,
peer_clients: Vec<(CompactString, CompactString, usize)>,
}
pub fn run_statistics_worker(
@ -68,13 +71,46 @@ pub fn run_statistics_worker(
"6".into(),
);
let mut peer_clients: IndexMap<PeerClient, (usize, CompactString)> = IndexMap::default();
loop {
::std::thread::sleep(Duration::from_secs(config.statistics.interval));
let start_time = Instant::now();
for message in statistics_receiver.try_iter() {
match message {
StatisticsMessage::Ipv4PeerHistogram(h) => ipv4_collector.add_histogram(&config, h),
StatisticsMessage::Ipv6PeerHistogram(h) => ipv6_collector.add_histogram(&config, h),
StatisticsMessage::PeerAdded(peer_id) => {
peer_clients
.entry(peer_id.client())
.or_insert((0, peer_id.first_8_bytes_hex()))
.0 += 1;
}
StatisticsMessage::PeerRemoved(peer_id) => {
let client = peer_id.client();
if let Some((count, _)) = peer_clients.get_mut(&client) {
if *count == 1 {
drop(count);
peer_clients.remove(&client);
} else {
*count -= 1;
}
}
}
}
}
#[cfg(feature = "prometheus")]
if config.statistics.run_prometheus_endpoint && config.statistics.extended {
for (peer_client, (count, first_8_bytes)) in peer_clients.iter() {
::metrics::gauge!(
"aquatic_peer_clients",
*count as f64,
"client" => peer_client.to_string(),
"peer_id_prefix_hex" => first_8_bytes.to_string(),
);
}
}
@ -107,6 +143,23 @@ pub fn run_statistics_worker(
}
if let Some(tt) = opt_tt.as_ref() {
let mut peer_clients = if config.statistics.extended {
peer_clients
.iter()
.map(|(peer_client, (count, first_8_bytes))| {
(
peer_client.to_compact_string(),
first_8_bytes.to_owned(),
*count,
)
})
.collect()
} else {
Vec::new()
};
peer_clients.sort_unstable_by(|a, b| b.2.cmp(&a.2));
let template_data = TemplateData {
stylesheet: STYLESHEET_CONTENTS.to_string(),
ipv4_active: config.network.ipv4_active(),
@ -118,12 +171,19 @@ pub fn run_statistics_worker(
.format(&Rfc2822)
.unwrap_or("(formatting error)".into()),
peer_update_interval: format!("{}", config.cleaning.torrent_cleaning_interval),
peer_clients,
};
if let Err(err) = save_html_to_file(&config, tt, &template_data) {
::log::error!("Couldn't save statistics to file: {:#}", err)
}
}
if let Some(time_remaining) =
Duration::from_secs(config.statistics.interval).checked_sub(start_time.elapsed())
{
::std::thread::sleep(time_remaining);
}
}
}

26
aquatic_udp/src/workers/swarm/mod.rs
View file

@ -50,6 +50,7 @@ pub fn run_swarm_worker(
let response = handle_announce_request(
&config,
&mut rng,
&statistics_sender,
&mut torrents.ipv4,
request,
ip,
@ -62,6 +63,7 @@ pub fn run_swarm_worker(
let response = handle_announce_request(
&config,
&mut rng,
&statistics_sender,
&mut torrents.ipv6,
request,
ip,
@ -88,28 +90,15 @@ pub fn run_swarm_worker(
peer_valid_until = ValidUntil::new(server_start_instant, config.cleaning.max_peer_age);
if now > last_cleaning + cleaning_interval {
let (ipv4, ipv6) = torrents.clean_and_get_statistics(
torrents.clean_and_update_statistics(
&config,
&state,
&statistics_sender,
&state.access_list,
server_start_instant,
worker_index,
);
if config.statistics.active() {
state.statistics_ipv4.peers[worker_index.0].store(ipv4.0, Ordering::Release);
state.statistics_ipv6.peers[worker_index.0].store(ipv6.0, Ordering::Release);
if let Some(message) = ipv4.1.map(StatisticsMessage::Ipv4PeerHistogram) {
if let Err(err) = statistics_sender.try_send(message) {
::log::error!("couldn't send statistics message: {:#}", err)
}
}
if let Some(message) = ipv6.1.map(StatisticsMessage::Ipv6PeerHistogram) {
if let Err(err) = statistics_sender.try_send(message) {
::log::error!("couldn't send statistics message: {:#}", err)
}
}
}
last_cleaning = now;
}
if config.statistics.active()
@ -131,6 +120,7 @@ pub fn run_swarm_worker(
fn handle_announce_request<I: Ip>(
config: &Config,
rng: &mut SmallRng,
statistics_sender: &Sender<StatisticsMessage>,
torrents: &mut TorrentMap<I>,
request: AnnounceRequest,
peer_ip: I,
@ -150,6 +140,8 @@ fn handle_announce_request<I: Ip>(
let peer_status = PeerStatus::from_event_and_bytes_left(request.event, request.bytes_left);
torrent_data.update_peer(
config,
statistics_sender,
request.peer_id,
peer_ip,
request.port,

77
aquatic_udp/src/workers/swarm/storage.rs
View file

@ -1,5 +1,6 @@
use std::net::Ipv4Addr;
use std::net::Ipv6Addr;
use std::sync::atomic::Ordering;
use std::sync::Arc;
use aquatic_common::IndexMap;
@ -11,6 +12,7 @@ use aquatic_common::{
};
use aquatic_udp_protocol::*;
use crossbeam_channel::Sender;
use hdrhistogram::Histogram;
use rand::prelude::SmallRng;
@ -46,6 +48,8 @@ pub struct TorrentData<I: Ip> {
impl<I: Ip> TorrentData<I> {
pub fn update_peer(
&mut self,
config: &Config,
statistics_sender: &Sender<StatisticsMessage>,
peer_id: PeerId,
ip_address: I,
port: Port,
@ -78,6 +82,20 @@ impl<I: Ip> TorrentData<I> {
PeerStatus::Stopped => self.peers.remove(&peer_id),
};
if config.statistics.extended {
match (status, opt_removed_peer.is_some()) {
// We added a new peer
(PeerStatus::Leeching | PeerStatus::Seeding, false) => {
statistics_sender.try_send(StatisticsMessage::PeerAdded(peer_id));
}
// We removed an existing peer
(PeerStatus::Stopped, true) => {
statistics_sender.try_send(StatisticsMessage::PeerRemoved(peer_id));
}
_ => (),
}
}
if let Some(Peer {
is_seeder: true, ..
}) = opt_removed_peer
@ -117,12 +135,22 @@ impl<I: Ip> TorrentData<I> {
}
/// Remove inactive peers and reclaim space
fn clean(&mut self, now: SecondsSinceServerStart) {
self.peers.retain(|_, peer| {
fn clean(
&mut self,
config: &Config,
statistics_sender: &Sender<StatisticsMessage>,
now: SecondsSinceServerStart,
) {
self.peers.retain(|peer_id, peer| {
let keep = peer.valid_until.valid(now);
if (!keep) & peer.is_seeder {
self.num_seeders -= 1;
if !keep {
if peer.is_seeder {
self.num_seeders -= 1;
}
if config.statistics.extended {
statistics_sender.try_send(StatisticsMessage::PeerRemoved(*peer_id));
}
}
keep
@ -151,6 +179,7 @@ impl<I: Ip> TorrentMap<I> {
fn clean_and_get_statistics(
&mut self,
config: &Config,
statistics_sender: &Sender<StatisticsMessage>,
access_list_cache: &mut AccessListCache,
access_list_mode: AccessListMode,
now: SecondsSinceServerStart,
@ -178,7 +207,7 @@ impl<I: Ip> TorrentMap<I> {
return false;
}
torrent.clean(now);
torrent.clean(config, statistics_sender, now);
num_peers += torrent.peers.len();
@ -225,28 +254,42 @@ impl Default for TorrentMaps {
}
impl TorrentMaps {
/// Remove forbidden or inactive torrents, reclaim space and return number of remaining peers
pub fn clean_and_get_statistics(
/// Remove forbidden or inactive torrents, reclaim space and update statistics
pub fn clean_and_update_statistics(
&mut self,
config: &Config,
state: &State,
statistics_sender: &Sender<StatisticsMessage>,
access_list: &Arc<AccessListArcSwap>,
server_start_instant: ServerStartInstant,
) -> (
(usize, Option<Histogram<u64>>),
(usize, Option<Histogram<u64>>),
worker_index: SwarmWorkerIndex,
) {
let mut cache = create_access_list_cache(access_list);
let mode = config.access_list.mode;
let now = server_start_instant.seconds_elapsed();
let ipv4 = self
.ipv4
.clean_and_get_statistics(config, &mut cache, mode, now);
let ipv6 = self
.ipv6
.clean_and_get_statistics(config, &mut cache, mode, now);
let ipv4 =
self.ipv4
.clean_and_get_statistics(config, statistics_sender, &mut cache, mode, now);
let ipv6 =
self.ipv6
.clean_and_get_statistics(config, statistics_sender, &mut cache, mode, now);
(ipv4, ipv6)
if config.statistics.active() {
state.statistics_ipv4.peers[worker_index.0].store(ipv4.0, Ordering::Release);
state.statistics_ipv6.peers[worker_index.0].store(ipv6.0, Ordering::Release);
if let Some(message) = ipv4.1.map(StatisticsMessage::Ipv4PeerHistogram) {
if let Err(err) = statistics_sender.try_send(message) {
::log::error!("couldn't send statistics message: {:#}", err);
}
}
if let Some(message) = ipv6.1.map(StatisticsMessage::Ipv6PeerHistogram) {
if let Err(err) = statistics_sender.try_send(message) {
::log::error!("couldn't send statistics message: {:#}", err);
}
}
}
}
}

25
aquatic_udp/templates/statistics.html
View file

@ -251,5 +251,30 @@
{{ endif }}
{{ endif }}
{{ if extended_active }}
<h2>Peer clients</h2>
<table>
<thead>
<tr>
<th>Client</th>
<th>Peer ID prefix (hex)</th>
<th>Count</th>
</tr>
</thead>
<tbody>
{{ for value in peer_clients }}
<tr>
<td>{ value.0 }</td>
<td>{ value.1 }</td>
<td>{ value.2 }</td>
</tr>
{{ endfor }}
</tbody>
</table>
{{ endif }}
</body>
</html>

2
aquatic_udp_protocol/Cargo.toml
View file

@ -11,6 +11,8 @@ readme.workspace = true
rust-version.workspace = true
[dependencies]
aquatic_peer_id.workspace = true
byteorder = "1"
either = "1"

18
aquatic_udp_protocol/src/common.rs
View file

@ -1,6 +1,8 @@
use std::fmt::Debug;
use std::net::{Ipv4Addr, Ipv6Addr};
pub use aquatic_peer_id::{PeerClient, PeerId};
pub trait Ip: Clone + Copy + Debug + PartialEq + Eq {}
impl Ip for Ipv4Addr {}
@ -30,9 +32,6 @@ pub struct NumberOfDownloads(pub i32);
#[derive(PartialEq, Eq, Hash, Clone, Copy, Debug)]
pub struct Port(pub u16);
#[derive(PartialEq, Eq, Hash, Clone, Copy, Debug, PartialOrd, Ord)]
pub struct PeerId(pub [u8; 20]);
#[derive(PartialEq, Eq, Hash, Clone, Copy, Debug)]
pub struct PeerKey(pub u32);
@ -55,19 +54,6 @@ impl quickcheck::Arbitrary for InfoHash {
}
}
#[cfg(test)]
impl quickcheck::Arbitrary for PeerId {
fn arbitrary(g: &mut quickcheck::Gen) -> Self {
let mut bytes = [0u8; 20];
for byte in bytes.iter_mut() {
*byte = u8::arbitrary(g);
}
Self(bytes)
}
}
#[cfg(test)]
impl<I: Ip + quickcheck::Arbitrary> quickcheck::Arbitrary for ResponsePeer<I> {
fn arbitrary(g: &mut quickcheck::Gen) -> Self {

2
aquatic_udp_protocol/src/request.rs
View file

@ -5,6 +5,8 @@ use std::net::Ipv4Addr;
use byteorder::{NetworkEndian, ReadBytesExt, WriteBytesExt};
use either::Either;
use aquatic_peer_id::PeerId;
use super::common::*;
const PROTOCOL_IDENTIFIER: i64 = 4_497_486_125_440;