From 084067ee74af6f1284c05a4383b5d1ef9b57c9f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20Frosteg=C3=A5rd?= Date: Sat, 25 Feb 2023 22:21:56 +0100 Subject: [PATCH 1/6] Update README --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 894e2db..e99def3 100644 --- a/README.md +++ b/README.md @@ -13,11 +13,11 @@ of sub-implementations for different protocols: [mio]: https://github.com/tokio-rs/mio [glommio]: https://github.com/DataDog/glommio -| Name | Protocol | OS requirements | -|--------------|--------------------------------------------|------------------------------| -| aquatic_udp | [BitTorrent over UDP] | Unix-like (using [mio]) | -| aquatic_http | [BitTorrent over HTTP] with TLS ([rustls]) | Linux 5.8+ (using [glommio]) | -| aquatic_ws | [WebTorrent] over TLS ([rustls], optional) | Linux 5.8+ (using [glommio]) | +| Name | Protocol | OS requirements | +|--------------|----------------------------------------------|------------------------------| +| aquatic_udp | [BitTorrent over UDP] | Unix-like (using [mio]) | +| aquatic_http | [BitTorrent over HTTP] over TLS ([rustls]) | Linux 5.8+ (using [glommio]) | +| aquatic_ws | [WebTorrent], optionally over TLS ([rustls]) | Linux 5.8+ (using [glommio]) | Features at a glance: From beb8c52fe6034aaef31a58d04ed751fd5a24cf1a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20Frosteg=C3=A5rd?= Date: Sat, 25 Feb 2023 22:24:41 +0100 Subject: [PATCH 2/6] Update TODO --- TODO.md | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/TODO.md b/TODO.md index 508cda7..cdebd4d 100644 --- a/TODO.md +++ b/TODO.md @@ -2,10 +2,18 @@ ## High priority -* Do new udp load test round -* udp: support link to arbitrary homepage as well as embedded tracker URL in statistics page +* udp prometheus metrics * ws: wait for crates release of glommio with membarrier fix (PR #558) * Release new version +* More non-CI integration tests? +* Remove aquatic_http_private? + +## Medium priority + +* Consider replacing unmaintained indexmap-amortized with plain indexmap +* Run cargo-fuzz on protocol crates +* udp: support link to arbitrary homepage as well as embedded tracker URL in statistics page + * Consider storing torrents in separate IndexMaps. The amount should be a power of 2 and should be configurable. They could be stored in a Vec and the index could be calculated by taking the first N bits of the info hash. Each such map @@ -14,11 +22,6 @@ the map will be cleaned. When doing the normal cleaning round, recently cleaned maps would be skipped. -## Medium priority - -* Consider replacing unmaintained indexmap-amortized with plain indexmap -* Run cargo-fuzz on protocol crates - * quit whole program if any thread panics * But it would be nice not to panic in workers, but to return errors instead. Once JoinHandle::is_finished is available in stable Rust (#90470), an From b42d55b003aa69aa9471b82bb3c31eceeb8750d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20Frosteg=C3=A5rd?= Date: Sat, 25 Feb 2023 22:37:54 +0100 Subject: [PATCH 3/6] Replace indexmap-amortized with plain (ahash) indexmap --- Cargo.lock | 33 ------------------- aquatic_common/Cargo.toml | 1 - aquatic_common/src/lib.rs | 3 -- aquatic_http/src/workers/swarm.rs | 9 ++--- .../src/workers/swarm/common.rs | 6 ++-- aquatic_udp/src/workers/swarm/storage.rs | 4 +-- aquatic_ws/src/workers/swarm.rs | 5 ++- 7 files changed, 11 insertions(+), 50 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3f4eb6c..e17c1d5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -81,7 +81,6 @@ dependencies = [ "hex", "hwloc", "indexmap", - "indexmap-amortized", "libc", "log", "privdrop", @@ -417,12 +416,6 @@ dependencies = [ "num-traits", ] -[[package]] -name = "atone" -version = "0.3.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a8a5a5e2e3b1e128ec3783765c91c1f392172c1fffc90fd1b96a13bb18e0dd6" - [[package]] name = "atty" version = "0.2.14" @@ -1228,15 +1221,6 @@ dependencies = [ "typenum", ] -[[package]] -name = "griddle" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e46a17d0c5d05b045e3814403914aeeb952aa9832f47b1d7ab123ab3d1660454" -dependencies = [ - "hashbrown 0.9.1", -] - [[package]] name = "half" version = "1.8.2" @@ -1253,12 +1237,6 @@ dependencies = [ "serde", ] -[[package]] -name = "hashbrown" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7afe4a420e3fe79967a00898cc1f4db7c8a49a9333a29f8a4bd76a253d5cd04" - [[package]] name = "hashbrown" version = "0.12.3" @@ -1457,17 +1435,6 @@ dependencies = [ "hashbrown 0.12.3", ] -[[package]] -name = "indexmap-amortized" -version = "1.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81b5a05ffb45214e51fdd40c1f773ab57c74d2a7b41cfadc7ea443acf0359df1" -dependencies = [ - "atone", - "autocfg", - "griddle", -] - [[package]] name = "indicatif" version = "0.17.3" diff --git a/aquatic_common/Cargo.toml b/aquatic_common/Cargo.toml index d3cd52a..2de3408 100644 --- a/aquatic_common/Cargo.toml +++ b/aquatic_common/Cargo.toml @@ -26,7 +26,6 @@ git-testament = "0.2" hashbrown = "0.13" hex = "0.4" indexmap = "1" -indexmap-amortized = "1" libc = "0.2" log = "0.4" privdrop = "0.5" diff --git a/aquatic_common/src/lib.rs b/aquatic_common/src/lib.rs index 7d51312..96804ff 100644 --- a/aquatic_common/src/lib.rs +++ b/aquatic_common/src/lib.rs @@ -16,9 +16,6 @@ pub mod rustls_config; /// IndexMap using AHash hasher pub type IndexMap = indexmap::IndexMap; -/// Amortized IndexMap using AHash hasher -pub type AmortizedIndexMap = indexmap_amortized::IndexMap; - /// Peer, connection or similar valid until this instant #[derive(Debug, Clone, Copy)] pub struct ValidUntil(SecondsSinceServerStart); diff --git a/aquatic_http/src/workers/swarm.rs b/aquatic_http/src/workers/swarm.rs index 76893ad..ded87d8 100644 --- a/aquatic_http/src/workers/swarm.rs +++ b/aquatic_http/src/workers/swarm.rs @@ -16,9 +16,10 @@ use rand::SeedableRng; use smartstring::{LazyCompact, SmartString}; use aquatic_common::access_list::{create_access_list_cache, AccessListArcSwap, AccessListCache}; -use aquatic_common::{extract_response_peers, IndexMap, PanicSentinel}; -use aquatic_common::{AmortizedIndexMap, CanonicalSocketAddr}; -use aquatic_common::{SecondsSinceServerStart, ServerStartInstant, ValidUntil}; +use aquatic_common::{ + extract_response_peers, CanonicalSocketAddr, IndexMap, PanicSentinel, SecondsSinceServerStart, + ServerStartInstant, ValidUntil, +}; use aquatic_http_protocol::common::*; use aquatic_http_protocol::request::*; use aquatic_http_protocol::response::ResponsePeer; @@ -117,7 +118,7 @@ impl TorrentData { } } -pub type TorrentMap = AmortizedIndexMap>; +pub type TorrentMap = IndexMap>; #[derive(Default)] pub struct TorrentMaps { diff --git a/aquatic_http_private/src/workers/swarm/common.rs b/aquatic_http_private/src/workers/swarm/common.rs index 277da34..22ad497 100644 --- a/aquatic_http_private/src/workers/swarm/common.rs +++ b/aquatic_http_private/src/workers/swarm/common.rs @@ -1,8 +1,6 @@ use std::net::{Ipv4Addr, Ipv6Addr}; -use aquatic_common::{ - AmortizedIndexMap, IndexMap, SecondsSinceServerStart, ServerStartInstant, ValidUntil, -}; +use aquatic_common::{IndexMap, SecondsSinceServerStart, ServerStartInstant, ValidUntil}; use aquatic_http_protocol::common::{AnnounceEvent, InfoHash, PeerId}; use aquatic_http_protocol::response::ResponsePeer; @@ -76,7 +74,7 @@ impl Default for TorrentData { } } -pub type TorrentMap = AmortizedIndexMap>; +pub type TorrentMap = IndexMap>; #[derive(Default)] pub struct TorrentMaps { diff --git a/aquatic_udp/src/workers/swarm/storage.rs b/aquatic_udp/src/workers/swarm/storage.rs index 6db4ae3..e96416c 100644 --- a/aquatic_udp/src/workers/swarm/storage.rs +++ b/aquatic_udp/src/workers/swarm/storage.rs @@ -7,7 +7,7 @@ use aquatic_common::SecondsSinceServerStart; use aquatic_common::ServerStartInstant; use aquatic_common::{ access_list::{create_access_list_cache, AccessListArcSwap, AccessListCache, AccessListMode}, - extract_response_peers, AmortizedIndexMap, ValidUntil, + extract_response_peers, ValidUntil, }; use aquatic_udp_protocol::*; @@ -144,7 +144,7 @@ impl Default for TorrentData { } #[derive(Default)] -pub struct TorrentMap(pub AmortizedIndexMap>); +pub struct TorrentMap(pub IndexMap>); impl TorrentMap { /// Remove forbidden or inactive torrents, reclaim space and return number of remaining peers diff --git a/aquatic_ws/src/workers/swarm.rs b/aquatic_ws/src/workers/swarm.rs index 21806a3..d852044 100644 --- a/aquatic_ws/src/workers/swarm.rs +++ b/aquatic_ws/src/workers/swarm.rs @@ -13,8 +13,7 @@ use hashbrown::HashMap; use rand::{rngs::SmallRng, SeedableRng}; use aquatic_common::{ - extract_response_peers, AmortizedIndexMap, IndexMap, PanicSentinel, SecondsSinceServerStart, - ServerStartInstant, + extract_response_peers, IndexMap, PanicSentinel, SecondsSinceServerStart, ServerStartInstant, }; use aquatic_ws_protocol::*; @@ -87,7 +86,7 @@ impl TorrentData { } } -type TorrentMap = AmortizedIndexMap; +type TorrentMap = IndexMap; #[derive(Default)] struct TorrentMaps { From 2fa1a046d773004c6c34e9cac36778427ce520f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20Frosteg=C3=A5rd?= Date: Sat, 25 Feb 2023 22:38:46 +0100 Subject: [PATCH 4/6] Run cargo update Updating syn v1.0.107 -> v1.0.109 Updating time v0.3.19 -> v0.3.20 Updating time-macros v0.2.7 -> v0.2.8 Updating tokio-stream v0.1.11 -> v0.1.12 --- Cargo.lock | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e17c1d5..d5ce2ab 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2745,9 +2745,9 @@ checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" [[package]] name = "syn" -version = "1.0.107" +version = "1.0.109" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f4064b5b16e03ae50984a5a8ed5d4f8803e6bc1fd170a3cda91a1be4b18e3f5" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" dependencies = [ "proc-macro2", "quote", @@ -2791,9 +2791,9 @@ dependencies = [ [[package]] name = "time" -version = "0.3.19" +version = "0.3.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53250a3b3fed8ff8fd988587d8925d26a83ac3845d9e03b220b37f34c2b8d6c2" +checksum = "cd0cbfecb4d19b5ea75bb31ad904eb5b9fa13f21079c3b92017ebdf4999a5890" dependencies = [ "itoa", "libc", @@ -2811,9 +2811,9 @@ checksum = "2e153e1f1acaef8acc537e68b44906d2db6436e2b35ac2c6b42640fff91f00fd" [[package]] name = "time-macros" -version = "0.2.7" +version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a460aeb8de6dcb0f381e1ee05f1cd56fcf5a5f6eb8187ff3d8f0b11078d38b7c" +checksum = "fd80a657e71da814b8e5d60d3374fc6d35045062245d80224748ae522dd76f36" dependencies = [ "time-core", ] @@ -2887,9 +2887,9 @@ dependencies = [ [[package]] name = "tokio-stream" -version = "0.1.11" +version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d660770404473ccd7bc9f8b28494a811bc18542b915c0855c51e8f419d5223ce" +checksum = "8fb52b74f05dbf495a8fba459fdc331812b96aa086d9eb78101fa0d4569c3313" dependencies = [ "futures-core", "pin-project-lite", From 1afe45c6f6b018987ef4cbf657b6ebc3e0ce3408 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20Frosteg=C3=A5rd?= Date: Sat, 25 Feb 2023 22:51:25 +0100 Subject: [PATCH 5/6] http: use CompactString in AnnounceRequest, ignore key in tracker --- Cargo.lock | 37 ++++++++++++++++++---------- aquatic_http/Cargo.toml | 1 - aquatic_http/src/workers/swarm.rs | 11 ++------- aquatic_http_protocol/Cargo.toml | 2 +- aquatic_http_protocol/src/request.rs | 4 +-- 5 files changed, 29 insertions(+), 26 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d5ce2ab..0dcb145 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -122,7 +122,6 @@ dependencies = [ "serde", "signal-hook", "slab", - "smartstring", "socket2 0.4.7", ] @@ -180,6 +179,7 @@ dependencies = [ "anyhow", "axum", "bendy", + "compact_str", "criterion", "hex", "httparse", @@ -190,7 +190,6 @@ dependencies = [ "quickcheck_macros", "serde", "serde_bencode", - "smartstring", "urlencoding", ] @@ -601,6 +600,15 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" +[[package]] +name = "castaway" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a17ed5635fc8536268e5d4de1e22e81ac34419e5f052d4d51f4e01dcc263fcc" +dependencies = [ + "rustversion", +] + [[package]] name = "cc" version = "1.0.79" @@ -635,6 +643,20 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "compact_str" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bff0805f79ecb1b35163f3957a6934ea8d04fcd36ef98b52e7316f63e72e73d1" +dependencies = [ + "castaway", + "cfg-if", + "itoa", + "ryu", + "serde", + "static_assertions", +] + [[package]] name = "concurrent-queue" version = "1.2.4" @@ -2542,17 +2564,6 @@ version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" -[[package]] -name = "smartstring" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fb72c633efbaa2dd666986505016c32c3044395ceaf881518399d2f4127ee29" -dependencies = [ - "autocfg", - "static_assertions", - "version_check", -] - [[package]] name = "snafu" version = "0.7.4" diff --git a/aquatic_http/Cargo.toml b/aquatic_http/Cargo.toml index d160bea..de3640c 100644 --- a/aquatic_http/Cargo.toml +++ b/aquatic_http/Cargo.toml @@ -46,7 +46,6 @@ rustls-pemfile = "1" serde = { version = "1", features = ["derive"] } signal-hook = { version = "0.3" } slab = "0.4" -smartstring = "1" socket2 = { version = "0.4", features = ["all"] } [dev-dependencies] diff --git a/aquatic_http/src/workers/swarm.rs b/aquatic_http/src/workers/swarm.rs index ded87d8..a6a5aeb 100644 --- a/aquatic_http/src/workers/swarm.rs +++ b/aquatic_http/src/workers/swarm.rs @@ -5,7 +5,6 @@ use std::rc::Rc; use std::sync::Arc; use std::time::Duration; -use either::Either; use futures_lite::{Stream, StreamExt}; use glommio::channels::channel_mesh::{MeshBuilder, Partial, Role}; use glommio::timer::TimerActionRepeat; @@ -13,7 +12,6 @@ use glommio::{enclose, prelude::*}; use rand::prelude::SmallRng; use rand::Rng; use rand::SeedableRng; -use smartstring::{LazyCompact, SmartString}; use aquatic_common::access_list::{create_access_list_cache, AccessListArcSwap, AccessListCache}; use aquatic_common::{ @@ -92,7 +90,7 @@ impl Peer { #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub struct PeerMapKey { pub peer_id: PeerId, - pub ip_or_key: Either>, + pub ip: I, } pub type PeerMap = IndexMap, Peer>; @@ -390,14 +388,9 @@ pub fn upsert_peer_and_get_response_peers( let peer_status = PeerStatus::from_event_and_bytes_left(request.event, Some(request.bytes_left)); - let ip_or_key = request - .key - .map(Either::Right) - .unwrap_or_else(|| Either::Left(peer_ip_address)); - let peer_map_key = PeerMapKey { peer_id: request.peer_id, - ip_or_key, + ip: peer_ip_address, }; let opt_removed_peer = match peer_status { diff --git a/aquatic_http_protocol/Cargo.toml b/aquatic_http_protocol/Cargo.toml index 9a1d599..606fb32 100644 --- a/aquatic_http_protocol/Cargo.toml +++ b/aquatic_http_protocol/Cargo.toml @@ -26,6 +26,7 @@ harness = false [dependencies] anyhow = "1" axum = { version = "0.5", optional = true, default-features = false } +compact_str = { version = "0.7", features = ["serde"] } hex = { version = "0.4", default-features = false } httparse = "1" itoa = "1" @@ -33,7 +34,6 @@ log = "0.4" memchr = "2" serde = { version = "1", features = ["derive"] } serde_bencode = "0.2" -smartstring = "1" urlencoding = "2" [dev-dependencies] diff --git a/aquatic_http_protocol/src/request.rs b/aquatic_http_protocol/src/request.rs index 520e0d1..db4c7e2 100644 --- a/aquatic_http_protocol/src/request.rs +++ b/aquatic_http_protocol/src/request.rs @@ -1,7 +1,7 @@ use std::io::Write; use anyhow::Context; -use smartstring::{LazyCompact, SmartString}; +use compact_str::CompactString; use super::common::*; use super::utils::*; @@ -17,7 +17,7 @@ pub struct AnnounceRequest { pub event: AnnounceEvent, /// Number of response peers wanted pub numwant: Option, - pub key: Option>, + pub key: Option, } impl AnnounceRequest { From 66ce936a1212292651e78058fcf0c141cba40317 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20Frosteg=C3=A5rd?= Date: Sat, 25 Feb 2023 23:00:54 +0100 Subject: [PATCH 6/6] Update CHANGELOG --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3bdd65c..80b629a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,8 +17,8 @@ * Use [Rust 1.64 workspace inheritance](https://blog.rust-lang.org/2022/09/22/Rust-1.64.0.html) * Reduce space taken by ValidUntil struct from 128 to 32 bits, reducing memory consumption for each stored peer by same amount -* Use regular (non-amortized) IndexMap for peer and pending scrape response - maps (but not for torrent maps) +* Use regular indexmap instead of amortized-indexmap. This goes for torrent, + peer and pending scrape response maps * Improve privilege dropping * Quit whole program if any thread panics * Update dependencies