Merge pull request #127 from greatest-ape/work-2023-02-25

Use regular instead of amortized index map; ignore request.key in aquatic_http; update dependencies

CHANGELOG.md

@@ -17,8 +17,8 @@
 * Use [Rust 1.64 workspace inheritance](https://blog.rust-lang.org/2022/09/22/Rust-1.64.0.html)
 * Reduce space taken by ValidUntil struct from 128 to 32 bits, reducing memory
   consumption for each stored peer by same amount
-* Use regular (non-amortized) IndexMap for peer and pending scrape response
+* Use regular indexmap instead of amortized-indexmap. This goes for torrent,
-  maps (but not for torrent maps)
+  peer and pending scrape response maps 
 * Improve privilege dropping
 * Quit whole program if any thread panics
 * Update dependencies

Cargo.lock

@@ -81,7 +81,6 @@ dependencies = [
  "hex",
  "hwloc",
  "indexmap",
- "indexmap-amortized",
  "libc",
  "log",
  "privdrop",
@@ -123,7 +122,6 @@ dependencies = [
  "serde",
  "signal-hook",
  "slab",
- "smartstring",
  "socket2 0.4.7",
 ]
 
@@ -181,6 +179,7 @@ dependencies = [
  "anyhow",
  "axum",
  "bendy",
+ "compact_str",
  "criterion",
  "hex",
  "httparse",
@@ -191,7 +190,6 @@ dependencies = [
  "quickcheck_macros",
  "serde",
  "serde_bencode",
- "smartstring",
  "urlencoding",
 ]
 
@@ -417,12 +415,6 @@ dependencies = [
  "num-traits",
 ]
 
-[[package]]
-name = "atone"
-version = "0.3.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5a8a5a5e2e3b1e128ec3783765c91c1f392172c1fffc90fd1b96a13bb18e0dd6"
-
 [[package]]
 name = "atty"
 version = "0.2.14"
@@ -608,6 +600,15 @@ version = "0.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5"
 
+[[package]]
+name = "castaway"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8a17ed5635fc8536268e5d4de1e22e81ac34419e5f052d4d51f4e01dcc263fcc"
+dependencies = [
+ "rustversion",
+]
+
 [[package]]
 name = "cc"
 version = "1.0.79"
@@ -642,6 +643,20 @@ dependencies = [
  "winapi 0.3.9",
 ]
 
+[[package]]
+name = "compact_str"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bff0805f79ecb1b35163f3957a6934ea8d04fcd36ef98b52e7316f63e72e73d1"
+dependencies = [
+ "castaway",
+ "cfg-if",
+ "itoa",
+ "ryu",
+ "serde",
+ "static_assertions",
+]
+
 [[package]]
 name = "concurrent-queue"
 version = "1.2.4"
@@ -1228,15 +1243,6 @@ dependencies = [
  "typenum",
 ]
 
-[[package]]
-name = "griddle"
-version = "0.4.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e46a17d0c5d05b045e3814403914aeeb952aa9832f47b1d7ab123ab3d1660454"
-dependencies = [
- "hashbrown 0.9.1",
-]
-
 [[package]]
 name = "half"
 version = "1.8.2"
@@ -1253,12 +1259,6 @@ dependencies = [
  "serde",
 ]
 
-[[package]]
-name = "hashbrown"
-version = "0.9.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d7afe4a420e3fe79967a00898cc1f4db7c8a49a9333a29f8a4bd76a253d5cd04"
-
 [[package]]
 name = "hashbrown"
 version = "0.12.3"
@@ -1457,17 +1457,6 @@ dependencies = [
  "hashbrown 0.12.3",
 ]
 
-[[package]]
-name = "indexmap-amortized"
-version = "1.6.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "81b5a05ffb45214e51fdd40c1f773ab57c74d2a7b41cfadc7ea443acf0359df1"
-dependencies = [
- "atone",
- "autocfg",
- "griddle",
-]
-
 [[package]]
 name = "indicatif"
 version = "0.17.3"
@@ -2575,17 +2564,6 @@ version = "1.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0"
 
-[[package]]
-name = "smartstring"
-version = "1.0.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3fb72c633efbaa2dd666986505016c32c3044395ceaf881518399d2f4127ee29"
-dependencies = [
- "autocfg",
- "static_assertions",
- "version_check",
-]
-
 [[package]]
 name = "snafu"
 version = "0.7.4"
@@ -2778,9 +2756,9 @@ checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601"
 
 [[package]]
 name = "syn"
-version = "1.0.107"
+version = "1.0.109"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1f4064b5b16e03ae50984a5a8ed5d4f8803e6bc1fd170a3cda91a1be4b18e3f5"
+checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -2824,9 +2802,9 @@ dependencies = [
 
 [[package]]
 name = "time"
-version = "0.3.19"
+version = "0.3.20"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "53250a3b3fed8ff8fd988587d8925d26a83ac3845d9e03b220b37f34c2b8d6c2"
+checksum = "cd0cbfecb4d19b5ea75bb31ad904eb5b9fa13f21079c3b92017ebdf4999a5890"
 dependencies = [
  "itoa",
  "libc",
@@ -2844,9 +2822,9 @@ checksum = "2e153e1f1acaef8acc537e68b44906d2db6436e2b35ac2c6b42640fff91f00fd"
 
 [[package]]
 name = "time-macros"
-version = "0.2.7"
+version = "0.2.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a460aeb8de6dcb0f381e1ee05f1cd56fcf5a5f6eb8187ff3d8f0b11078d38b7c"
+checksum = "fd80a657e71da814b8e5d60d3374fc6d35045062245d80224748ae522dd76f36"
 dependencies = [
  "time-core",
 ]
@@ -2920,9 +2898,9 @@ dependencies = [
 
 [[package]]
 name = "tokio-stream"
-version = "0.1.11"
+version = "0.1.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d660770404473ccd7bc9f8b28494a811bc18542b915c0855c51e8f419d5223ce"
+checksum = "8fb52b74f05dbf495a8fba459fdc331812b96aa086d9eb78101fa0d4569c3313"
 dependencies = [
  "futures-core",
  "pin-project-lite",

README.md

@@ -13,11 +13,11 @@ of sub-implementations for different protocols:
 [mio]: https://github.com/tokio-rs/mio
 [glommio]: https://github.com/DataDog/glommio
 
-| Name         | Protocol                                   | OS requirements              |
+| Name         | Protocol                                     | OS requirements              |
-|--------------|--------------------------------------------|------------------------------|
+|--------------|----------------------------------------------|------------------------------|
-| aquatic_udp  | [BitTorrent over UDP]                      | Unix-like (using [mio])      |
+| aquatic_udp  | [BitTorrent over UDP]                        | Unix-like (using [mio])      |
-| aquatic_http | [BitTorrent over HTTP] with TLS ([rustls]) | Linux 5.8+ (using [glommio]) |
+| aquatic_http | [BitTorrent over HTTP] over TLS ([rustls])   | Linux 5.8+ (using [glommio]) |
-| aquatic_ws   | [WebTorrent] over TLS ([rustls], optional) | Linux 5.8+ (using [glommio]) |
+| aquatic_ws   | [WebTorrent], optionally over TLS ([rustls]) | Linux 5.8+ (using [glommio]) |
 
 Features at a glance:
 

TODO.md

@@ -2,10 +2,18 @@
 
 ## High priority
 
-* Do new udp load test round
+* udp prometheus metrics
-* udp: support link to arbitrary homepage as well as embedded tracker URL in statistics page
 * ws: wait for crates release of glommio with membarrier fix (PR #558)
 * Release new version
+* More non-CI integration tests?
+* Remove aquatic_http_private?
+
+## Medium priority
+
+* Consider replacing unmaintained indexmap-amortized with plain indexmap
+* Run cargo-fuzz on protocol crates
+* udp: support link to arbitrary homepage as well as embedded tracker URL in statistics page
+
 * Consider storing torrents in separate IndexMaps. The amount should be a power
   of 2 and should be configurable. They could be stored in a Vec and the index
   could be calculated by taking the first N bits of the info hash. Each such map
@@ -14,11 +22,6 @@
   the map will be cleaned. When doing the normal cleaning round, recently
   cleaned maps would be skipped.
 
-## Medium priority
-
-* Consider replacing unmaintained indexmap-amortized with plain indexmap
-* Run cargo-fuzz on protocol crates
-
 * quit whole program if any thread panics
   * But it would be nice not to panic in workers, but to return errors instead.
     Once JoinHandle::is_finished is available in stable Rust (#90470), an

aquatic_common/Cargo.toml

@@ -26,7 +26,6 @@ git-testament = "0.2"
 hashbrown = "0.13"
 hex = "0.4"
 indexmap = "1"
-indexmap-amortized = "1"
 libc = "0.2"
 log = "0.4"
 privdrop = "0.5"

aquatic_common/src/lib.rs

@@ -16,9 +16,6 @@ pub mod rustls_config;
 /// IndexMap using AHash hasher
 pub type IndexMap<K, V> = indexmap::IndexMap<K, V, RandomState>;
 
-/// Amortized IndexMap using AHash hasher
-pub type AmortizedIndexMap<K, V> = indexmap_amortized::IndexMap<K, V, RandomState>;
-
 /// Peer, connection or similar valid until this instant
 #[derive(Debug, Clone, Copy)]
 pub struct ValidUntil(SecondsSinceServerStart);

aquatic_http/Cargo.toml

@@ -46,7 +46,6 @@ rustls-pemfile = "1"
 serde = { version = "1", features = ["derive"] }
 signal-hook = { version = "0.3" }
 slab = "0.4"
-smartstring = "1"
 socket2 = { version = "0.4", features = ["all"] }
 
 [dev-dependencies]

aquatic_http/src/workers/swarm.rs

@@ -5,7 +5,6 @@ use std::rc::Rc;
 use std::sync::Arc;
 use std::time::Duration;
 
-use either::Either;
 use futures_lite::{Stream, StreamExt};
 use glommio::channels::channel_mesh::{MeshBuilder, Partial, Role};
 use glommio::timer::TimerActionRepeat;
@@ -13,12 +12,12 @@ use glommio::{enclose, prelude::*};
 use rand::prelude::SmallRng;
 use rand::Rng;
 use rand::SeedableRng;
-use smartstring::{LazyCompact, SmartString};
 
 use aquatic_common::access_list::{create_access_list_cache, AccessListArcSwap, AccessListCache};
-use aquatic_common::{extract_response_peers, IndexMap, PanicSentinel};
+use aquatic_common::{
-use aquatic_common::{AmortizedIndexMap, CanonicalSocketAddr};
+    extract_response_peers, CanonicalSocketAddr, IndexMap, PanicSentinel, SecondsSinceServerStart,
-use aquatic_common::{SecondsSinceServerStart, ServerStartInstant, ValidUntil};
+    ServerStartInstant, ValidUntil,
+};
 use aquatic_http_protocol::common::*;
 use aquatic_http_protocol::request::*;
 use aquatic_http_protocol::response::ResponsePeer;
@@ -91,7 +90,7 @@ impl<I: Ip> Peer<I> {
 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
 pub struct PeerMapKey<I: Ip> {
     pub peer_id: PeerId,
-    pub ip_or_key: Either<I, SmartString<LazyCompact>>,
+    pub ip: I,
 }
 
 pub type PeerMap<I> = IndexMap<PeerMapKey<I>, Peer<I>>;
@@ -117,7 +116,7 @@ impl<I: Ip> TorrentData<I> {
     }
 }
 
-pub type TorrentMap<I> = AmortizedIndexMap<InfoHash, TorrentData<I>>;
+pub type TorrentMap<I> = IndexMap<InfoHash, TorrentData<I>>;
 
 #[derive(Default)]
 pub struct TorrentMaps {
@@ -389,14 +388,9 @@ pub fn upsert_peer_and_get_response_peers<I: Ip>(
     let peer_status =
         PeerStatus::from_event_and_bytes_left(request.event, Some(request.bytes_left));
 
-    let ip_or_key = request
-        .key
-        .map(Either::Right)
-        .unwrap_or_else(|| Either::Left(peer_ip_address));
-
     let peer_map_key = PeerMapKey {
         peer_id: request.peer_id,
-        ip_or_key,
+        ip: peer_ip_address,
     };
 
     let opt_removed_peer = match peer_status {

aquatic_http_private/src/workers/swarm/common.rs

@@ -1,8 +1,6 @@
 use std::net::{Ipv4Addr, Ipv6Addr};
 
-use aquatic_common::{
+use aquatic_common::{IndexMap, SecondsSinceServerStart, ServerStartInstant, ValidUntil};
-    AmortizedIndexMap, IndexMap, SecondsSinceServerStart, ServerStartInstant, ValidUntil,
-};
 use aquatic_http_protocol::common::{AnnounceEvent, InfoHash, PeerId};
 use aquatic_http_protocol::response::ResponsePeer;
 
@@ -76,7 +74,7 @@ impl<I: Ip> Default for TorrentData<I> {
     }
 }
 
-pub type TorrentMap<I> = AmortizedIndexMap<InfoHash, TorrentData<I>>;
+pub type TorrentMap<I> = IndexMap<InfoHash, TorrentData<I>>;
 
 #[derive(Default)]
 pub struct TorrentMaps {

aquatic_http_protocol/Cargo.toml

@@ -26,6 +26,7 @@ harness = false
 [dependencies]
 anyhow = "1"
 axum = { version = "0.5", optional = true, default-features = false }
+compact_str = { version = "0.7", features = ["serde"] }
 hex = { version = "0.4", default-features = false }
 httparse = "1"
 itoa = "1"
@@ -33,7 +34,6 @@ log = "0.4"
 memchr = "2"
 serde = { version = "1", features = ["derive"] }
 serde_bencode = "0.2"
-smartstring = "1"
 urlencoding = "2"
 
 [dev-dependencies]

aquatic_http_protocol/src/request.rs

@@ -1,7 +1,7 @@
 use std::io::Write;
 
 use anyhow::Context;
-use smartstring::{LazyCompact, SmartString};
+use compact_str::CompactString;
 
 use super::common::*;
 use super::utils::*;
@@ -17,7 +17,7 @@ pub struct AnnounceRequest {
     pub event: AnnounceEvent,
     /// Number of response peers wanted
     pub numwant: Option<usize>,
-    pub key: Option<SmartString<LazyCompact>>,
+    pub key: Option<CompactString>,
 }
 
 impl AnnounceRequest {

aquatic_udp/src/workers/swarm/storage.rs

@@ -7,7 +7,7 @@ use aquatic_common::SecondsSinceServerStart;
 use aquatic_common::ServerStartInstant;
 use aquatic_common::{
     access_list::{create_access_list_cache, AccessListArcSwap, AccessListCache, AccessListMode},
-    extract_response_peers, AmortizedIndexMap, ValidUntil,
+    extract_response_peers, ValidUntil,
 };
 
 use aquatic_udp_protocol::*;
@@ -144,7 +144,7 @@ impl<I: Ip> Default for TorrentData<I> {
 }
 
 #[derive(Default)]
-pub struct TorrentMap<I: Ip>(pub AmortizedIndexMap<InfoHash, TorrentData<I>>);
+pub struct TorrentMap<I: Ip>(pub IndexMap<InfoHash, TorrentData<I>>);
 
 impl<I: Ip> TorrentMap<I> {
     /// Remove forbidden or inactive torrents, reclaim space and return number of remaining peers

aquatic_ws/src/workers/swarm.rs

@@ -13,8 +13,7 @@ use hashbrown::HashMap;
 use rand::{rngs::SmallRng, SeedableRng};
 
 use aquatic_common::{
-    extract_response_peers, AmortizedIndexMap, IndexMap, PanicSentinel, SecondsSinceServerStart,
+    extract_response_peers, IndexMap, PanicSentinel, SecondsSinceServerStart, ServerStartInstant,
-    ServerStartInstant,
 };
 use aquatic_ws_protocol::*;
 
@@ -87,7 +86,7 @@ impl TorrentData {
     }
 }
 
-type TorrentMap = AmortizedIndexMap<InfoHash, TorrentData>;
+type TorrentMap = IndexMap<InfoHash, TorrentData>;
 
 #[derive(Default)]
 struct TorrentMaps {