mirror of
https://github.com/YGGverse/aquatic.git
synced 2026-04-01 02:05:30 +00:00
Merge pull request #127 from greatest-ape/work-2023-02-25
Use regular instead of amortized index map; ignore request.key in aquatic_http; update dependencies
This commit is contained in:
Joakim Frostegård
GitHub
commit
e4b7c8451d
13 changed files with 65 additions and 98 deletions
|
|
@ -17,8 +17,8 @@
|
|||
* Use [Rust 1.64 workspace inheritance](https://blog.rust-lang.org/2022/09/22/Rust-1.64.0.html)
|
||||
* Reduce space taken by ValidUntil struct from 128 to 32 bits, reducing memory
|
||||
consumption for each stored peer by same amount
|
||||
* Use regular (non-amortized) IndexMap for peer and pending scrape response
|
||||
maps (but not for torrent maps)
|
||||
* Use regular indexmap instead of amortized-indexmap. This goes for torrent,
|
||||
peer and pending scrape response maps
|
||||
* Improve privilege dropping
|
||||
* Quit whole program if any thread panics
|
||||
* Update dependencies
|
||||
|
|
|
|||
86
Cargo.lock
generated
86
Cargo.lock
generated
|
|
@ -81,7 +81,6 @@ dependencies = [
|
|||
"hex",
|
||||
"hwloc",
|
||||
"indexmap",
|
||||
"indexmap-amortized",
|
||||
"libc",
|
||||
"log",
|
||||
"privdrop",
|
||||
|
|
@ -123,7 +122,6 @@ dependencies = [
|
|||
"serde",
|
||||
"signal-hook",
|
||||
"slab",
|
||||
"smartstring",
|
||||
"socket2 0.4.7",
|
||||
]
|
||||
|
||||
|
|
@ -181,6 +179,7 @@ dependencies = [
|
|||
"anyhow",
|
||||
"axum",
|
||||
"bendy",
|
||||
"compact_str",
|
||||
"criterion",
|
||||
"hex",
|
||||
"httparse",
|
||||
|
|
@ -191,7 +190,6 @@ dependencies = [
|
|||
"quickcheck_macros",
|
||||
"serde",
|
||||
"serde_bencode",
|
||||
"smartstring",
|
||||
"urlencoding",
|
||||
]
|
||||
|
||||
|
|
@ -417,12 +415,6 @@ dependencies = [
|
|||
"num-traits",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "atone"
|
||||
version = "0.3.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5a8a5a5e2e3b1e128ec3783765c91c1f392172c1fffc90fd1b96a13bb18e0dd6"
|
||||
|
||||
[[package]]
|
||||
name = "atty"
|
||||
version = "0.2.14"
|
||||
|
|
@ -608,6 +600,15 @@ version = "0.3.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5"
|
||||
|
||||
[[package]]
|
||||
name = "castaway"
|
||||
version = "0.2.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8a17ed5635fc8536268e5d4de1e22e81ac34419e5f052d4d51f4e01dcc263fcc"
|
||||
dependencies = [
|
||||
"rustversion",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cc"
|
||||
version = "1.0.79"
|
||||
|
|
@ -642,6 +643,20 @@ dependencies = [
|
|||
"winapi 0.3.9",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "compact_str"
|
||||
version = "0.7.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bff0805f79ecb1b35163f3957a6934ea8d04fcd36ef98b52e7316f63e72e73d1"
|
||||
dependencies = [
|
||||
"castaway",
|
||||
"cfg-if",
|
||||
"itoa",
|
||||
"ryu",
|
||||
"serde",
|
||||
"static_assertions",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "concurrent-queue"
|
||||
version = "1.2.4"
|
||||
|
|
@ -1228,15 +1243,6 @@ dependencies = [
|
|||
"typenum",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "griddle"
|
||||
version = "0.4.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e46a17d0c5d05b045e3814403914aeeb952aa9832f47b1d7ab123ab3d1660454"
|
||||
dependencies = [
|
||||
"hashbrown 0.9.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "half"
|
||||
version = "1.8.2"
|
||||
|
|
@ -1253,12 +1259,6 @@ dependencies = [
|
|||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hashbrown"
|
||||
version = "0.9.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d7afe4a420e3fe79967a00898cc1f4db7c8a49a9333a29f8a4bd76a253d5cd04"
|
||||
|
||||
[[package]]
|
||||
name = "hashbrown"
|
||||
version = "0.12.3"
|
||||
|
|
@ -1457,17 +1457,6 @@ dependencies = [
|
|||
"hashbrown 0.12.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "indexmap-amortized"
|
||||
version = "1.6.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "81b5a05ffb45214e51fdd40c1f773ab57c74d2a7b41cfadc7ea443acf0359df1"
|
||||
dependencies = [
|
||||
"atone",
|
||||
"autocfg",
|
||||
"griddle",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "indicatif"
|
||||
version = "0.17.3"
|
||||
|
|
@ -2575,17 +2564,6 @@ version = "1.10.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0"
|
||||
|
||||
[[package]]
|
||||
name = "smartstring"
|
||||
version = "1.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3fb72c633efbaa2dd666986505016c32c3044395ceaf881518399d2f4127ee29"
|
||||
dependencies = [
|
||||
"autocfg",
|
||||
"static_assertions",
|
||||
"version_check",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "snafu"
|
||||
version = "0.7.4"
|
||||
|
|
@ -2778,9 +2756,9 @@ checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601"
|
|||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "1.0.107"
|
||||
version = "1.0.109"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1f4064b5b16e03ae50984a5a8ed5d4f8803e6bc1fd170a3cda91a1be4b18e3f5"
|
||||
checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
|
|
@ -2824,9 +2802,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "time"
|
||||
version = "0.3.19"
|
||||
version = "0.3.20"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "53250a3b3fed8ff8fd988587d8925d26a83ac3845d9e03b220b37f34c2b8d6c2"
|
||||
checksum = "cd0cbfecb4d19b5ea75bb31ad904eb5b9fa13f21079c3b92017ebdf4999a5890"
|
||||
dependencies = [
|
||||
"itoa",
|
||||
"libc",
|
||||
|
|
@ -2844,9 +2822,9 @@ checksum = "2e153e1f1acaef8acc537e68b44906d2db6436e2b35ac2c6b42640fff91f00fd"
|
|||
|
||||
[[package]]
|
||||
name = "time-macros"
|
||||
version = "0.2.7"
|
||||
version = "0.2.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a460aeb8de6dcb0f381e1ee05f1cd56fcf5a5f6eb8187ff3d8f0b11078d38b7c"
|
||||
checksum = "fd80a657e71da814b8e5d60d3374fc6d35045062245d80224748ae522dd76f36"
|
||||
dependencies = [
|
||||
"time-core",
|
||||
]
|
||||
|
|
@ -2920,9 +2898,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "tokio-stream"
|
||||
version = "0.1.11"
|
||||
version = "0.1.12"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d660770404473ccd7bc9f8b28494a811bc18542b915c0855c51e8f419d5223ce"
|
||||
checksum = "8fb52b74f05dbf495a8fba459fdc331812b96aa086d9eb78101fa0d4569c3313"
|
||||
dependencies = [
|
||||
"futures-core",
|
||||
"pin-project-lite",
|
||||
|
|
|
|||
10
README.md
10
README.md
|
|
@ -13,11 +13,11 @@ of sub-implementations for different protocols:
|
|||
[mio]: https://github.com/tokio-rs/mio
|
||||
[glommio]: https://github.com/DataDog/glommio
|
||||
|
||||
| Name | Protocol | OS requirements |
|
||||
|--------------|--------------------------------------------|------------------------------|
|
||||
| aquatic_udp | [BitTorrent over UDP] | Unix-like (using [mio]) |
|
||||
| aquatic_http | [BitTorrent over HTTP] with TLS ([rustls]) | Linux 5.8+ (using [glommio]) |
|
||||
| aquatic_ws | [WebTorrent] over TLS ([rustls], optional) | Linux 5.8+ (using [glommio]) |
|
||||
| Name | Protocol | OS requirements |
|
||||
|--------------|----------------------------------------------|------------------------------|
|
||||
| aquatic_udp | [BitTorrent over UDP] | Unix-like (using [mio]) |
|
||||
| aquatic_http | [BitTorrent over HTTP] over TLS ([rustls]) | Linux 5.8+ (using [glommio]) |
|
||||
| aquatic_ws | [WebTorrent], optionally over TLS ([rustls]) | Linux 5.8+ (using [glommio]) |
|
||||
|
||||
Features at a glance:
|
||||
|
||||
|
|
|
|||
17
TODO.md
17
TODO.md
|
|
@ -2,10 +2,18 @@
|
|||
|
||||
## High priority
|
||||
|
||||
* Do new udp load test round
|
||||
* udp: support link to arbitrary homepage as well as embedded tracker URL in statistics page
|
||||
* udp prometheus metrics
|
||||
* ws: wait for crates release of glommio with membarrier fix (PR #558)
|
||||
* Release new version
|
||||
* More non-CI integration tests?
|
||||
* Remove aquatic_http_private?
|
||||
|
||||
## Medium priority
|
||||
|
||||
* Consider replacing unmaintained indexmap-amortized with plain indexmap
|
||||
* Run cargo-fuzz on protocol crates
|
||||
* udp: support link to arbitrary homepage as well as embedded tracker URL in statistics page
|
||||
|
||||
* Consider storing torrents in separate IndexMaps. The amount should be a power
|
||||
of 2 and should be configurable. They could be stored in a Vec and the index
|
||||
could be calculated by taking the first N bits of the info hash. Each such map
|
||||
|
|
@ -14,11 +22,6 @@
|
|||
the map will be cleaned. When doing the normal cleaning round, recently
|
||||
cleaned maps would be skipped.
|
||||
|
||||
## Medium priority
|
||||
|
||||
* Consider replacing unmaintained indexmap-amortized with plain indexmap
|
||||
* Run cargo-fuzz on protocol crates
|
||||
|
||||
* quit whole program if any thread panics
|
||||
* But it would be nice not to panic in workers, but to return errors instead.
|
||||
Once JoinHandle::is_finished is available in stable Rust (#90470), an
|
||||
|
|
|
|||
|
|
@ -26,7 +26,6 @@ git-testament = "0.2"
|
|||
hashbrown = "0.13"
|
||||
hex = "0.4"
|
||||
indexmap = "1"
|
||||
indexmap-amortized = "1"
|
||||
libc = "0.2"
|
||||
log = "0.4"
|
||||
privdrop = "0.5"
|
||||
|
|
|
|||
|
|
@ -16,9 +16,6 @@ pub mod rustls_config;
|
|||
/// IndexMap using AHash hasher
|
||||
pub type IndexMap<K, V> = indexmap::IndexMap<K, V, RandomState>;
|
||||
|
||||
/// Amortized IndexMap using AHash hasher
|
||||
pub type AmortizedIndexMap<K, V> = indexmap_amortized::IndexMap<K, V, RandomState>;
|
||||
|
||||
/// Peer, connection or similar valid until this instant
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub struct ValidUntil(SecondsSinceServerStart);
|
||||
|
|
|
|||
|
|
@ -46,7 +46,6 @@ rustls-pemfile = "1"
|
|||
serde = { version = "1", features = ["derive"] }
|
||||
signal-hook = { version = "0.3" }
|
||||
slab = "0.4"
|
||||
smartstring = "1"
|
||||
socket2 = { version = "0.4", features = ["all"] }
|
||||
|
||||
[dev-dependencies]
|
||||
|
|
|
|||
|
|
@ -5,7 +5,6 @@ use std::rc::Rc;
|
|||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use either::Either;
|
||||
use futures_lite::{Stream, StreamExt};
|
||||
use glommio::channels::channel_mesh::{MeshBuilder, Partial, Role};
|
||||
use glommio::timer::TimerActionRepeat;
|
||||
|
|
@ -13,12 +12,12 @@ use glommio::{enclose, prelude::*};
|
|||
use rand::prelude::SmallRng;
|
||||
use rand::Rng;
|
||||
use rand::SeedableRng;
|
||||
use smartstring::{LazyCompact, SmartString};
|
||||
|
||||
use aquatic_common::access_list::{create_access_list_cache, AccessListArcSwap, AccessListCache};
|
||||
use aquatic_common::{extract_response_peers, IndexMap, PanicSentinel};
|
||||
use aquatic_common::{AmortizedIndexMap, CanonicalSocketAddr};
|
||||
use aquatic_common::{SecondsSinceServerStart, ServerStartInstant, ValidUntil};
|
||||
use aquatic_common::{
|
||||
extract_response_peers, CanonicalSocketAddr, IndexMap, PanicSentinel, SecondsSinceServerStart,
|
||||
ServerStartInstant, ValidUntil,
|
||||
};
|
||||
use aquatic_http_protocol::common::*;
|
||||
use aquatic_http_protocol::request::*;
|
||||
use aquatic_http_protocol::response::ResponsePeer;
|
||||
|
|
@ -91,7 +90,7 @@ impl<I: Ip> Peer<I> {
|
|||
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
|
||||
pub struct PeerMapKey<I: Ip> {
|
||||
pub peer_id: PeerId,
|
||||
pub ip_or_key: Either<I, SmartString<LazyCompact>>,
|
||||
pub ip: I,
|
||||
}
|
||||
|
||||
pub type PeerMap<I> = IndexMap<PeerMapKey<I>, Peer<I>>;
|
||||
|
|
@ -117,7 +116,7 @@ impl<I: Ip> TorrentData<I> {
|
|||
}
|
||||
}
|
||||
|
||||
pub type TorrentMap<I> = AmortizedIndexMap<InfoHash, TorrentData<I>>;
|
||||
pub type TorrentMap<I> = IndexMap<InfoHash, TorrentData<I>>;
|
||||
|
||||
#[derive(Default)]
|
||||
pub struct TorrentMaps {
|
||||
|
|
@ -389,14 +388,9 @@ pub fn upsert_peer_and_get_response_peers<I: Ip>(
|
|||
let peer_status =
|
||||
PeerStatus::from_event_and_bytes_left(request.event, Some(request.bytes_left));
|
||||
|
||||
let ip_or_key = request
|
||||
.key
|
||||
.map(Either::Right)
|
||||
.unwrap_or_else(|| Either::Left(peer_ip_address));
|
||||
|
||||
let peer_map_key = PeerMapKey {
|
||||
peer_id: request.peer_id,
|
||||
ip_or_key,
|
||||
ip: peer_ip_address,
|
||||
};
|
||||
|
||||
let opt_removed_peer = match peer_status {
|
||||
|
|
|
|||
|
|
@ -1,8 +1,6 @@
|
|||
use std::net::{Ipv4Addr, Ipv6Addr};
|
||||
|
||||
use aquatic_common::{
|
||||
AmortizedIndexMap, IndexMap, SecondsSinceServerStart, ServerStartInstant, ValidUntil,
|
||||
};
|
||||
use aquatic_common::{IndexMap, SecondsSinceServerStart, ServerStartInstant, ValidUntil};
|
||||
use aquatic_http_protocol::common::{AnnounceEvent, InfoHash, PeerId};
|
||||
use aquatic_http_protocol::response::ResponsePeer;
|
||||
|
||||
|
|
@ -76,7 +74,7 @@ impl<I: Ip> Default for TorrentData<I> {
|
|||
}
|
||||
}
|
||||
|
||||
pub type TorrentMap<I> = AmortizedIndexMap<InfoHash, TorrentData<I>>;
|
||||
pub type TorrentMap<I> = IndexMap<InfoHash, TorrentData<I>>;
|
||||
|
||||
#[derive(Default)]
|
||||
pub struct TorrentMaps {
|
||||
|
|
|
|||
|
|
@ -26,6 +26,7 @@ harness = false
|
|||
[dependencies]
|
||||
anyhow = "1"
|
||||
axum = { version = "0.5", optional = true, default-features = false }
|
||||
compact_str = { version = "0.7", features = ["serde"] }
|
||||
hex = { version = "0.4", default-features = false }
|
||||
httparse = "1"
|
||||
itoa = "1"
|
||||
|
|
@ -33,7 +34,6 @@ log = "0.4"
|
|||
memchr = "2"
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
serde_bencode = "0.2"
|
||||
smartstring = "1"
|
||||
urlencoding = "2"
|
||||
|
||||
[dev-dependencies]
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
use std::io::Write;
|
||||
|
||||
use anyhow::Context;
|
||||
use smartstring::{LazyCompact, SmartString};
|
||||
use compact_str::CompactString;
|
||||
|
||||
use super::common::*;
|
||||
use super::utils::*;
|
||||
|
|
@ -17,7 +17,7 @@ pub struct AnnounceRequest {
|
|||
pub event: AnnounceEvent,
|
||||
/// Number of response peers wanted
|
||||
pub numwant: Option<usize>,
|
||||
pub key: Option<SmartString<LazyCompact>>,
|
||||
pub key: Option<CompactString>,
|
||||
}
|
||||
|
||||
impl AnnounceRequest {
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ use aquatic_common::SecondsSinceServerStart;
|
|||
use aquatic_common::ServerStartInstant;
|
||||
use aquatic_common::{
|
||||
access_list::{create_access_list_cache, AccessListArcSwap, AccessListCache, AccessListMode},
|
||||
extract_response_peers, AmortizedIndexMap, ValidUntil,
|
||||
extract_response_peers, ValidUntil,
|
||||
};
|
||||
|
||||
use aquatic_udp_protocol::*;
|
||||
|
|
@ -144,7 +144,7 @@ impl<I: Ip> Default for TorrentData<I> {
|
|||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub struct TorrentMap<I: Ip>(pub AmortizedIndexMap<InfoHash, TorrentData<I>>);
|
||||
pub struct TorrentMap<I: Ip>(pub IndexMap<InfoHash, TorrentData<I>>);
|
||||
|
||||
impl<I: Ip> TorrentMap<I> {
|
||||
/// Remove forbidden or inactive torrents, reclaim space and return number of remaining peers
|
||||
|
|
|
|||
|
|
@ -13,8 +13,7 @@ use hashbrown::HashMap;
|
|||
use rand::{rngs::SmallRng, SeedableRng};
|
||||
|
||||
use aquatic_common::{
|
||||
extract_response_peers, AmortizedIndexMap, IndexMap, PanicSentinel, SecondsSinceServerStart,
|
||||
ServerStartInstant,
|
||||
extract_response_peers, IndexMap, PanicSentinel, SecondsSinceServerStart, ServerStartInstant,
|
||||
};
|
||||
use aquatic_ws_protocol::*;
|
||||
|
||||
|
|
@ -87,7 +86,7 @@ impl TorrentData {
|
|||
}
|
||||
}
|
||||
|
||||
type TorrentMap = AmortizedIndexMap<InfoHash, TorrentData>;
|
||||
type TorrentMap = IndexMap<InfoHash, TorrentData>;
|
||||
|
||||
#[derive(Default)]
|
||||
struct TorrentMaps {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue