Merge pull request #146 from greatest-ape/work-2023-08-28

update some dependencies, improve udp invalid connection id integration test, clean up
This commit is contained in:
Joakim Frostegård 2023-08-29 00:05:11 +02:00 committed by GitHub
commit 418b70b0fd
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
11 changed files with 77 additions and 232 deletions

59
Cargo.lock generated
View file

@ -227,7 +227,7 @@ dependencies = [
"blake3",
"cfg-if",
"compact_str",
"constant_time_eq 0.2.6",
"constant_time_eq",
"crossbeam-channel",
"getrandom",
"hashbrown 0.14.0",
@ -497,7 +497,7 @@ dependencies = [
"arrayvec",
"cc",
"cfg-if",
"constant_time_eq 0.3.0",
"constant_time_eq",
"digest",
]
@ -611,18 +611,18 @@ dependencies = [
[[package]]
name = "clap"
version = "4.4.0"
version = "4.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1d5f1946157a96594eb2d2c10eb7ad9a2b27518cb3000209dec700c35df9197d"
checksum = "7c8d502cbaec4595d2e7d5f61e318f05417bd2b66fdc3809498f0d3fdf0bea27"
dependencies = [
"clap_builder",
]
[[package]]
name = "clap_builder"
version = "4.4.0"
version = "4.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "78116e32a042dd73c2901f0dc30790d20ff3447f3e3472fad359e8c3d282bcd6"
checksum = "5891c7bc0edb3e1c2204fc5e94009affabeb1821c9e5fdc3959536c5c0bb984d"
dependencies = [
"anstyle",
"clap_lex 0.5.1",
@ -690,12 +690,6 @@ dependencies = [
"windows-sys 0.45.0",
]
[[package]]
name = "constant_time_eq"
version = "0.2.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "21a53c0a4d288377e7415b53dcfc3c04da5cdc2cc95c8d5ac178b58f0b861ad6"
[[package]]
name = "constant_time_eq"
version = "0.3.0"
@ -755,7 +749,7 @@ dependencies = [
"anes",
"cast",
"ciborium",
"clap 4.4.0",
"clap 4.4.1",
"criterion-plot",
"is-terminal",
"itertools",
@ -1644,9 +1638,9 @@ dependencies = [
[[package]]
name = "memchr"
version = "2.5.0"
version = "2.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d"
checksum = "76fc44e2588d5b436dbc3c6cf62aef290f90dab6235744a93dfe1cc18f451e2c"
[[package]]
name = "memoffset"
@ -1802,16 +1796,15 @@ dependencies = [
[[package]]
name = "nix"
version = "0.26.2"
version = "0.26.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bfdda3d196821d6af13126e40375cdf7da646a96114af134d5f417a9a1dc8e1a"
checksum = "598beaf3cc6fdd9a5dfb1630c2800c7acd31df7aaf0f565796fba2b53ca1af1b"
dependencies = [
"bitflags 1.3.2",
"cfg-if",
"libc",
"memoffset 0.7.1",
"pin-utils",
"static_assertions",
]
[[package]]
@ -2046,7 +2039,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9bc12de3935536ed9b69488faea4450a298dac44179b54f71806e63f55034bf9"
dependencies = [
"libc",
"nix 0.26.2",
"nix 0.26.4",
]
[[package]]
@ -2221,9 +2214,9 @@ dependencies = [
[[package]]
name = "regex"
version = "1.9.3"
version = "1.9.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "81bc1d4caf89fac26a70747fe603c130093b53c773888797a6329091246d651a"
checksum = "12de2eff854e5fa4b1295edd650e227e9d8fb0c9e90b12e7f36d6a6811791a29"
dependencies = [
"aho-corasick",
"memchr",
@ -2233,9 +2226,9 @@ dependencies = [
[[package]]
name = "regex-automata"
version = "0.3.6"
version = "0.3.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fed1ceff11a1dddaee50c9dc8e4938bd106e9d89ae372f192311e7da498e3b69"
checksum = "49530408a136e16e5b486e883fbb6ba058e8e4e8ae6621a77b048b314336e629"
dependencies = [
"aho-corasick",
"memchr",
@ -2244,9 +2237,9 @@ dependencies = [
[[package]]
name = "regex-syntax"
version = "0.7.4"
version = "0.7.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e5ea92a5b6195c6ef2a0295ea818b312502c6fc94dde986c5553242e18fd4ce2"
checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da"
[[package]]
name = "ring"
@ -2293,9 +2286,9 @@ dependencies = [
[[package]]
name = "rustls"
version = "0.21.6"
version = "0.21.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1d1feddffcfcc0b33f5c6ce9a29e341e4cd59c3f78e7ee45f4a40c038b1d6cbb"
checksum = "cd8d6c9f025a446bc4d18ad9632e69aec8f287aa84499ee335599fabd20c3fd8"
dependencies = [
"log",
"ring",
@ -2636,9 +2629,9 @@ dependencies = [
[[package]]
name = "time"
version = "0.3.27"
version = "0.3.28"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0bb39ee79a6d8de55f48f2293a830e040392f1c5f16e336bdd1788cd0aadce07"
checksum = "17f6bb557fd245c28e6411aa56b6403c689ad95061f50e4be16c274e70a17e48"
dependencies = [
"deranged",
"itoa",
@ -2657,9 +2650,9 @@ checksum = "7300fbefb4dadc1af235a9cef3737cea692a9d97e1b9cbcd4ebdae6f8868e6fb"
[[package]]
name = "time-macros"
version = "0.2.13"
version = "0.2.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "733d258752e9303d392b94b75230d07b0b9c489350c69b851fc6c065fde3e8f9"
checksum = "1a942f44339478ef67935ab2bbaec2fb0322496cf3cbe84b261e06ac3814c572"
dependencies = [
"time-core",
]
@ -2816,9 +2809,9 @@ checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a"
[[package]]
name = "url"
version = "2.4.0"
version = "2.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "50bff7831e19200a85b17131d085c25d7811bc4e186efdaf54bbd132994a88cb"
checksum = "143b538f18257fac9cad154828a57c6bf5157e1aa604d4816b5995bf6de87ae5"
dependencies = [
"form_urlencoded",
"idna",

View file

@ -13,11 +13,11 @@ of sub-implementations for different protocols:
[mio]: https://github.com/tokio-rs/mio
[glommio]: https://github.com/DataDog/glommio
| Name | Protocol | OS requirements |
|--------------|-----------------------------------|--------------------------------------|
| aquatic_udp | [BitTorrent over UDP] | Unix-like / Linux 6.0+ with io_uring |
| aquatic_http | [BitTorrent over HTTP] over TLS | Linux 5.8+ |
| aquatic_ws | [WebTorrent], optionally over TLS | Linux 5.8+ |
| Name | Protocol | OS requirements |
|--------------|-----------------------------------|-----------------|
| aquatic_udp | [BitTorrent over UDP] | Unix-like |
| aquatic_http | [BitTorrent over HTTP] over TLS | Linux 5.8+ |
| aquatic_ws | [WebTorrent], optionally over TLS | Linux 5.8+ |
Features at a glance:
@ -192,9 +192,10 @@ This is the most mature of the implementations. I consider it ready for producti
#### io_uring
An experimental io_uring backend is available. It currently requires Linux
6.0 or later and will attempt to fall back to the [mio] backend if run with
older kernels. To enable it, pass the `io-uring` feature when compiling:
An experimental (and possibly unsound) io_uring backend is available. It
currently requires Linux 6.0 or later and will attempt to fall back to the
[mio] backend if run with older kernels. To enable it, pass the `io-uring`
feature when compiling:
```sh
cargo build --release -p aquatic_udp --features "io-uring"
@ -204,7 +205,7 @@ cargo build --release -p aquatic_udp --features "io-uring"
![UDP BitTorrent tracker throughput comparison](./documents/aquatic-udp-load-test-illustration-2023-01-11.png)
The mio backend was used. More details are available [here](./documents/aquatic-udp-load-test-2023-01-11.pdf).
The default backend was used. More details are available [here](./documents/aquatic-udp-load-test-2023-01-11.pdf).
---

26
TODO.md
View file

@ -11,20 +11,13 @@
* Non-trivial dependency updates
* toml v0.7
* syn v2.0
* simd-json v0.7
## Medium priority
* stagger cleaning tasks?
* Run cargo-fuzz on protocol crates
* udp: support link to arbitrary homepage as well as embedded tracker URL in statistics page
* Consider storing torrents in separate IndexMaps. The amount should be a power
of 2 and should be configurable. They could be stored in a Vec and the index
could be calculated by taking the first N bits of the info hash. Each such map
would also store when it was last cleaned. There would then be a small
configurable random chance that when an announce request is being processed,
the map will be cleaned. When doing the normal cleaning round, recently
cleaned maps would be skipped.
* udp: support link to arbitrary homepage as well as embedded tracker URL in statistics page
* quit whole program if any thread panics
* But it would be nice not to panic in workers, but to return errors instead.
@ -36,11 +29,6 @@
* Run cargo-deny in CI
* udp: add IP blocklist, which would be more flexible than just adding option
for disallowing requests (claiming to be) from localhost
* stagger cleaning tasks?
* aquatic_ws
* Add cleaning task for ConnectionHandle.announced_info_hashes?
* RES memory still high after traffic stops, even if torrent maps and connection slabs go down to 0 len and capacity
@ -48,13 +36,6 @@
* SinkExt::send maybe doesn't wake up properly?
* related to https://github.com/sdroege/async-tungstenite/blob/master/src/compat.rs#L18 ?
* aquatic_http_private
* Consider not setting Content-type: text/plain for responses and send vec as default octet stream instead
* stored procedure
* test ip format
* check user token length
* site will likely want num_seeders and num_leechers for all torrents..
* Performance hyperoptimization (receive interrupts on correct core)
* If there is no network card RSS support, do eBPF XDP CpuMap redirect based on packet info, to
cpus where socket workers run. Support is work in progress in the larger Rust eBPF
@ -74,9 +55,6 @@
* aquatic_ws
* large amount of temporary allocations in serialize_20_bytes, pretty many in deserialize_20_bytes
* extract response peers: extract "one extra" to compensate for removal,
of sender if present in selection?
# Not important
* aquatic_http:

View file

@ -31,7 +31,7 @@ anyhow = "1"
blake3 = "1"
cfg-if = "1"
compact_str = "0.7"
constant_time_eq = "0.2"
constant_time_eq = "0.3"
crossbeam-channel = "0.5"
getrandom = "0.2"
hashbrown = { version = "0.14", default-features = false }

View file

@ -12,11 +12,11 @@ use anyhow::Context;
use aquatic_udp::{common::BUFFER_SIZE, config::Config};
use aquatic_udp_protocol::{
common::PeerId, AnnounceEvent, AnnounceRequest, ConnectionId, InfoHash, NumberOfBytes,
NumberOfPeers, PeerKey, Port, Request, TransactionId,
NumberOfPeers, PeerKey, Port, Request, ScrapeRequest, TransactionId,
};
#[test]
fn test_announce_with_invalid_connection_id() -> anyhow::Result<()> {
fn test_invalid_connection_id() -> anyhow::Result<()> {
const TRACKER_PORT: u16 = 40_112;
let mut config = Config::default();
@ -32,29 +32,49 @@ fn test_announce_with_invalid_connection_id() -> anyhow::Result<()> {
socket.set_read_timeout(Some(Duration::from_secs(1)))?;
// Make sure that the tracker in fact responds to requests
// Send connect request to make sure that the tracker in fact responds to
// valid requests
let connection_id = connect(&socket, tracker_addr).with_context(|| "connect")?;
let invalid_connection_id = ConnectionId(!connection_id.0);
let announce_request = Request::Announce(AnnounceRequest {
connection_id: invalid_connection_id,
transaction_id: TransactionId(0),
info_hash: InfoHash([0; 20]),
peer_id: PeerId([0; 20]),
bytes_downloaded: NumberOfBytes(0),
bytes_uploaded: NumberOfBytes(0),
bytes_left: NumberOfBytes(0),
event: AnnounceEvent::Started,
ip_address: None,
key: PeerKey(0),
peers_wanted: NumberOfPeers(10),
port: Port(1),
});
let scrape_request = Request::Scrape(ScrapeRequest {
connection_id: invalid_connection_id,
transaction_id: TransactionId(0),
info_hashes: vec![InfoHash([0; 20])],
});
no_response(&socket, tracker_addr, announce_request).with_context(|| "announce")?;
no_response(&socket, tracker_addr, scrape_request).with_context(|| "scrape")?;
Ok(())
}
fn no_response(
socket: &UdpSocket,
tracker_addr: SocketAddr,
request: Request,
) -> anyhow::Result<()> {
let mut buffer = [0u8; BUFFER_SIZE];
{
let mut buffer = Cursor::new(&mut buffer[..]);
let request = Request::Announce(AnnounceRequest {
connection_id: ConnectionId(!connection_id.0),
transaction_id: TransactionId(0),
info_hash: InfoHash([0; 20]),
peer_id: PeerId([0; 20]),
bytes_downloaded: NumberOfBytes(0),
bytes_uploaded: NumberOfBytes(0),
bytes_left: NumberOfBytes(0),
event: AnnounceEvent::Started,
ip_address: None,
key: PeerKey(0),
peers_wanted: NumberOfPeers(-1),
port: Port(1),
});
request
.write(&mut buffer)
.with_context(|| "write request")?;

View file

@ -1,58 +0,0 @@
# syntax=docker/dockerfile:1
# aquatic_ws
#
# WORK IN PROGRESS: currently doesn't work due to issues with spawning worker
# threads, possibly related to https://github.com/DataDog/glommio/issues/547
#
# Customize by setting CONFIG_FILE_CONTENTS and
# ACCESS_LIST_CONTENTS environment variables.
#
# If no changes are made to configuration, aquatic_ws is run:
# - on port 3000
# - without TLS
# - with http health checks enabled
# - only allowing announces for hashes in access list, e.g., contained
# in ACCESS_LIST_CONTENTS env var
#
# Run from root directory of aquatic repository with:
# $ DOCKER_BUILDKIT=1 docker build -t aquatic-ws -f docker/aquatic_ws.Dockerfile .
# $ docker run -it --ulimit memlock=65536:65536 -p 0.0.0.0:3000:3000 --name aquatic-ws aquatic-ws
#
# Pass --network="host" to run command for much better performance.
FROM rust:latest AS builder
WORKDIR /usr/src/aquatic
COPY . .
RUN . ./scripts/env-native-cpu-without-avx-512 && cargo build --release -p aquatic_ws
FROM debian:stable-slim
ENV CONFIG_FILE_CONTENTS "\
log_level = 'info'\n\
[network]\n\
enable_http_health_checks = true\n\
[access_list]\n\
mode = 'allow'\n\
"
ENV ACCESS_LIST_CONTENTS "0f0f0f0f0f1f1f1f1f1f2f2f2f2f2f3f3f3f3f3f"
WORKDIR /root/
COPY --from=builder /usr/src/aquatic/target/release/aquatic_ws ./
# Create entry point script for setting config and access
# list file contents at runtime
COPY <<-"EOT" ./entrypoint.sh
#!/bin/bash
echo -e "$CONFIG_FILE_CONTENTS" > ./config.toml
echo -e "$ACCESS_LIST_CONTENTS" > ./access-list.txt
exec ./aquatic_ws -c ./config.toml "$@"
EOT
RUN chmod +x ./entrypoint.sh
ENTRYPOINT ["./entrypoint.sh"]

View file

@ -1,5 +0,0 @@
#!/bin/sh
. ./scripts/env-native-cpu-without-avx-512
cargo build --release --bin aquatic

View file

@ -1,8 +0,0 @@
#!/bin/bash
# Profile
perf record --call-graph=dwarf,16384 -e cpu-clock -F 997 target/release/aquatic_udp
# Generate flamegraph (make sure nginx is installed for stdout path)
# Info: https://gist.github.com/dlaehnemann/df31787c41bd50c0fe223df07cf6eb89
perf script | stackcollapse-perf.pl | c++filt | flamegraph.pl > /var/www/html/flame.svg

View file

@ -1,5 +0,0 @@
#!/bin/sh
. ./scripts/env-native-cpu-without-avx-512
cargo run --profile "release-debug" --bin aquatic -- $@

View file

@ -1,64 +0,0 @@
#!/bin/bash
# Run this to setup benchmark prerequisites on debian
set -e
echo "This script installs various dependencies for benchmarking aquatic."
echo "It is meant for be run on a Debian buster system after OS install."
read -p "Setup benchmarking prerequisites? [y/N]" -n 1 -r
if [[ ! $REPLY =~ ^[Yy]$ ]]
then
[[ "$0" = "$BASH_SOURCE" ]] && exit 1 || return 1
fi
apt-get update && apt-get upgrade -y
apt-get install -y vim screen build-essential libz-dev cvs htop curl linux-perf cmake c++filt numactl git nginx
curl https://sh.rustup.rs -sSf | sh
echo 'export RUSTFLAGS="-Ctarget-cpu=native"' >> ~/.profile
echo 'export EDITOR=vim' >> ~/.profile
echo 'net.core.rmem_max=104857600' >> /etc/sysctl.d/local.conf
echo 'net.core.rmem_default=104857600' >> /etc/sysctl.d/local.conf
sysctl -p
source ~/.profile
cd ~
mkdir -p projects
cd projects
# Download and compile opentracker and dependencies
wget https://www.fefe.de/libowfat/libowfat-0.32.tar.xz
tar -xf libowfat-0.32.tar.xz
rm libowfat-0.32.tar.xz
mv libowfat-0.32 libowfat
cd libowfat
make
cd ..
git clone git://erdgeist.org/opentracker
cd opentracker
sed -i "s/^OPTS_production=-O3/OPTS_production=-O3 -march=native -mtune=native/g" Makefile
make
cp opentracker.conf.example config
cd ..
# Download and compile aquatic
git clone https://github.com/greatest-ape/aquatic.git
cd aquatic
export RUSTFLAGS="-C target-cpu=native"
cargo build --release --bin aquatic_udp
cargo build --release --bin aquatic_udp_load_test
cd ..
# Download flamegraph stuff
git clone https://github.com/brendangregg/FlameGraph
cd FlameGraph
echo 'export PATH="$HOME/projects/FlameGraph:$PATH"' >> ~/.profile
source ~/.profile

View file

@ -1,7 +0,0 @@
#!/bin/sh
# Not chosen for exact values, only to be larger than defaults
export QUICKCHECK_TESTS=2000
export QUICKCHECK_GENERATOR_SIZE=1000
cargo test