Merge pull request #146 from greatest-ape/work-2023-08-28

update some dependencies, improve udp invalid connection id integration test, clean up

Cargo.lock

@@ -227,7 +227,7 @@ dependencies = [
  "blake3",
  "cfg-if",
  "compact_str",
- "constant_time_eq 0.2.6",
+ "constant_time_eq",
  "crossbeam-channel",
  "getrandom",
  "hashbrown 0.14.0",
@@ -497,7 +497,7 @@ dependencies = [
  "arrayvec",
  "cc",
  "cfg-if",
- "constant_time_eq 0.3.0",
+ "constant_time_eq",
  "digest",
 ]
 
@@ -611,18 +611,18 @@ dependencies = [
 
 [[package]]
 name = "clap"
-version = "4.4.0"
+version = "4.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1d5f1946157a96594eb2d2c10eb7ad9a2b27518cb3000209dec700c35df9197d"
+checksum = "7c8d502cbaec4595d2e7d5f61e318f05417bd2b66fdc3809498f0d3fdf0bea27"
 dependencies = [
  "clap_builder",
 ]
 
 [[package]]
 name = "clap_builder"
-version = "4.4.0"
+version = "4.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "78116e32a042dd73c2901f0dc30790d20ff3447f3e3472fad359e8c3d282bcd6"
+checksum = "5891c7bc0edb3e1c2204fc5e94009affabeb1821c9e5fdc3959536c5c0bb984d"
 dependencies = [
  "anstyle",
  "clap_lex 0.5.1",
@@ -690,12 +690,6 @@ dependencies = [
  "windows-sys 0.45.0",
 ]
 
-[[package]]
-name = "constant_time_eq"
-version = "0.2.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "21a53c0a4d288377e7415b53dcfc3c04da5cdc2cc95c8d5ac178b58f0b861ad6"
-
 [[package]]
 name = "constant_time_eq"
 version = "0.3.0"
@@ -755,7 +749,7 @@ dependencies = [
  "anes",
  "cast",
  "ciborium",
- "clap 4.4.0",
+ "clap 4.4.1",
  "criterion-plot",
  "is-terminal",
  "itertools",
@@ -1644,9 +1638,9 @@ dependencies = [
 
 [[package]]
 name = "memchr"
-version = "2.5.0"
+version = "2.6.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d"
+checksum = "76fc44e2588d5b436dbc3c6cf62aef290f90dab6235744a93dfe1cc18f451e2c"
 
 [[package]]
 name = "memoffset"
@@ -1802,16 +1796,15 @@ dependencies = [
 
 [[package]]
 name = "nix"
-version = "0.26.2"
+version = "0.26.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bfdda3d196821d6af13126e40375cdf7da646a96114af134d5f417a9a1dc8e1a"
+checksum = "598beaf3cc6fdd9a5dfb1630c2800c7acd31df7aaf0f565796fba2b53ca1af1b"
 dependencies = [
  "bitflags 1.3.2",
  "cfg-if",
  "libc",
  "memoffset 0.7.1",
  "pin-utils",
- "static_assertions",
 ]
 
 [[package]]
@@ -2046,7 +2039,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9bc12de3935536ed9b69488faea4450a298dac44179b54f71806e63f55034bf9"
 dependencies = [
  "libc",
- "nix 0.26.2",
+ "nix 0.26.4",
 ]
 
 [[package]]
@@ -2221,9 +2214,9 @@ dependencies = [
 
 [[package]]
 name = "regex"
-version = "1.9.3"
+version = "1.9.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "81bc1d4caf89fac26a70747fe603c130093b53c773888797a6329091246d651a"
+checksum = "12de2eff854e5fa4b1295edd650e227e9d8fb0c9e90b12e7f36d6a6811791a29"
 dependencies = [
  "aho-corasick",
  "memchr",
@@ -2233,9 +2226,9 @@ dependencies = [
 
 [[package]]
 name = "regex-automata"
-version = "0.3.6"
+version = "0.3.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fed1ceff11a1dddaee50c9dc8e4938bd106e9d89ae372f192311e7da498e3b69"
+checksum = "49530408a136e16e5b486e883fbb6ba058e8e4e8ae6621a77b048b314336e629"
 dependencies = [
  "aho-corasick",
  "memchr",
@@ -2244,9 +2237,9 @@ dependencies = [
 
 [[package]]
 name = "regex-syntax"
-version = "0.7.4"
+version = "0.7.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e5ea92a5b6195c6ef2a0295ea818b312502c6fc94dde986c5553242e18fd4ce2"
+checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da"
 
 [[package]]
 name = "ring"
@@ -2293,9 +2286,9 @@ dependencies = [
 
 [[package]]
 name = "rustls"
-version = "0.21.6"
+version = "0.21.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1d1feddffcfcc0b33f5c6ce9a29e341e4cd59c3f78e7ee45f4a40c038b1d6cbb"
+checksum = "cd8d6c9f025a446bc4d18ad9632e69aec8f287aa84499ee335599fabd20c3fd8"
 dependencies = [
  "log",
  "ring",
@@ -2636,9 +2629,9 @@ dependencies = [
 
 [[package]]
 name = "time"
-version = "0.3.27"
+version = "0.3.28"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0bb39ee79a6d8de55f48f2293a830e040392f1c5f16e336bdd1788cd0aadce07"
+checksum = "17f6bb557fd245c28e6411aa56b6403c689ad95061f50e4be16c274e70a17e48"
 dependencies = [
  "deranged",
  "itoa",
@@ -2657,9 +2650,9 @@ checksum = "7300fbefb4dadc1af235a9cef3737cea692a9d97e1b9cbcd4ebdae6f8868e6fb"
 
 [[package]]
 name = "time-macros"
-version = "0.2.13"
+version = "0.2.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "733d258752e9303d392b94b75230d07b0b9c489350c69b851fc6c065fde3e8f9"
+checksum = "1a942f44339478ef67935ab2bbaec2fb0322496cf3cbe84b261e06ac3814c572"
 dependencies = [
  "time-core",
 ]
@@ -2816,9 +2809,9 @@ checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a"
 
 [[package]]
 name = "url"
-version = "2.4.0"
+version = "2.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "50bff7831e19200a85b17131d085c25d7811bc4e186efdaf54bbd132994a88cb"
+checksum = "143b538f18257fac9cad154828a57c6bf5157e1aa604d4816b5995bf6de87ae5"
 dependencies = [
  "form_urlencoded",
  "idna",

README.md

@@ -14,8 +14,8 @@ of sub-implementations for different protocols:
 [glommio]: https://github.com/DataDog/glommio
 
 | Name         | Protocol                          | OS requirements |
-|--------------|-----------------------------------|--------------------------------------|
+|--------------|-----------------------------------|-----------------|
-| aquatic_udp  | [BitTorrent over UDP]             | Unix-like / Linux 6.0+ with io_uring |
+| aquatic_udp  | [BitTorrent over UDP]             | Unix-like       |
 | aquatic_http | [BitTorrent over HTTP] over TLS   | Linux 5.8+      |
 | aquatic_ws   | [WebTorrent], optionally over TLS | Linux 5.8+      |
 
@@ -192,9 +192,10 @@ This is the most mature of the implementations. I consider it ready for producti
 
 #### io_uring
 
-An experimental io_uring backend is available. It currently requires Linux
+An experimental (and possibly unsound) io_uring backend is available. It
-6.0 or later and will attempt to fall back to the [mio] backend if run with
+currently requires Linux 6.0 or later and will attempt to fall back to the
-older kernels. To enable it, pass the `io-uring` feature when compiling:
+[mio] backend if run with older kernels. To enable it, pass the `io-uring`
+feature when compiling:
 
 ```sh
 cargo build --release -p aquatic_udp --features "io-uring"
@@ -204,7 +205,7 @@ cargo build --release -p aquatic_udp --features "io-uring"
 
 ![UDP BitTorrent tracker throughput comparison](./documents/aquatic-udp-load-test-illustration-2023-01-11.png)
 
-The mio backend was used. More details are available [here](./documents/aquatic-udp-load-test-2023-01-11.pdf).
+The default backend was used. More details are available [here](./documents/aquatic-udp-load-test-2023-01-11.pdf).
 
 ---
 

TODO.md

@@ -11,20 +11,13 @@
 * Non-trivial dependency updates
   * toml v0.7
   * syn v2.0
-  * simd-json v0.7
 
 ## Medium priority
 
+* stagger cleaning tasks?
 * Run cargo-fuzz on protocol crates
-* udp: support link to arbitrary homepage as well as embedded tracker URL in statistics page
 
-* Consider storing torrents in separate IndexMaps. The amount should be a power
+* udp: support link to arbitrary homepage as well as embedded tracker URL in statistics page
-  of 2 and should be configurable. They could be stored in a Vec and the index
-  could be calculated by taking the first N bits of the info hash. Each such map
-  would also store when it was last cleaned. There would then be a small
-  configurable random chance that when an announce request is being processed,
-  the map will be cleaned. When doing the normal cleaning round, recently
-  cleaned maps would be skipped.
 
 * quit whole program if any thread panics
   * But it would be nice not to panic in workers, but to return errors instead.
@@ -36,11 +29,6 @@
 
 * Run cargo-deny in CI
 
-* udp: add IP blocklist, which would be more flexible than just adding option
-  for disallowing requests (claiming to be) from localhost
-
-* stagger cleaning tasks?
-
 * aquatic_ws
   * Add cleaning task for ConnectionHandle.announced_info_hashes?
   * RES memory still high after traffic stops, even if torrent maps and connection slabs go down to 0 len and capacity
@@ -48,13 +36,6 @@
   * SinkExt::send maybe doesn't wake up properly?
     * related to https://github.com/sdroege/async-tungstenite/blob/master/src/compat.rs#L18 ?
 
-* aquatic_http_private
-  * Consider not setting Content-type: text/plain for responses and send vec as default octet stream instead
-  * stored procedure
-    * test ip format
-    * check user token length
-  * site will likely want num_seeders and num_leechers for all torrents..
-
 * Performance hyperoptimization (receive interrupts on correct core)
   * If there is no network card RSS support, do eBPF XDP CpuMap redirect based on packet info, to
     cpus where socket workers run. Support is work in progress in the larger Rust eBPF
@@ -74,9 +55,6 @@
 * aquatic_ws
   * large amount of temporary allocations in serialize_20_bytes, pretty many in deserialize_20_bytes
 
-* extract response peers: extract "one extra" to compensate for removal,
-  of sender if present in selection?
-
 # Not important
 
 * aquatic_http:

aquatic_udp/Cargo.toml

@@ -31,7 +31,7 @@ anyhow = "1"
 blake3 = "1"
 cfg-if = "1"
 compact_str = "0.7"
-constant_time_eq = "0.2"
+constant_time_eq = "0.3"
 crossbeam-channel = "0.5"
 getrandom = "0.2"
 hashbrown = { version = "0.14", default-features = false }

aquatic_udp/tests/invalid_connection_id.rs

@@ -12,11 +12,11 @@ use anyhow::Context;
 use aquatic_udp::{common::BUFFER_SIZE, config::Config};
 use aquatic_udp_protocol::{
     common::PeerId, AnnounceEvent, AnnounceRequest, ConnectionId, InfoHash, NumberOfBytes,
-    NumberOfPeers, PeerKey, Port, Request, TransactionId,
+    NumberOfPeers, PeerKey, Port, Request, ScrapeRequest, TransactionId,
 };
 
 #[test]
-fn test_announce_with_invalid_connection_id() -> anyhow::Result<()> {
+fn test_invalid_connection_id() -> anyhow::Result<()> {
     const TRACKER_PORT: u16 = 40_112;
 
     let mut config = Config::default();
@@ -32,16 +32,14 @@ fn test_announce_with_invalid_connection_id() -> anyhow::Result<()> {
 
     socket.set_read_timeout(Some(Duration::from_secs(1)))?;
 
-    // Make sure that the tracker in fact responds to requests
+    // Send connect request to make sure that the tracker in fact responds to
+    // valid requests
     let connection_id = connect(&socket, tracker_addr).with_context(|| "connect")?;
 
-    let mut buffer = [0u8; BUFFER_SIZE];
+    let invalid_connection_id = ConnectionId(!connection_id.0);
 
-    {
+    let announce_request = Request::Announce(AnnounceRequest {
-        let mut buffer = Cursor::new(&mut buffer[..]);
+        connection_id: invalid_connection_id,
-
-        let request = Request::Announce(AnnounceRequest {
-            connection_id: ConnectionId(!connection_id.0),
         transaction_id: TransactionId(0),
         info_hash: InfoHash([0; 20]),
         peer_id: PeerId([0; 20]),
@@ -51,10 +49,32 @@ fn test_announce_with_invalid_connection_id() -> anyhow::Result<()> {
         event: AnnounceEvent::Started,
         ip_address: None,
         key: PeerKey(0),
-            peers_wanted: NumberOfPeers(-1),
+        peers_wanted: NumberOfPeers(10),
         port: Port(1),
     });
 
+    let scrape_request = Request::Scrape(ScrapeRequest {
+        connection_id: invalid_connection_id,
+        transaction_id: TransactionId(0),
+        info_hashes: vec![InfoHash([0; 20])],
+    });
+
+    no_response(&socket, tracker_addr, announce_request).with_context(|| "announce")?;
+    no_response(&socket, tracker_addr, scrape_request).with_context(|| "scrape")?;
+
+    Ok(())
+}
+
+fn no_response(
+    socket: &UdpSocket,
+    tracker_addr: SocketAddr,
+    request: Request,
+) -> anyhow::Result<()> {
+    let mut buffer = [0u8; BUFFER_SIZE];
+
+    {
+        let mut buffer = Cursor::new(&mut buffer[..]);
+
         request
             .write(&mut buffer)
             .with_context(|| "write request")?;

docker/aquatic_ws.Dockerfile

@@ -1,58 +0,0 @@
-# syntax=docker/dockerfile:1
-
-# aquatic_ws
-#
-# WORK IN PROGRESS: currently doesn't work due to issues with spawning worker
-# threads, possibly related to https://github.com/DataDog/glommio/issues/547
-#
-# Customize by setting CONFIG_FILE_CONTENTS and
-# ACCESS_LIST_CONTENTS environment variables.
-#
-# If no changes are made to configuration, aquatic_ws is run:
-# - on port 3000
-# - without TLS
-# - with http health checks enabled
-# - only allowing announces for hashes in access list, e.g., contained
-#   in ACCESS_LIST_CONTENTS env var
-#
-# Run from root directory of aquatic repository with:
-# $ DOCKER_BUILDKIT=1 docker build -t aquatic-ws -f docker/aquatic_ws.Dockerfile .
-# $ docker run -it --ulimit memlock=65536:65536 -p 0.0.0.0:3000:3000 --name aquatic-ws aquatic-ws
-#
-# Pass --network="host" to run command for much better performance.
-
-FROM rust:latest AS builder
-
-WORKDIR /usr/src/aquatic
-
-COPY . .
-
-RUN . ./scripts/env-native-cpu-without-avx-512 && cargo build --release -p aquatic_ws
-
-FROM debian:stable-slim
-
-ENV CONFIG_FILE_CONTENTS "\
-    log_level = 'info'\n\
-    [network]\n\
-    enable_http_health_checks = true\n\
-    [access_list]\n\
-    mode = 'allow'\n\
-    "
-ENV ACCESS_LIST_CONTENTS "0f0f0f0f0f1f1f1f1f1f2f2f2f2f2f3f3f3f3f3f"
-
-WORKDIR /root/
-
-COPY --from=builder /usr/src/aquatic/target/release/aquatic_ws ./
-
-# Create entry point script for setting config and access
-# list file contents at runtime
-COPY <<-"EOT" ./entrypoint.sh
-#!/bin/bash
-echo -e "$CONFIG_FILE_CONTENTS" > ./config.toml
-echo -e "$ACCESS_LIST_CONTENTS" > ./access-list.txt
-exec ./aquatic_ws -c ./config.toml "$@"
-EOT
-
-RUN chmod +x ./entrypoint.sh
-
-ENTRYPOINT ["./entrypoint.sh"]

scripts/build-aquatic.sh

@@ -1,5 +0,0 @@
-#!/bin/sh 
-
-. ./scripts/env-native-cpu-without-avx-512
-
-cargo build --release --bin aquatic

scripts/flamegraph-aquatic-udp-on-linux.sh

@@ -1,8 +0,0 @@
-#!/bin/bash
-
-# Profile
-perf record --call-graph=dwarf,16384 -e cpu-clock -F 997 target/release/aquatic_udp
-
-# Generate flamegraph (make sure nginx is installed for stdout path)
-# Info: https://gist.github.com/dlaehnemann/df31787c41bd50c0fe223df07cf6eb89
-perf script | stackcollapse-perf.pl | c++filt | flamegraph.pl > /var/www/html/flame.svg

scripts/run-aquatic.sh

@@ -1,5 +0,0 @@
-#!/bin/sh 
-
-. ./scripts/env-native-cpu-without-avx-512
-
-cargo run --profile "release-debug" --bin aquatic -- $@

scripts/setup-bench-on-debian.sh

@@ -1,64 +0,0 @@
-#!/bin/bash
-# Run this to setup benchmark prerequisites on debian
-
-set -e
-
-echo "This script installs various dependencies for benchmarking aquatic."
-echo "It is meant for be run on a Debian buster system after OS install."
-
-read -p "Setup benchmarking prerequisites? [y/N]" -n 1 -r
-if [[ ! $REPLY =~ ^[Yy]$ ]]
-then
-    [[ "$0" = "$BASH_SOURCE" ]] && exit 1 || return 1
-fi
-
-apt-get update && apt-get upgrade -y
-apt-get install -y vim screen build-essential libz-dev cvs htop curl linux-perf cmake c++filt numactl git nginx
-
-curl https://sh.rustup.rs -sSf | sh
-
-echo 'export RUSTFLAGS="-Ctarget-cpu=native"' >> ~/.profile
-echo 'export EDITOR=vim' >> ~/.profile
-echo 'net.core.rmem_max=104857600' >> /etc/sysctl.d/local.conf
-echo 'net.core.rmem_default=104857600' >> /etc/sysctl.d/local.conf
-
-sysctl -p
-
-source ~/.profile
-
-cd ~
-mkdir -p projects
-cd projects
-
-# Download and compile opentracker and dependencies
-
-wget https://www.fefe.de/libowfat/libowfat-0.32.tar.xz
-tar -xf libowfat-0.32.tar.xz
-rm libowfat-0.32.tar.xz
-mv libowfat-0.32 libowfat
-cd libowfat
-make
-cd ..
-
-git clone git://erdgeist.org/opentracker
-cd opentracker
-sed -i "s/^OPTS_production=-O3/OPTS_production=-O3 -march=native -mtune=native/g" Makefile
-make
-cp opentracker.conf.example config
-cd ..
-
-# Download and compile aquatic
-
-git clone https://github.com/greatest-ape/aquatic.git
-cd aquatic
-export RUSTFLAGS="-C target-cpu=native"
-cargo build --release --bin aquatic_udp
-cargo build --release --bin aquatic_udp_load_test
-cd ..
-
-# Download flamegraph stuff
-
-git clone https://github.com/brendangregg/FlameGraph
-cd FlameGraph
-echo 'export PATH="$HOME/projects/FlameGraph:$PATH"' >> ~/.profile
-source ~/.profile

scripts/test.sh

@@ -1,7 +0,0 @@
-#!/bin/sh
-
-# Not chosen for exact values, only to be larger than defaults
-export QUICKCHECK_TESTS=2000
-export QUICKCHECK_GENERATOR_SIZE=1000
-
-cargo test