Split README into separate files (#150)

* Create a separate udp README file

* Split README into separate files for all three implementations

* Minor README fixes

crates/http/Cargo.toml

@@ -7,9 +7,10 @@ authors.workspace = true
 edition.workspace = true
 license.workspace = true
 repository.workspace = true
-readme.workspace = true
 rust-version.workspace = true
 
+readme = "./README.md"
+
 [lib]
 name = "aquatic_http"
 

crates/http/README.md

@@ -0,0 +1,117 @@
+# aquatic_http: high-performance open HTTP BitTorrent tracker
+
+[![CI](https://github.com/greatest-ape/aquatic/actions/workflows/ci.yml/badge.svg)](https://github.com/greatest-ape/aquatic/actions/workflows/ci.yml)
+
+High-performance open HTTP BitTorrent tracker for Linux 5.8 or later.
+
+Features at a glance:
+
+- Multithreaded design for handling large amounts of traffic
+- All data is stored in-memory (no database needed)
+- IPv4 and IPv6 support
+- Supports forbidding/allowing info hashes
+- Prometheus metrics
+- Automated CI testing of full file transfers
+
+## Performance
+
+![HTTP BitTorrent tracker throughput comparison](../../documents/aquatic-http-load-test-illustration-2023-01-25.png)
+
+More benchmark details are available [here](../../documents/aquatic-http-load-test-2023-01-25.pdf).
+
+## Usage
+
+### Compiling
+
+- Install Rust with [rustup](https://rustup.rs/) (latest stable release is recommended)
+- Install cmake with your package manager (e.g., `apt-get install cmake`)
+- Clone this git repository and enter its root directory
+- Build the application:
+
+```sh
+# Recommended: tell Rust to enable support for all SIMD extensions present on
+# current CPU except for those relating to AVX-512. (If you run a processor
+# that doesn't clock down when using AVX-512, you can enable those instructions
+# too.)
+. ./scripts/env-native-cpu-without-avx-512
+
+cargo build --release -p aquatic_http
+```
+
+### Configuring
+
+Generate the configuration file:
+
+```sh
+./target/release/aquatic_http -p > "aquatic-http-config.toml"
+```
+
+Make necessary adjustments to the file. You will likely want to adjust `address`
+(listening address) under the `network` section.
+
+`aquatic_http` __only__ runs over TLS, so configuring certificate and private
+key files is required.
+
+Running behind a reverse proxy is currently not supported due to the
+[difficulties of determining the originating IP address](https://adam-p.ca/blog/2022/03/x-forwarded-for/)
+without knowing the exact setup.
+
+### Running
+
+Make sure locked memory limits are sufficient:
+- If you're using a systemd service file, add `LimitMEMLOCK=65536000` to it
+- Otherwise, add the following lines to
+`/etc/security/limits.conf`, and then log out and back in:
+
+```
+*    hard    memlock    65536
+*    soft    memlock    65536
+```
+
+Once done, start the application:
+
+```sh
+./target/release/aquatic_http -c "aquatic-http-config.toml"
+```
+
+If your server is pointed to by domain `example.com` and you configured the
+tracker to run on port 3000, people can now use it by adding the URL
+`https://example.com:3000/announce` to their torrent files or magnet links.
+
+### Load testing
+
+A load test application is available. It supports generation and loading of
+configuration files in a similar manner to the tracker application.
+
+After starting the tracker, run the load tester:
+
+```sh
+./scripts/run-load-test-http.sh
+```
+
+## Details
+
+[BEP 003]: https://www.bittorrent.org/beps/bep_0003.html
+[BEP 007]: https://www.bittorrent.org/beps/bep_0007.html
+[BEP 023]: https://www.bittorrent.org/beps/bep_0023.html
+[BEP 048]: https://www.bittorrent.org/beps/bep_0048.html
+
+Implements:
+  * [BEP 003]: HTTP BitTorrent protocol ([more details](https://wiki.theory.org/index.php/BitTorrentSpecification#Tracker_HTTP.2FHTTPS_Protocol)). Exceptions:
+    * Only runs over TLS
+    * Doesn't track the number of torrent downloads (0 is always sent)
+    * Only compact responses are supported
+  * [BEP 023]: Compact HTTP responses
+  * [BEP 007]: IPv6 support
+  * [BEP 048]: HTTP scrape support. Notes:
+    * Doesn't allow full scrapes, i.e. of all registered info hashes
+
+`aquatic_http` has not been tested as much as `aquatic_udp`, but likely works
+fine in production.
+
+## Copyright and license
+
+Copyright (c) 2020-2023 Joakim Frostegård
+
+Distributed under the terms of the Apache 2.0 license. Please refer to the
+`LICENSE` file in the repository root directory for details.

crates/http/src/config.rs

@@ -15,10 +15,14 @@ use aquatic_common::cli::LogLevel;
 #[derive(Clone, Debug, PartialEq, TomlConfig, Deserialize)]
 #[serde(default, deny_unknown_fields)]
 pub struct Config {
+    /// Number of socket worker. One per physical core is recommended.
+    ///
     /// Socket workers receive requests from the socket, parse them and send
     /// them on to the swarm workers. They then receive responses from the
     /// swarm workers, encode them and send them back over the socket.
     pub socket_workers: usize,
+    /// Number of swarm workers. One is enough in almost all cases
+    ///
     /// Swarm workers receive a number of requests from socket workers,
     /// generate responses and send them back to the socket workers.
     pub swarm_workers: usize,
@@ -27,6 +31,12 @@ pub struct Config {
     pub protocol: ProtocolConfig,
     pub cleaning: CleaningConfig,
     pub privileges: PrivilegeConfig,
+    /// Access list configuration
+    /// 
+    /// The file is read on start and when the program receives `SIGUSR1`. If
+    /// initial parsing fails, the program exits. Later failures result in in
+    /// emitting of an error-level log message, while successful updates of the
+    /// access list result in emitting of an info-level log message.
     pub access_list: AccessListConfig,
     pub cpu_pinning: CpuPinningConfigAsc,
     #[cfg(feature = "metrics")]

crates/udp/Cargo.toml

@@ -7,9 +7,10 @@ authors.workspace = true
 edition.workspace = true
 license.workspace = true
 repository.workspace = true
-readme.workspace = true
 rust-version.workspace = true
 
+readme = "./README.md"
+
 [lib]
 name = "aquatic_udp"
 

crates/udp/README.md

@@ -0,0 +1,91 @@
+# aquatic_udp: high-performance open UDP BitTorrent tracker
+
+[![CI](https://github.com/greatest-ape/aquatic/actions/workflows/ci.yml/badge.svg)](https://github.com/greatest-ape/aquatic/actions/workflows/ci.yml)
+
+High-performance open UDP BitTorrent tracker for Unix-like operating systems.
+
+Features at a glance:
+
+- Multithreaded design for handling large amounts of traffic
+- All data is stored in-memory (no database needed)
+- IPv4 and IPv6 support
+- Supports forbidding/allowing info hashes
+- Prometheus metrics
+- Automated CI testing of full file transfers
+
+Known users:
+
+- [explodie.org public tracker](https://explodie.org/opentracker.html) (`udp://explodie.org:6969`), typically [serving ~100,000 requests per second](https://explodie.org/tracker-stats.html)
+
+This is the most mature implementation in the aquatic family. I consider it fully ready for production use.
+
+## Performance
+
+![UDP BitTorrent tracker throughput comparison](../../documents/aquatic-udp-load-test-illustration-2023-01-11.png)
+
+More benchmark details are available [here](../../documents/aquatic-udp-load-test-2023-01-11.pdf).
+
+## Usage
+
+### Compiling
+
+- Install Rust with [rustup](https://rustup.rs/) (latest stable release is recommended)
+- Install cmake with your package manager (e.g., `apt-get install cmake`)
+- Clone this git repository and enter its root directory
+- Build the application:
+
+```sh
+# Recommended: tell Rust to enable support for all SIMD extensions present on
+# current CPU except for those relating to AVX-512. (If you run a processor
+# that doesn't clock down when using AVX-512, you can enable those instructions
+# too.)
+. ./scripts/env-native-cpu-without-avx-512
+
+cargo build --release -p aquatic_udp
+```
+
+### Configuring and running
+
+Generate the configuration file:
+
+```sh
+./target/release/aquatic_udp -p > "aquatic-udp-config.toml"
+```
+
+Make necessary adjustments to the file. You will likely want to adjust `address`
+(listening address) under the `network` section.
+
+Once done, start the application:
+
+```sh
+./target/release/aquatic_udp -c "aquatic-udp-config.toml"
+```
+
+If your server is pointed to by domain `example.com` and you configured the
+tracker to run on port 3000, people can now use it by adding the URL
+`udp://example.com:3000` to their torrent files or magnet links.
+
+### Load testing
+
+A load test application is available. It supports generation and loading of
+configuration files in a similar manner to the tracker application.
+
+After starting the tracker, run the load tester:
+
+```sh
+./scripts/run-load-test-udp.sh
+```
+
+## Details
+
+Implements [BEP 015](https://www.bittorrent.org/beps/bep_0015.html) ([more details](https://libtorrent.org/udp_tracker_protocol.html)) with the following exceptions:
+
+- Ignores IP addresses sent in announce requests. The packet source IP is always used.
+- Doesn't track the number of torrent downloads (0 is always sent). 
+
+## Copyright and license
+
+Copyright (c) 2020-2023 Joakim Frostegård
+
+Distributed under the terms of the Apache 2.0 license. Please refer to the
+`LICENSE` file in the repository root directory for details.

crates/udp/src/config.rs

@@ -11,7 +11,7 @@ use aquatic_toml_config::TomlConfig;
 #[derive(Clone, Debug, PartialEq, TomlConfig, Deserialize)]
 #[serde(default, deny_unknown_fields)]
 pub struct Config {
-    /// Number of socket workers. Increase with core count
+    /// Number of socket worker. One per physical core is recommended.
     ///
     /// Socket workers receive requests from clients and parse them.
     /// Responses to connect requests are sent back immediately. Announce and
@@ -41,6 +41,13 @@ pub struct Config {
     pub statistics: StatisticsConfig,
     pub cleaning: CleaningConfig,
     pub privileges: PrivilegeConfig,
+
+    /// Access list configuration
+    /// 
+    /// The file is read on start and when the program receives `SIGUSR1`. If
+    /// initial parsing fails, the program exits. Later failures result in in
+    /// emitting of an error-level log message, while successful updates of the
+    /// access list result in emitting of an info-level log message.
     pub access_list: AccessListConfig,
     #[cfg(feature = "cpu-pinning")]
     pub cpu_pinning: aquatic_common::cpu_pinning::asc::CpuPinningConfigAsc,

crates/ws/Cargo.toml

@@ -7,7 +7,8 @@ authors.workspace = true
 edition.workspace = true
 license.workspace = true
 repository.workspace = true
-readme.workspace = true
+
+readme = "./README.md"
 rust-version = "1.70"
 
 [lib]

crates/ws/README.md

@@ -0,0 +1,105 @@
+# aquatic_ws: high-performance open WebTorrent tracker
+
+[![CI](https://github.com/greatest-ape/aquatic/actions/workflows/ci.yml/badge.svg)](https://github.com/greatest-ape/aquatic/actions/workflows/ci.yml)
+
+High-performance WebTorrent tracker for Linux 5.8 or later.
+
+Features at a glance:
+
+- Multithreaded design for handling large amounts of traffic
+- All data is stored in-memory (no database needed)
+- IPv4 and IPv6 support
+- Supports forbidding/allowing info hashes
+- Prometheus metrics
+- Automated CI testing of full file transfers
+
+## Performance
+
+![WebTorrent tracker throughput comparison](../../documents/aquatic-ws-load-test-illustration-2023-01-25.png)
+
+More details are available [here](../../documents/aquatic-ws-load-test-2023-01-25.pdf).
+
+## Usage
+
+### Compiling
+
+- Install Rust with [rustup](https://rustup.rs/) (latest stable release is recommended)
+- Install cmake with your package manager (e.g., `apt-get install cmake`)
+- Clone this git repository and enter its root directory
+- Build the application:
+
+```sh
+# Recommended: tell Rust to enable support for all SIMD extensions present on
+# current CPU except for those relating to AVX-512. (If you run a processor
+# that doesn't clock down when using AVX-512, you can enable those instructions
+# too.)
+. ./scripts/env-native-cpu-without-avx-512
+
+cargo build --release -p aquatic_ws
+```
+
+### Configuring
+
+Generate the configuration file:
+
+```sh
+./target/release/aquatic_ws -p > "aquatic-ws-config.toml"
+```
+
+Make necessary adjustments to the file. You will likely want to adjust `address`
+(listening address) under the `network` section.
+
+Running behind a reverse proxy is supported, as long as IPv4 requests are
+proxied to IPv4 requests, and IPv6 requests to IPv6 requests.
+
+### Running
+
+Make sure locked memory limits are sufficient:
+- If you're using a systemd service file, add `LimitMEMLOCK=65536000` to it
+- Otherwise, add the following lines to
+`/etc/security/limits.conf`, and then log out and back in:
+
+```
+*    hard    memlock    65536
+*    soft    memlock    65536
+```
+
+Once done, start the application:
+
+```sh
+./target/release/aquatic_ws -c "aquatic-ws-config.toml"
+```
+
+If your server is pointed to by domain `example.com` and you configured the
+tracker to run on port 3000, people can now use it by adding the URL
+`wss://example.com:3000` to their torrent files or magnet links.
+
+### Load testing
+
+A load test application is available. It supports generation and loading of
+configuration files in a similar manner to the tracker application.
+
+After starting the tracker, run the load tester:
+
+```sh
+./scripts/run-load-test-ws.sh
+```
+
+## Details
+
+Aims for compatibility with [WebTorrent](https://github.com/webtorrent)
+clients. Notes:
+
+  * Doesn't track the number of torrent downloads (0 is always sent). 
+  * Doesn't allow full scrapes, i.e. of all registered info hashes
+
+`aquatic_ws` has not been tested as much as `aquatic_udp`, but likely works
+fine in production.
+
+## Copyright and license
+
+Copyright (c) 2020-2023 Joakim Frostegård
+
+Distributed under the terms of the Apache 2.0 license. Please refer to the
+`LICENSE` file in the repository root directory for details.
+

crates/ws/src/config.rs

@@ -15,10 +15,21 @@ use aquatic_toml_config::TomlConfig;
 #[derive(Clone, Debug, PartialEq, TomlConfig, Deserialize)]
 #[serde(default, deny_unknown_fields)]
 pub struct Config {
+    /// Number of socket workers.
+    /// 
+    /// On servers with 1-7 physical cores, using a worker per core is
+    /// recommended. With more cores, using two workers less than the
+    /// number of cores is recommended.
+    ///
     /// Socket workers receive requests from the socket, parse them and send
     /// them on to the swarm workers. They then receive responses from the
     /// swarm workers, encode them and send them back over the socket.
     pub socket_workers: usize,
+    /// Number of swarm workers.
+    /// 
+    /// A single worker is recommended for servers with 1-7 physical cores.
+    /// With more cores, using two workers is recommended.
+    ///
     /// Swarm workers receive a number of requests from socket workers,
     /// generate responses and send them back to the socket workers.
     pub swarm_workers: usize,
@@ -27,6 +38,12 @@ pub struct Config {
     pub protocol: ProtocolConfig,
     pub cleaning: CleaningConfig,
     pub privileges: PrivilegeConfig,
+    /// Access list configuration
+    /// 
+    /// The file is read on start and when the program receives `SIGUSR1`. If
+    /// initial parsing fails, the program exits. Later failures result in in
+    /// emitting of an error-level log message, while successful updates of the
+    /// access list result in emitting of an info-level log message.
     pub access_list: AccessListConfig,
     #[cfg(feature = "metrics")]
     pub metrics: MetricsConfig,