YGGverse/aquatic
1
0
Fork 0
mirror of https://github.com/YGGverse/aquatic.git synced 2026-04-02 10:45:30 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #173 from greatest-ape/work-2024-01-20

Browse source 
udp mem use and stability improvements; update deps; fix clippy warnings
This commit is contained in:
Joakim Frostegård 2024-01-20 11:18:42 +01:00 • committed by GitHub
commit 0286d25346
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
49 changed files with 822 additions and 919 deletions
Download patch file Download diff file Expand all files Collapse all files

17
CHANGELOG.md
View file

@ -15,6 +15,10 @@
#### Added #### Added
* Add support for reporting peer client information * Add support for reporting peer client information
* Speed up parsing and serialization of requests and responses with
[zerocopy](https://crates.io/crates/zerocopy)
* Store torrents with up to two peers without an extra heap allocation for the
peers.
#### Changed #### Changed
@ -24,22 +28,18 @@
* Remove support for unbounded worker channels * Remove support for unbounded worker channels
* Add backpressure in socket workers. They will postpone reading from the * Add backpressure in socket workers. They will postpone reading from the
socket if sending a request to a swarm worker failed socket if sending a request to a swarm worker failed
* Reuse allocations in swarm response channel
* Remove config key `network.poll_event_capacity`
* Harden ConnectionValidator to make IP spoofing even more costly
* Distribute announce responses from swarm workers over socket workers to * Distribute announce responses from swarm workers over socket workers to
decrease performance loss due to underutilized threads decrease performance loss due to underutilized threads
* Harden ConnectionValidator to make IP spoofing even more costly
* Remove config key `network.poll_event_capacity` (always use 1)
### aquatic_http ### aquatic_http
#### Added #### Added
* Reload TLS certificate (and key) on SIGUSR1 * Reload TLS certificate (and key) on SIGUSR1
* Support running without TLS
#### Changed * Support running behind reverse proxy
* Allow running without TLS
* Allow running behind reverse proxy
#### Fixed #### Fixed
@ -64,6 +64,7 @@
#### Fixed #### Fixed
* Fix memory leak
* Fix bug where clean up after closing connections wasn't always done * Fix bug where clean up after closing connections wasn't always done
* Fix double counting of error responses * Fix double counting of error responses
* Actually close connections that are too slow to send responses to * Actually close connections that are too slow to send responses to

205
Cargo.lock generated
View file

@ -64,9 +64,9 @@ checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299"
[[package]] [[package]]
name = "anstream" name = "anstream"
version = "0.6.5" version = "0.6.11"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d664a92ecae85fd0a7392615844904654d1d5f5514837f471ddef4a057aba1b6" checksum = "6e2e1ebcb11de5c03c67de28a7df593d32191b44939c482e97702baaaa6ab6a5"
dependencies = [ dependencies = [
"anstyle", "anstyle",
"anstyle-parse", "anstyle-parse",
@ -134,7 +134,7 @@ dependencies = [
"anyhow", "anyhow",
"aquatic_udp", "aquatic_udp",
"aquatic_udp_load_test", "aquatic_udp_load_test",
"clap 4.4.13", "clap 4.4.18",
"humanize-bytes", "humanize-bytes",
"indexmap 2.1.0", "indexmap 2.1.0",
"indoc", "indoc",
@ -293,6 +293,7 @@ dependencies = [
"aquatic_common", "aquatic_common",
"aquatic_toml_config", "aquatic_toml_config",
"aquatic_udp_protocol", "aquatic_udp_protocol",
"arrayvec",
"blake3", "blake3",
"cfg-if", "cfg-if",
"compact_str", "compact_str",
@ -305,9 +306,9 @@ dependencies = [
"io-uring", "io-uring",
"libc", "libc",
"log", "log",
"metrics 0.21.1", "metrics 0.22.0",
"metrics-exporter-prometheus 0.12.2", "metrics-exporter-prometheus 0.13.0",
"metrics-util 0.15.1", "metrics-util 0.16.0",
"mimalloc", "mimalloc",
"mio", "mio",
"num-format", "num-format",
@ -319,7 +320,6 @@ dependencies = [
"slab", "slab",
"socket2 0.5.5", "socket2 0.5.5",
"tempfile", "tempfile",
"thingbuf",
"time", "time",
"tinytemplate", "tinytemplate",
] ]
@ -498,9 +498,9 @@ dependencies = [
[[package]] [[package]]
name = "base64" name = "base64"
version = "0.21.5" version = "0.21.7"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "35636a1494ede3b646cc98f74f8e62c773a38a659ebc777a2cf26b9b74171df9" checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567"
[[package]] [[package]]
name = "bendy" name = "bendy"
@ -528,9 +528,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
[[package]] [[package]]
name = "bitflags" name = "bitflags"
version = "2.4.1" version = "2.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07" checksum = "ed570934406eb16438a4e976b1b4500774099c13b8cb96eec99f620f05090ddf"
[[package]] [[package]]
name = "bitmaps" name = "bitmaps"
@ -661,9 +661,9 @@ dependencies = [
[[package]] [[package]]
name = "clap" name = "clap"
version = "4.4.13" version = "4.4.18"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "52bdc885e4cacc7f7c9eedc1ef6da641603180c783c41a15c264944deeaab642" checksum = "1e578d6ec4194633722ccf9544794b71b1385c3c027efe0c55db226fc880865c"
dependencies = [ dependencies = [
"clap_builder", "clap_builder",
"clap_derive", "clap_derive",
@ -671,9 +671,9 @@ dependencies = [
[[package]] [[package]]
name = "clap_builder" name = "clap_builder"
version = "4.4.12" version = "4.4.18"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fb7fb5e4e979aec3be7791562fcba452f94ad85e954da024396433e0e25a79e9" checksum = "4df4df40ec50c46000231c914968278b1eb05098cf8f1b3a518a95030e71d1c7"
dependencies = [ dependencies = [
"anstream", "anstream",
"anstyle", "anstyle",
@ -806,7 +806,7 @@ dependencies = [
"anes", "anes",
"cast", "cast",
"ciborium", "ciborium",
"clap 4.4.13", "clap 4.4.18",
"criterion-plot", "criterion-plot",
"is-terminal", "is-terminal",
"itertools 0.10.5", "itertools 0.10.5",
@ -835,11 +835,10 @@ dependencies = [
[[package]] [[package]]
name = "crossbeam" name = "crossbeam"
version = "0.8.3" version = "0.8.4"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6eb9105919ca8e40d437fc9cbb8f1975d916f1bd28afe795a48aae32a2cc8920" checksum = "1137cd7e7fc0fb5d3c5a8678be38ec56e819125d8d7907411fe24ccb943faca8"
dependencies = [ dependencies = [
"cfg-if",
"crossbeam-channel", "crossbeam-channel",
"crossbeam-deque", "crossbeam-deque",
"crossbeam-epoch", "crossbeam-epoch",
@ -849,54 +848,46 @@ dependencies = [
[[package]] [[package]]
name = "crossbeam-channel" name = "crossbeam-channel"
version = "0.5.10" version = "0.5.11"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "82a9b73a36529d9c47029b9fb3a6f0ea3cc916a261195352ba19e770fc1748b2" checksum = "176dc175b78f56c0f321911d9c8eb2b77a78a4860b9c19db83835fea1a46649b"
dependencies = [ dependencies = [
"cfg-if",
"crossbeam-utils", "crossbeam-utils",
] ]
[[package]] [[package]]
name = "crossbeam-deque" name = "crossbeam-deque"
version = "0.8.4" version = "0.8.5"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fca89a0e215bab21874660c67903c5f143333cab1da83d041c7ded6053774751" checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d"
dependencies = [ dependencies = [
"cfg-if",
"crossbeam-epoch", "crossbeam-epoch",
"crossbeam-utils", "crossbeam-utils",
] ]
[[package]] [[package]]
name = "crossbeam-epoch" name = "crossbeam-epoch"
version = "0.9.17" version = "0.9.18"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0e3681d554572a651dda4186cd47240627c3d0114d45a95f6ad27f2f22e7548d" checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e"
dependencies = [ dependencies = [
"autocfg",
"cfg-if",
"crossbeam-utils", "crossbeam-utils",
] ]
[[package]] [[package]]
name = "crossbeam-queue" name = "crossbeam-queue"
version = "0.3.10" version = "0.3.11"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "adc6598521bb5a83d491e8c1fe51db7296019d2ca3cb93cc6c2a20369a4d78a2" checksum = "df0346b5d5e76ac2fe4e327c5fd1118d6be7c51dfb18f9b7922923f287471e35"
dependencies = [ dependencies = [
"cfg-if",
"crossbeam-utils", "crossbeam-utils",
] ]
[[package]] [[package]]
name = "crossbeam-utils" name = "crossbeam-utils"
version = "0.8.18" version = "0.8.19"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c3a430a770ebd84726f584a90ee7f020d28db52c6d02138900f22341f866d39c" checksum = "248e3bacc7dc6baa3b21e405ee045c3047101a49145e7e9eca583ab4c2ca5345"
dependencies = [
"cfg-if",
]
[[package]] [[package]]
name = "crypto-common" name = "crypto-common"
@ -1158,9 +1149,9 @@ dependencies = [
[[package]] [[package]]
name = "futures-rustls" name = "futures-rustls"
version = "0.25.0" version = "0.25.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3afda89bce8f65072d24f8b99a2127e229462d8008182ca93f1d5d2e5df8f22f" checksum = "c8d8a2499f0fecc0492eb3e47eab4e92da7875e1028ad2528f214ac3346ca04e"
dependencies = [ dependencies = [
"futures-io", "futures-io",
"rustls", "rustls",
@ -1209,9 +1200,9 @@ dependencies = [
[[package]] [[package]]
name = "getrandom" name = "getrandom"
version = "0.2.11" version = "0.2.12"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fe9006bed769170c11f845cf00c7c1e9092aeb3f268e007c3e760ac68008070f" checksum = "190092ea657667030ac6a35e305e62fc4dd69fd98ac98631e5d3a2b1575a12b5"
dependencies = [ dependencies = [
"cfg-if", "cfg-if",
"js-sys", "js-sys",
@ -1356,9 +1347,9 @@ dependencies = [
[[package]] [[package]]
name = "hermit-abi" name = "hermit-abi"
version = "0.3.3" version = "0.3.4"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7" checksum = "5d3d0e0f38255e7fa3cf31335b3a56f05febd18025f4db5ef7a0cfb4f8da651f"
[[package]] [[package]]
name = "hex" name = "hex"
@ -1534,7 +1525,7 @@ version = "0.4.10"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0bad00257d07be169d870ab665980b06cdb366d792ad690bf2e76876dc503455" checksum = "0bad00257d07be169d870ab665980b06cdb366d792ad690bf2e76876dc503455"
dependencies = [ dependencies = [
"hermit-abi 0.3.3", "hermit-abi 0.3.4",
"rustix", "rustix",
"windows-sys 0.52.0", "windows-sys 0.52.0",
] ]
@ -1565,9 +1556,9 @@ checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c"
[[package]] [[package]]
name = "js-sys" name = "js-sys"
version = "0.3.66" version = "0.3.67"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cee9c64da59eae3b50095c18d3e74f8b73c0b86d2792824ff01bbce68ba229ca" checksum = "9a1d36f1235bc969acba30b7f5990b864423a6068a10f7c90ae8f0112e3a59d1"
dependencies = [ dependencies = [
"wasm-bindgen", "wasm-bindgen",
] ]
@ -1654,9 +1645,9 @@ dependencies = [
[[package]] [[package]]
name = "libc" name = "libc"
version = "0.2.151" version = "0.2.152"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "302d7ab3130588088d277783b1e2d2e10c9e9e4a16dd9050e6ec93fb3e7048f4" checksum = "13e3bf6590cbc649f4d1a3eefc9d5d6eb746f5200ffb04e5e142700b8faa56e7"
[[package]] [[package]]
name = "libm" name = "libm"
@ -1676,9 +1667,9 @@ dependencies = [
[[package]] [[package]]
name = "linux-raw-sys" name = "linux-raw-sys"
version = "0.4.12" version = "0.4.13"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c4cd1a83af159aa67994778be9070f0ae1bd732942279cabb14f86f986a21456" checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c"
[[package]] [[package]]
name = "lock_api" name = "lock_api"
@ -1819,16 +1810,12 @@ version = "0.15.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4de2ed6e491ed114b40b732e4d1659a9d53992ebd87490c44a6ffe23739d973e" checksum = "4de2ed6e491ed114b40b732e4d1659a9d53992ebd87490c44a6ffe23739d973e"
dependencies = [ dependencies = [
"aho-corasick",
"crossbeam-epoch", "crossbeam-epoch",
"crossbeam-utils", "crossbeam-utils",
"hashbrown 0.13.1", "hashbrown 0.13.1",
"indexmap 1.9.3",
"metrics 0.21.1", "metrics 0.21.1",
"num_cpus", "num_cpus",
"ordered-float 3.9.2",
"quanta 0.11.1", "quanta 0.11.1",
"radix_trie",
"sketches-ddsketch 0.2.1", "sketches-ddsketch 0.2.1",
] ]
@ -1845,7 +1832,7 @@ dependencies = [
"indexmap 1.9.3", "indexmap 1.9.3",
"metrics 0.22.0", "metrics 0.22.0",
"num_cpus", "num_cpus",
"ordered-float 4.2.0", "ordered-float",
"quanta 0.12.2", "quanta 0.12.2",
"radix_trie", "radix_trie",
"sketches-ddsketch 0.2.1", "sketches-ddsketch 0.2.1",
@ -2008,7 +1995,7 @@ version = "1.16.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43"
dependencies = [ dependencies = [
"hermit-abi 0.3.3", "hermit-abi 0.3.4",
"libc", "libc",
] ]
@ -2042,15 +2029,6 @@ version = "11.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575" checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575"
[[package]]
name = "ordered-float"
version = "3.9.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f1e1c390732d15f1d48471625cd92d154e66db2c56645e29a9cd26f4699f72dc"
dependencies = [
"num-traits",
]
[[package]] [[package]]
name = "ordered-float" name = "ordered-float"
version = "4.2.0" version = "4.2.0"
@ -2078,29 +2056,6 @@ version = "2.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bb813b8af86854136c6922af0598d719255ecb2179515e6e7730d468f05c9cae" checksum = "bb813b8af86854136c6922af0598d719255ecb2179515e6e7730d468f05c9cae"
[[package]]
name = "parking_lot"
version = "0.12.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f"
dependencies = [
"lock_api",
"parking_lot_core",
]
[[package]]
name = "parking_lot_core"
version = "0.9.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4c42a9226546d68acdd9c0a280d17ce19bfe27a46bf68784e4066115788d008e"
dependencies = [
"cfg-if",
"libc",
"redox_syscall",
"smallvec",
"windows-targets 0.48.5",
]
[[package]] [[package]]
name = "percent-encoding" name = "percent-encoding"
version = "2.3.1" version = "2.3.1"
@ -2141,9 +2096,9 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
[[package]] [[package]]
name = "pkg-config" name = "pkg-config"
version = "0.3.28" version = "0.3.29"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "69d3587f8a9e599cc7ec2c00e331f71c4e69a5f9a4b8a6efd5b07466b9736f9a" checksum = "2900ede94e305130c13ddd391e0ab7cbaeb783945ae07a279c268cb05109c6cb"
[[package]] [[package]]
name = "plotters" name = "plotters"
@ -2371,14 +2326,14 @@ version = "11.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9d86a7c4638d42c44551f4791a20e687dbb4c3de1f33c43dd71e355cd429def1" checksum = "9d86a7c4638d42c44551f4791a20e687dbb4c3de1f33c43dd71e355cd429def1"
dependencies = [ dependencies = [
"bitflags 2.4.1", "bitflags 2.4.2",
] ]
[[package]] [[package]]
name = "rayon" name = "rayon"
version = "1.8.0" version = "1.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9c27db03db7734835b3f53954b534c91069375ce6ccaa2e065441e07d9b6cdb1" checksum = "fa7237101a77a10773db45d62004a272517633fbcc3df19d96455ede1122e051"
dependencies = [ dependencies = [
"either", "either",
"rayon-core", "rayon-core",
@ -2386,9 +2341,9 @@ dependencies = [
[[package]] [[package]]
name = "rayon-core" name = "rayon-core"
version = "1.12.0" version = "1.12.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5ce3fb6ad83f861aac485e76e1985cd109d9a3713802152be56c3b1f0e0658ed" checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2"
dependencies = [ dependencies = [
"crossbeam-deque", "crossbeam-deque",
"crossbeam-utils", "crossbeam-utils",
@ -2483,11 +2438,11 @@ checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76"
[[package]] [[package]]
name = "rustix" name = "rustix"
version = "0.38.28" version = "0.38.30"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "72e572a5e8ca657d7366229cdde4bd14c4eb5499a9573d4d366fe1b599daa316" checksum = "322394588aaf33c24007e8bb3238ee3e4c5c09c084ab32bc73890b99ff326bca"
dependencies = [ dependencies = [
"bitflags 2.4.1", "bitflags 2.4.2",
"errno 0.3.8", "errno 0.3.8",
"libc", "libc",
"linux-raw-sys", "linux-raw-sys",
@ -2496,9 +2451,9 @@ dependencies = [
[[package]] [[package]]
name = "rustls" name = "rustls"
version = "0.22.1" version = "0.22.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fe6b63262c9fcac8659abfaa96cac103d28166d3ff3eaf8f412e19f3ae9e5a48" checksum = "e87c9956bd9807afa1f77e0f7594af32566e830e088a5576d27c5b6f30f49d41"
dependencies = [ dependencies = [
"log", "log",
"ring", "ring",
@ -2659,9 +2614,9 @@ dependencies = [
[[package]] [[package]]
name = "simd-json" name = "simd-json"
version = "0.13.6" version = "0.13.8"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0c58001aca67fc467da571f35e7e1dc9c094e91b099cc54bd3cead2962db2432" checksum = "2faf8f101b9bc484337a6a6b0409cf76c139f2fb70a9e3aee6b6774be7bfbf76"
dependencies = [ dependencies = [
"getrandom", "getrandom",
"halfbrown", "halfbrown",
@ -2723,9 +2678,9 @@ dependencies = [
[[package]] [[package]]
name = "smallvec" name = "smallvec"
version = "1.11.2" version = "1.13.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4dccd0940a2dcdf68d092b8cbab7dc0ad8fa938bf95787e1b916b0e3d0e8e970" checksum = "e6ecd384b10a64542d77071bd64bd7b231f4ed5940fba55e98c3de13824cf3d7"
[[package]] [[package]]
name = "smartstring" name = "smartstring"
@ -2848,16 +2803,6 @@ version = "0.16.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "222a222a5bfe1bba4a77b45ec488a741b3cb8872e5e499451fd7d0129c9c7c3d" checksum = "222a222a5bfe1bba4a77b45ec488a741b3cb8872e5e499451fd7d0129c9c7c3d"
[[package]]
name = "thingbuf"
version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4706f1bfb859af03f099ada2de3cea3e515843c2d3e93b7893f16d94a37f9415"
dependencies = [
"parking_lot",
"pin-project",
]
[[package]] [[package]]
name = "thiserror" name = "thiserror"
version = "1.0.56" version = "1.0.56"
@ -3061,9 +3006,9 @@ checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825"
[[package]] [[package]]
name = "unicode-bidi" name = "unicode-bidi"
version = "0.3.14" version = "0.3.15"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6f2528f27a9eb2b21e69c95319b30bd0efd85d09c379741b0f78ea1d86be2416" checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75"
[[package]] [[package]]
name = "unicode-ident" name = "unicode-ident"
@ -3166,9 +3111,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
[[package]] [[package]]
name = "wasm-bindgen" name = "wasm-bindgen"
version = "0.2.89" version = "0.2.90"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0ed0d4f68a3015cc185aff4db9506a015f4b96f95303897bfa23f846db54064e" checksum = "b1223296a201415c7fad14792dbefaace9bd52b62d33453ade1c5b5f07555406"
dependencies = [ dependencies = [
"cfg-if", "cfg-if",
"wasm-bindgen-macro", "wasm-bindgen-macro",
@ -3176,9 +3121,9 @@ dependencies = [
[[package]] [[package]]
name = "wasm-bindgen-backend" name = "wasm-bindgen-backend"
version = "0.2.89" version = "0.2.90"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1b56f625e64f3a1084ded111c4d5f477df9f8c92df113852fa5a374dbda78826" checksum = "fcdc935b63408d58a32f8cc9738a0bffd8f05cc7c002086c6ef20b7312ad9dcd"
dependencies = [ dependencies = [
"bumpalo", "bumpalo",
"log", "log",
@ -3191,9 +3136,9 @@ dependencies = [
[[package]] [[package]]
name = "wasm-bindgen-macro" name = "wasm-bindgen-macro"
version = "0.2.89" version = "0.2.90"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0162dbf37223cd2afce98f3d0785506dcb8d266223983e4b5b525859e6e182b2" checksum = "3e4c238561b2d428924c49815533a8b9121c664599558a5d9ec51f8a1740a999"
dependencies = [ dependencies = [
"quote", "quote",
"wasm-bindgen-macro-support", "wasm-bindgen-macro-support",
@ -3201,9 +3146,9 @@ dependencies = [
[[package]] [[package]]
name = "wasm-bindgen-macro-support" name = "wasm-bindgen-macro-support"
version = "0.2.89" version = "0.2.90"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f0eb82fcb7930ae6219a7ecfd55b217f5f0893484b7a13022ebb2b2bf20b5283" checksum = "bae1abb6806dc1ad9e560ed242107c0f6c84335f1749dd4e8ddb012ebd5e25a7"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
@ -3214,15 +3159,15 @@ dependencies = [
[[package]] [[package]]
name = "wasm-bindgen-shared" name = "wasm-bindgen-shared"
version = "0.2.89" version = "0.2.90"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7ab9b36309365056cd639da3134bf87fa8f3d86008abf99e612384a6eecd459f" checksum = "4d91413b1c31d7539ba5ef2451af3f0b833a005eb27a631cec32bc0635a8602b"
[[package]] [[package]]
name = "web-sys" name = "web-sys"
version = "0.3.66" version = "0.3.67"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "50c24a44ec86bb68fbecd1b3efed7e85ea5621b39b35ef2766b66cd984f8010f" checksum = "58cd2333b6e0be7a39605f0e255892fd7418a682d8da8fe042fe25128794d2ed"
dependencies = [ dependencies = [
"js-sys", "js-sys",
"wasm-bindgen", "wasm-bindgen",
@ -3405,9 +3350,9 @@ checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04"
[[package]] [[package]]
name = "winnow" name = "winnow"
version = "0.5.33" version = "0.5.34"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b7520bbdec7211caa7c4e682eb1fbe07abe20cee6756b6e00f537c82c11816aa" checksum = "b7cf47b659b318dccbd69cc4797a39ae128f533dce7902a1096044d1967b9c16"
dependencies = [ dependencies = [
"memchr", "memchr",
] ]

12
crates/bencher/src/common.rs
View file

@ -27,17 +27,11 @@ impl TaskSetCpuList {
let indicator = self.0.iter().map(|indicator| match indicator { let indicator = self.0.iter().map(|indicator| match indicator {
TaskSetCpuIndicator::Single(i) => i.to_string(), TaskSetCpuIndicator::Single(i) => i.to_string(),
TaskSetCpuIndicator::Range(range) => { TaskSetCpuIndicator::Range(range) => {
format!( format!("{}-{}", range.start, range.clone().last().unwrap())
"{}-{}",
range.start,
range.clone().into_iter().last().unwrap()
)
} }
}); });
Itertools::intersperse_with(indicator, || ",".to_string()) Itertools::intersperse_with(indicator, || ",".to_string()).collect()
.into_iter()
.collect()
} }
pub fn new( pub fn new(
@ -163,7 +157,7 @@ pub fn simple_load_test_runs(
workers: &[(usize, Priority)], workers: &[(usize, Priority)],
) -> Vec<(usize, Priority, TaskSetCpuList)> { ) -> Vec<(usize, Priority, TaskSetCpuList)> {
workers workers
.into_iter() .iter()
.copied() .copied()
.map(|(workers, priority)| { .map(|(workers, priority)| {
( (

2
crates/bencher/src/html.rs
View file

@ -191,7 +191,7 @@ pub fn html_all_runs(all_results: &[TrackerCoreCountResults]) -> String {
load_test_key_names = load_test_key_names.iter() load_test_key_names = load_test_key_names.iter()
.map(|name| format!("<th>Load test {}</th>", name)) .map(|name| format!("<th>Load test {}</th>", name))
.join("\n"), .join("\n"),
body = results.into_iter().map(|r| { body = results.iter_mut().map(|r| {
formatdoc! { formatdoc! {
" "
<tr> <tr>

5
crates/bencher/src/protocols/udp.rs
View file

@ -259,6 +259,7 @@ struct AquaticUdpRunner {
} }
impl AquaticUdpRunner { impl AquaticUdpRunner {
#[allow(clippy::new_ret_no_self)]
fn new( fn new(
socket_workers: usize, socket_workers: usize,
swarm_workers: usize, swarm_workers: usize,
@ -275,6 +276,7 @@ impl AquaticUdpRunner {
impl ProcessRunner for AquaticUdpRunner { impl ProcessRunner for AquaticUdpRunner {
type Command = UdpCommand; type Command = UdpCommand;
#[allow(clippy::field_reassign_with_default)]
fn run( fn run(
&self, &self,
command: &Self::Command, command: &Self::Command,
@ -322,6 +324,7 @@ struct OpenTrackerUdpRunner {
} }
impl OpenTrackerUdpRunner { impl OpenTrackerUdpRunner {
#[allow(clippy::new_ret_no_self)]
fn new(workers: usize, priority: Priority) -> Rc<dyn ProcessRunner<Command = UdpCommand>> { fn new(workers: usize, priority: Priority) -> Rc<dyn ProcessRunner<Command = UdpCommand>> {
Rc::new(Self { workers, priority }) Rc::new(Self { workers, priority })
} }
@ -368,6 +371,7 @@ impl ProcessRunner for OpenTrackerUdpRunner {
struct ChihayaUdpRunner; struct ChihayaUdpRunner;
impl ChihayaUdpRunner { impl ChihayaUdpRunner {
#[allow(clippy::new_ret_no_self)]
fn new() -> Rc<dyn ProcessRunner<Command = UdpCommand>> { fn new() -> Rc<dyn ProcessRunner<Command = UdpCommand>> {
Rc::new(Self {}) Rc::new(Self {})
} }
@ -426,6 +430,7 @@ struct AquaticUdpLoadTestRunner {
impl ProcessRunner for AquaticUdpLoadTestRunner { impl ProcessRunner for AquaticUdpLoadTestRunner {
type Command = UdpCommand; type Command = UdpCommand;
#[allow(clippy::field_reassign_with_default)]
fn run( fn run(
&self, &self,
command: &Self::Command, command: &Self::Command,

8
crates/bencher/src/run.rs
View file

@ -59,9 +59,7 @@ impl<C> RunConfig<C> {
.run(command, &self.tracker_vcpus, &mut tracker_config_file) .run(command, &self.tracker_vcpus, &mut tracker_config_file)
{ {
Ok(handle) => ChildWrapper(handle), Ok(handle) => ChildWrapper(handle),
Err(err) => { Err(err) => return Err(RunErrorResults::new(self).set_error(err, "run tracker")),
return Err(RunErrorResults::new(self).set_error(err.into(), "run tracker"))
}
}; };
::std::thread::sleep(Duration::from_secs(1)); ::std::thread::sleep(Duration::from_secs(1));
@ -74,7 +72,7 @@ impl<C> RunConfig<C> {
Ok(handle) => ChildWrapper(handle), Ok(handle) => ChildWrapper(handle),
Err(err) => { Err(err) => {
return Err(RunErrorResults::new(self) return Err(RunErrorResults::new(self)
.set_error(err.into(), "run load test") .set_error(err, "run load test")
.set_tracker_outputs(tracker)) .set_tracker_outputs(tracker))
} }
}; };
@ -328,7 +326,7 @@ impl FromStr for ProcessStats {
type Err = (); type Err = ();
fn from_str(s: &str) -> Result<Self, Self::Err> { fn from_str(s: &str) -> Result<Self, Self::Err> {
let mut parts = s.trim().split_whitespace(); let mut parts = s.split_whitespace();
let avg_cpu_utilization = parts.next().ok_or(())?.parse().map_err(|_| ())?; let avg_cpu_utilization = parts.next().ok_or(())?.parse().map_err(|_| ())?;
let peak_rss_kb: f32 = parts.next().ok_or(())?.parse().map_err(|_| ())?; let peak_rss_kb: f32 = parts.next().ok_or(())?.parse().map_err(|_| ())?;

6
crates/bencher/src/set.rs
View file

@ -73,13 +73,13 @@ pub fn run_sets<C, F, I>(
(minutes / 60, minutes % 60) (minutes / 60, minutes % 60)
}; };
println!(""); println!();
println!("Total number of load test runs: {}", total_num_runs); println!("Total number of load test runs: {}", total_num_runs);
println!( println!(
"Estimated duration: {} hours, {} minutes", "Estimated duration: {} hours, {} minutes",
estimated_hours, estimated_minutes estimated_hours, estimated_minutes
); );
println!(""); println!();
let results = set_configs let results = set_configs
.into_iter() .into_iter()
@ -115,7 +115,7 @@ pub fn run_sets<C, F, I>(
&load_test_gen, &load_test_gen,
load_test_parameters, load_test_parameters,
implementation, implementation,
&tracker_run, tracker_run,
tracker_vcpus.clone(), tracker_vcpus.clone(),
load_test_vcpus, load_test_vcpus,
) )

6
crates/combined_binary/src/main.rs
View file

@ -12,12 +12,12 @@ fn main() {
::std::process::exit(match run() { ::std::process::exit(match run() {
Ok(()) => 0, Ok(()) => 0,
Err(None) => { Err(None) => {
print_help(|| gen_info(), None); print_help(gen_info, None);
0 0
} }
Err(opt_err @ Some(_)) => { Err(opt_err @ Some(_)) => {
print_help(|| gen_info(), opt_err); print_help(gen_info, opt_err);
1 1
} }
@ -62,7 +62,7 @@ fn run() -> Result<(), Option<String>> {
arg => { arg => {
let opt_err = if arg == "-h" || arg == "--help" { let opt_err = if arg == "-h" || arg == "--help" {
None None
} else if arg.chars().next() == Some('-') { } else if arg.starts_with('-') {
Some("First argument must be protocol".to_string()) Some("First argument must be protocol".to_string())
} else { } else {
Some("Invalid protocol".to_string()) Some("Invalid protocol".to_string())

11
crates/common/src/access_list.rs
View file

@ -71,7 +71,7 @@ impl AccessList {
} }
new_list new_list
.insert_from_line(&line) .insert_from_line(line)
.with_context(|| format!("Invalid line in access list: {}", line))?; .with_context(|| format!("Invalid line in access list: {}", line))?;
} }
@ -86,6 +86,7 @@ impl AccessList {
} }
} }
#[allow(clippy::len_without_is_empty)]
pub fn len(&self) -> usize { pub fn len(&self) -> usize {
self.0.len() self.0.len()
} }
@ -155,10 +156,10 @@ mod tests {
fn test_parse_info_hash() { fn test_parse_info_hash() {
let f = parse_info_hash; let f = parse_info_hash;
assert!(f("aaaabbbbccccddddeeeeaaaabbbbccccddddeeee".into()).is_ok()); assert!(f("aaaabbbbccccddddeeeeaaaabbbbccccddddeeee").is_ok());
assert!(f("aaaabbbbccccddddeeeeaaaabbbbccccddddeeeef".into()).is_err()); assert!(f("aaaabbbbccccddddeeeeaaaabbbbccccddddeeeef").is_err());
assert!(f("aaaabbbbccccddddeeeeaaaabbbbccccddddeee".into()).is_err()); assert!(f("aaaabbbbccccddddeeeeaaaabbbbccccddddeee").is_err());
assert!(f("aaaabbbbccccddddeeeeaaaabbbbccccddddeeeö".into()).is_err()); assert!(f("aaaabbbbccccddddeeeeaaaabbbbccccddddeeeö").is_err());
} }
#[test] #[test]

1
crates/common/src/cli.rs
View file

@ -47,6 +47,7 @@ impl Options {
{ {
let mut options = Options::default(); let mut options = Options::default();
#[allow(clippy::while_let_loop)] // False positive
loop { loop {
if let Some(arg) = arg_iter.next() { if let Some(arg) = arg_iter.next() {
match arg.as_str() { match arg.as_str() {

4
crates/common/src/cpu_pinning.rs
View file

@ -296,12 +296,12 @@ pub fn pin_current_if_configured_to<C: CpuPinningConfig>(
let cpu_set = core_cpu_sets let cpu_set = core_cpu_sets
.get(core_index) .get(core_index)
.expect(&format!("get cpu set for core {}", core_index)) .unwrap_or_else(|| panic!("get cpu set for core {}", core_index))
.to_owned(); .to_owned();
topology topology
.set_cpubind(cpu_set, CPUBIND_THREAD) .set_cpubind(cpu_set, CPUBIND_THREAD)
.expect(&format!("bind thread to core {}", core_index)); .unwrap_or_else(|err| panic!("bind thread to core {}: {:?}", core_index, err));
::log::info!( ::log::info!(
"Pinned worker {:?} to cpu core {}", "Pinned worker {:?} to cpu core {}",

5
crates/common/src/lib.rs
View file

@ -39,6 +39,7 @@ impl ValidUntil {
pub struct ServerStartInstant(Instant); pub struct ServerStartInstant(Instant);
impl ServerStartInstant { impl ServerStartInstant {
#[allow(clippy::new_without_default)] // I prefer ::new here
pub fn new() -> Self { pub fn new() -> Self {
Self(Instant::now()) Self(Instant::now())
} }
@ -82,8 +83,7 @@ impl Drop for PanicSentinel {
if ::std::thread::panicking() { if ::std::thread::panicking() {
let already_triggered = self.0.fetch_or(true, Ordering::SeqCst); let already_triggered = self.0.fetch_or(true, Ordering::SeqCst);
if !already_triggered { if !already_triggered && unsafe { libc::raise(15) } == -1 {
if unsafe { libc::raise(15) } == -1 {
panic!( panic!(
"Could not raise SIGTERM: {:#}", "Could not raise SIGTERM: {:#}",
::std::io::Error::last_os_error() ::std::io::Error::last_os_error()
@ -91,7 +91,6 @@ impl Drop for PanicSentinel {
} }
} }
} }
}
} }
/// SocketAddr that is not an IPv6-mapped IPv4 address /// SocketAddr that is not an IPv6-mapped IPv4 address

4
crates/common/src/privileges.rs
View file

@ -48,8 +48,7 @@ impl PrivilegeDropper {
} }
pub fn after_socket_creation(self) -> anyhow::Result<()> { pub fn after_socket_creation(self) -> anyhow::Result<()> {
if self.config.drop_privileges { if self.config.drop_privileges && self.barrier.wait().is_leader() {
if self.barrier.wait().is_leader() {
PrivDrop::default() PrivDrop::default()
.chroot(self.config.chroot_path.clone()) .chroot(self.config.chroot_path.clone())
.group(self.config.group.clone()) .group(self.config.group.clone())
@ -57,7 +56,6 @@ impl PrivilegeDropper {
.apply() .apply()
.with_context(|| "couldn't drop privileges after socket creation")?; .with_context(|| "couldn't drop privileges after socket creation")?;
} }
}
Ok(()) Ok(())
} }

1
crates/common/src/rustls_config.rs
View file

@ -46,6 +46,7 @@ pub fn create_rustls_config(
.next() .next()
.ok_or(anyhow::anyhow!("No private keys in file"))??; .ok_or(anyhow::anyhow!("No private keys in file"))??;
#[allow(clippy::let_and_return)] // Using temporary variable fixes lifetime issue
key key
}; };

30
crates/http/src/workers/socket/connection.rs
View file

@ -16,11 +16,9 @@ use aquatic_http_protocol::response::{
use arc_swap::ArcSwap; use arc_swap::ArcSwap;
use either::Either; use either::Either;
use futures::stream::FuturesUnordered; use futures::stream::FuturesUnordered;
use futures_lite::future::race;
use futures_lite::{AsyncReadExt, AsyncWriteExt, StreamExt}; use futures_lite::{AsyncReadExt, AsyncWriteExt, StreamExt};
use futures_rustls::TlsAcceptor; use futures_rustls::TlsAcceptor;
use glommio::channels::channel_mesh::Senders; use glommio::channels::channel_mesh::Senders;
use glommio::channels::local_channel::LocalReceiver;
use glommio::channels::shared_channel::{self, SharedReceiver}; use glommio::channels::shared_channel::{self, SharedReceiver};
use glommio::net::TcpStream; use glommio::net::TcpStream;
use once_cell::sync::Lazy; use once_cell::sync::Lazy;
@ -76,7 +74,6 @@ pub(super) async fn run_connection(
server_start_instant: ServerStartInstant, server_start_instant: ServerStartInstant,
opt_tls_config: Option<Arc<ArcSwap<RustlsConfig>>>, opt_tls_config: Option<Arc<ArcSwap<RustlsConfig>>>,
valid_until: Rc<RefCell<ValidUntil>>, valid_until: Rc<RefCell<ValidUntil>>,
close_conn_receiver: LocalReceiver<()>,
stream: TcpStream, stream: TcpStream,
) -> Result<(), ConnectionError> { ) -> Result<(), ConnectionError> {
let access_list_cache = create_access_list_cache(&access_list); let access_list_cache = create_access_list_cache(&access_list);
@ -119,7 +116,7 @@ pub(super) async fn run_connection(
stream, stream,
}; };
conn.run(close_conn_receiver).await?; conn.run().await
} else { } else {
let mut conn = Connection { let mut conn = Connection {
config, config,
@ -135,10 +132,8 @@ pub(super) async fn run_connection(
stream, stream,
}; };
conn.run(close_conn_receiver).await?; conn.run().await
} }
Ok(())
} }
struct Connection<S> { struct Connection<S> {
@ -159,18 +154,7 @@ impl<S> Connection<S>
where where
S: futures::AsyncRead + futures::AsyncWrite + Unpin + 'static, S: futures::AsyncRead + futures::AsyncWrite + Unpin + 'static,
{ {
async fn run(&mut self, close_conn_receiver: LocalReceiver<()>) -> Result<(), ConnectionError> { async fn run(&mut self) -> Result<(), ConnectionError> {
let f1 = async { self.run_request_response_loop().await };
let f2 = async {
close_conn_receiver.recv().await;
Err(ConnectionError::Inactive)
};
race(f1, f2).await
}
async fn run_request_response_loop(&mut self) -> Result<(), ConnectionError> {
loop { loop {
let response = match self.read_request().await? { let response = match self.read_request().await? {
Either::Left(response) => Response::Failure(response), Either::Left(response) => Response::Failure(response),
@ -399,7 +383,7 @@ where
let body_len = response let body_len = response
.write(&mut &mut self.response_buffer[position..]) .write(&mut &mut self.response_buffer[position..])
.map_err(|err| ConnectionError::ResponseBufferWrite(err))?; .map_err(ConnectionError::ResponseBufferWrite)?;
position += body_len; position += body_len;
@ -407,7 +391,7 @@ where
return Err(ConnectionError::ResponseBufferFull); return Err(ConnectionError::ResponseBufferFull);
} }
(&mut self.response_buffer[position..position + 2]).copy_from_slice(b"\r\n"); self.response_buffer[position..position + 2].copy_from_slice(b"\r\n");
position += 2; position += 2;
@ -419,7 +403,7 @@ where
let start = RESPONSE_HEADER_A.len(); let start = RESPONSE_HEADER_A.len();
let end = start + RESPONSE_HEADER_B.len(); let end = start + RESPONSE_HEADER_B.len();
(&mut self.response_buffer[start..end]).copy_from_slice(RESPONSE_HEADER_B); self.response_buffer[start..end].copy_from_slice(RESPONSE_HEADER_B);
} }
// Set content-len header value // Set content-len header value
@ -431,7 +415,7 @@ where
let start = RESPONSE_HEADER_A.len(); let start = RESPONSE_HEADER_A.len();
let end = start + content_len_bytes.len(); let end = start + content_len_bytes.len();
(&mut self.response_buffer[start..end]).copy_from_slice(content_len_bytes); self.response_buffer[start..end].copy_from_slice(content_len_bytes);
} }
// Write buffer to stream // Write buffer to stream

15
crates/http/src/workers/socket/mod.rs
View file

@ -12,6 +12,7 @@ use aquatic_common::privileges::PrivilegeDropper;
use aquatic_common::rustls_config::RustlsConfig; use aquatic_common::rustls_config::RustlsConfig;
use aquatic_common::{CanonicalSocketAddr, PanicSentinel, ServerStartInstant}; use aquatic_common::{CanonicalSocketAddr, PanicSentinel, ServerStartInstant};
use arc_swap::ArcSwap; use arc_swap::ArcSwap;
use futures_lite::future::race;
use futures_lite::StreamExt; use futures_lite::StreamExt;
use glommio::channels::channel_mesh::{MeshBuilder, Partial, Role}; use glommio::channels::channel_mesh::{MeshBuilder, Partial, Role};
use glommio::channels::local_channel::{new_bounded, LocalSender}; use glommio::channels::local_channel::{new_bounded, LocalSender};
@ -32,6 +33,7 @@ struct ConnectionHandle {
valid_until: Rc<RefCell<ValidUntil>>, valid_until: Rc<RefCell<ValidUntil>>,
} }
#[allow(clippy::too_many_arguments)]
pub async fn run_socket_worker( pub async fn run_socket_worker(
_sentinel: PanicSentinel, _sentinel: PanicSentinel,
config: Config, config: Config,
@ -97,16 +99,23 @@ pub async fn run_socket_worker(
"worker_index" => worker_index.to_string(), "worker_index" => worker_index.to_string(),
); );
let result = run_connection( let f1 = async { run_connection(
config, config,
access_list, access_list,
request_senders, request_senders,
server_start_instant, server_start_instant,
opt_tls_config, opt_tls_config,
valid_until.clone(), valid_until.clone(),
close_conn_receiver,
stream, stream,
).await; ).await
};
let f2 = async {
close_conn_receiver.recv().await;
Err(ConnectionError::Inactive)
};
let result = race(f1, f2).await;
#[cfg(feature = "metrics")] #[cfg(feature = "metrics")]
::metrics::decrement_gauge!( ::metrics::decrement_gauge!(

2
crates/http/src/workers/socket/request.rs
View file

@ -52,7 +52,7 @@ fn parse_forwarded_header(
header_format: ReverseProxyPeerIpHeaderFormat, header_format: ReverseProxyPeerIpHeaderFormat,
headers: &[httparse::Header<'_>], headers: &[httparse::Header<'_>],
) -> anyhow::Result<IpAddr> { ) -> anyhow::Result<IpAddr> {
for header in headers.into_iter().rev() { for header in headers.iter().rev() {
if header.name == header_name { if header.name == header_name {
match header_format { match header_format {
ReverseProxyPeerIpHeaderFormat::LastAddress => { ReverseProxyPeerIpHeaderFormat::LastAddress => {

12
crates/http/src/workers/swarm/storage.rs
View file

@ -66,16 +66,14 @@ impl TorrentMaps {
valid_until, valid_until,
); );
let response = AnnounceResponse { AnnounceResponse {
complete: seeders, complete: seeders,
incomplete: leechers, incomplete: leechers,
announce_interval: config.protocol.peer_announce_interval, announce_interval: config.protocol.peer_announce_interval,
peers: ResponsePeerListV4(response_peers), peers: ResponsePeerListV4(response_peers),
peers6: ResponsePeerListV6(vec![]), peers6: ResponsePeerListV6(vec![]),
warning_message: None, warning_message: None,
}; }
response
} }
IpAddr::V6(peer_ip_address) => { IpAddr::V6(peer_ip_address) => {
let (seeders, leechers, response_peers) = self let (seeders, leechers, response_peers) = self
@ -90,16 +88,14 @@ impl TorrentMaps {
valid_until, valid_until,
); );
let response = AnnounceResponse { AnnounceResponse {
complete: seeders, complete: seeders,
incomplete: leechers, incomplete: leechers,
announce_interval: config.protocol.peer_announce_interval, announce_interval: config.protocol.peer_announce_interval,
peers: ResponsePeerListV4(vec![]), peers: ResponsePeerListV4(vec![]),
peers6: ResponsePeerListV6(response_peers), peers6: ResponsePeerListV6(response_peers),
warning_message: None, warning_message: None,
}; }
response
} }
} }
} }

6
crates/http_load_test/src/utils.rs
View file

@ -19,7 +19,7 @@ pub fn create_random_request(
let items = [RequestType::Announce, RequestType::Scrape]; let items = [RequestType::Announce, RequestType::Scrape];
let dist = WeightedIndex::new(&weights).expect("random request weighted index"); let dist = WeightedIndex::new(weights).expect("random request weighted index");
match items[dist.sample(rng)] { match items[dist.sample(rng)] {
RequestType::Announce => create_announce_request(config, state, rng), RequestType::Announce => create_announce_request(config, state, rng),
@ -37,7 +37,7 @@ fn create_announce_request(config: &Config, state: &LoadTestState, rng: &mut imp
} }
}; };
let info_hash_index = select_info_hash_index(config, &state, rng); let info_hash_index = select_info_hash_index(config, state, rng);
Request::Announce(AnnounceRequest { Request::Announce(AnnounceRequest {
info_hash: state.info_hashes[info_hash_index], info_hash: state.info_hashes[info_hash_index],
@ -57,7 +57,7 @@ fn create_scrape_request(config: &Config, state: &LoadTestState, rng: &mut impl
let mut scrape_hashes = Vec::with_capacity(5); let mut scrape_hashes = Vec::with_capacity(5);
for _ in 0..5 { for _ in 0..5 {
let info_hash_index = select_info_hash_index(config, &state, rng); let info_hash_index = select_info_hash_index(config, state, rng);
scrape_hashes.push(state.info_hashes[info_hash_index]); scrape_hashes.push(state.info_hashes[info_hash_index]);
} }

8
crates/http_protocol/src/request.rs
View file

@ -86,7 +86,7 @@ impl AnnounceRequest {
let mut position = 0usize; let mut position = 0usize;
for equal_sign_index in ::memchr::memchr_iter(b'=', query_string_bytes) { for equal_sign_index in ::memchr::memchr_iter(b'=', query_string_bytes) {
let segment_end = ampersand_iter.next().unwrap_or_else(|| query_string.len()); let segment_end = ampersand_iter.next().unwrap_or(query_string.len());
let key = query_string let key = query_string
.get(position..equal_sign_index) .get(position..equal_sign_index)
@ -207,7 +207,7 @@ impl ScrapeRequest {
let mut position = 0usize; let mut position = 0usize;
for equal_sign_index in ::memchr::memchr_iter(b'=', query_string_bytes) { for equal_sign_index in ::memchr::memchr_iter(b'=', query_string_bytes) {
let segment_end = ampersand_iter.next().unwrap_or_else(|| query_string.len()); let segment_end = ampersand_iter.next().unwrap_or(query_string.len());
let key = query_string let key = query_string
.get(position..equal_sign_index) .get(position..equal_sign_index)
@ -348,7 +348,7 @@ mod tests {
let mut bytes = Vec::new(); let mut bytes = Vec::new();
bytes.extend_from_slice(b"GET "); bytes.extend_from_slice(b"GET ");
bytes.extend_from_slice(&ANNOUNCE_REQUEST_PATH.as_bytes()); bytes.extend_from_slice(ANNOUNCE_REQUEST_PATH.as_bytes());
bytes.extend_from_slice(b" HTTP/1.1\r\n\r\n"); bytes.extend_from_slice(b" HTTP/1.1\r\n\r\n");
let parsed_request = Request::from_bytes(&bytes[..]).unwrap().unwrap(); let parsed_request = Request::from_bytes(&bytes[..]).unwrap().unwrap();
@ -362,7 +362,7 @@ mod tests {
let mut bytes = Vec::new(); let mut bytes = Vec::new();
bytes.extend_from_slice(b"GET "); bytes.extend_from_slice(b"GET ");
bytes.extend_from_slice(&SCRAPE_REQUEST_PATH.as_bytes()); bytes.extend_from_slice(SCRAPE_REQUEST_PATH.as_bytes());
bytes.extend_from_slice(b" HTTP/1.1\r\n\r\n"); bytes.extend_from_slice(b" HTTP/1.1\r\n\r\n");
let parsed_request = Request::from_bytes(&bytes[..]).unwrap().unwrap(); let parsed_request = Request::from_bytes(&bytes[..]).unwrap().unwrap();

3
crates/http_protocol/src/utils.rs
View file

@ -43,7 +43,7 @@ pub fn urldecode_20_bytes(value: &str) -> anyhow::Result<[u8; 20]> {
let hex = [first as u8, second as u8]; let hex = [first as u8, second as u8];
hex::decode_to_slice(&hex, &mut out_arr[i..i + 1]) hex::decode_to_slice(hex, &mut out_arr[i..i + 1])
.map_err(|err| anyhow::anyhow!("hex decode error: {:?}", err))?; .map_err(|err| anyhow::anyhow!("hex decode error: {:?}", err))?;
} else { } else {
out_arr[i] = c as u8; out_arr[i] = c as u8;
@ -280,6 +280,7 @@ mod tests {
} }
} }
#[allow(clippy::too_many_arguments)]
#[quickcheck] #[quickcheck]
fn test_urlencode_urldecode_20_bytes( fn test_urlencode_urldecode_20_bytes(
a: u8, a: u8,

2
crates/peer_id/src/lib.rs
View file

@ -234,7 +234,7 @@ mod tests {
let len = bytes.len(); let len = bytes.len();
(&mut peer_id.0[..len]).copy_from_slice(bytes); peer_id.0[..len].copy_from_slice(bytes);
peer_id peer_id
} }

5
crates/toml_config_derive/src/lib.rs
View file

@ -146,8 +146,7 @@ fn extract_comment_string(attrs: Vec<Attribute>) -> TokenStream {
} }
for token_tree in attr.tokens { for token_tree in attr.tokens {
match token_tree { if let TokenTree::Literal(literal) = token_tree {
TokenTree::Literal(literal) => {
let mut comment = format!("{}", literal); let mut comment = format!("{}", literal);
// Strip leading and trailing quotation marks // Strip leading and trailing quotation marks
@ -160,8 +159,6 @@ fn extract_comment_string(attrs: Vec<Attribute>) -> TokenStream {
output.push_str(&comment); output.push_str(&comment);
output.push('\n'); output.push('\n');
} }
_ => {}
}
} }
} }

9
crates/udp/Cargo.toml
View file

@ -32,6 +32,7 @@ aquatic_toml_config.workspace = true
aquatic_udp_protocol.workspace = true aquatic_udp_protocol.workspace = true
anyhow = "1" anyhow = "1"
arrayvec = "0.7"
blake3 = "1" blake3 = "1"
cfg-if = "1" cfg-if = "1"
compact_str = "0.7" compact_str = "0.7"
@ -51,20 +52,18 @@ serde = { version = "1", features = ["derive"] }
signal-hook = { version = "0.3" } signal-hook = { version = "0.3" }
slab = "0.4" slab = "0.4"
socket2 = { version = "0.5", features = ["all"] } socket2 = { version = "0.5", features = ["all"] }
thingbuf = "0.1"
time = { version = "0.3", features = ["formatting"] } time = { version = "0.3", features = ["formatting"] }
tinytemplate = "1" tinytemplate = "1"
# prometheus feature # prometheus feature
metrics = { version = "0.21", optional = true } metrics = { version = "0.22", optional = true }
metrics-util = { version = "0.15", optional = true } metrics-util = { version = "0.16", optional = true }
metrics-exporter-prometheus = { version = "0.12", optional = true, default-features = false, features = ["http-listener"] } metrics-exporter-prometheus = { version = "0.13", optional = true, default-features = false, features = ["http-listener"] }
# io-uring feature # io-uring feature
io-uring = { version = "0.6", optional = true } io-uring = { version = "0.6", optional = true }
[dev-dependencies] [dev-dependencies]
hex = "0.4"
tempfile = "3" tempfile = "3"
quickcheck = "1" quickcheck = "1"
quickcheck_macros = "1" quickcheck_macros = "1"

152
crates/udp/src/common.rs
View file

@ -1,68 +1,19 @@
use std::borrow::Cow;
use std::collections::BTreeMap; use std::collections::BTreeMap;
use std::hash::Hash; use std::hash::Hash;
use std::io::Write;
use std::mem::size_of;
use std::net::{SocketAddr, SocketAddrV4};
use std::sync::atomic::AtomicUsize; use std::sync::atomic::AtomicUsize;
use std::sync::Arc; use std::sync::Arc;
use crossbeam_channel::{Sender, TrySendError}; use crossbeam_channel::{Receiver, SendError, Sender, TrySendError};
use aquatic_common::access_list::AccessListArcSwap; use aquatic_common::access_list::AccessListArcSwap;
use aquatic_common::CanonicalSocketAddr; use aquatic_common::CanonicalSocketAddr;
use aquatic_udp_protocol::*; use aquatic_udp_protocol::*;
use hdrhistogram::Histogram; use hdrhistogram::Histogram;
use thingbuf::mpsc::blocking::SendRef;
use crate::config::Config; use crate::config::Config;
pub const BUFFER_SIZE: usize = 8192; pub const BUFFER_SIZE: usize = 8192;
#[derive(PartialEq, Eq, Clone, Debug)]
pub enum CowResponse<'a> {
Connect(Cow<'a, ConnectResponse>),
AnnounceIpv4(Cow<'a, AnnounceResponse<Ipv4AddrBytes>>),
AnnounceIpv6(Cow<'a, AnnounceResponse<Ipv6AddrBytes>>),
Scrape(Cow<'a, ScrapeResponse>),
Error(Cow<'a, ErrorResponse>),
}
impl From<Response> for CowResponse<'_> {
fn from(value: Response) -> Self {
match value {
Response::AnnounceIpv4(r) => Self::AnnounceIpv4(Cow::Owned(r)),
Response::AnnounceIpv6(r) => Self::AnnounceIpv6(Cow::Owned(r)),
Response::Connect(r) => Self::Connect(Cow::Owned(r)),
Response::Scrape(r) => Self::Scrape(Cow::Owned(r)),
Response::Error(r) => Self::Error(Cow::Owned(r)),
}
}
}
impl<'a> CowResponse<'a> {
pub fn into_owned(self) -> Response {
match self {
CowResponse::Connect(r) => Response::Connect(r.into_owned()),
CowResponse::AnnounceIpv4(r) => Response::AnnounceIpv4(r.into_owned()),
CowResponse::AnnounceIpv6(r) => Response::AnnounceIpv6(r.into_owned()),
CowResponse::Scrape(r) => Response::Scrape(r.into_owned()),
CowResponse::Error(r) => Response::Error(r.into_owned()),
}
}
#[inline]
pub fn write(&self, bytes: &mut impl Write) -> Result<(), ::std::io::Error> {
match self {
Self::Connect(r) => r.write(bytes),
Self::AnnounceIpv4(r) => r.write(bytes),
Self::AnnounceIpv6(r) => r.write(bytes),
Self::Scrape(r) => r.write(bytes),
Self::Error(r) => r.write(bytes),
}
}
}
#[derive(Debug)] #[derive(Debug)]
pub struct PendingScrapeRequest { pub struct PendingScrapeRequest {
pub slab_key: usize, pub slab_key: usize,
@ -88,52 +39,6 @@ pub enum ConnectedResponse {
Scrape(PendingScrapeResponse), Scrape(PendingScrapeResponse),
} }
pub enum ConnectedResponseKind {
AnnounceIpv4,
AnnounceIpv6,
Scrape,
}
pub struct ConnectedResponseWithAddr {
pub kind: ConnectedResponseKind,
pub announce_ipv4: AnnounceResponse<Ipv4AddrBytes>,
pub announce_ipv6: AnnounceResponse<Ipv6AddrBytes>,
pub scrape: PendingScrapeResponse,
pub addr: CanonicalSocketAddr,
}
impl ConnectedResponseWithAddr {
pub fn estimated_max_size(config: &Config) -> usize {
size_of::<Self>()
+ config.protocol.max_response_peers
* (size_of::<ResponsePeer<Ipv4AddrBytes>>()
+ size_of::<ResponsePeer<Ipv6AddrBytes>>())
}
}
pub struct Recycler;
impl thingbuf::Recycle<ConnectedResponseWithAddr> for Recycler {
fn new_element(&self) -> ConnectedResponseWithAddr {
ConnectedResponseWithAddr {
kind: ConnectedResponseKind::AnnounceIpv4,
announce_ipv4: AnnounceResponse::empty(),
announce_ipv6: AnnounceResponse::empty(),
scrape: PendingScrapeResponse {
slab_key: 0,
torrent_stats: Default::default(),
},
addr: CanonicalSocketAddr::new(SocketAddr::V4(SocketAddrV4::new(0.into(), 0))),
}
}
fn recycle(&self, element: &mut ConnectedResponseWithAddr) {
element.announce_ipv4.peers.clear();
element.announce_ipv6.peers.clear();
element.scrape.torrent_stats.clear();
element.addr = CanonicalSocketAddr::new(SocketAddr::V4(SocketAddrV4::new(0.into(), 0)));
}
}
#[derive(Clone, Copy, Debug)] #[derive(Clone, Copy, Debug)]
pub struct SocketWorkerIndex(pub usize); pub struct SocketWorkerIndex(pub usize);
@ -180,54 +85,73 @@ impl ConnectedRequestSender {
} }
pub struct ConnectedResponseSender { pub struct ConnectedResponseSender {
senders: Vec<thingbuf::mpsc::blocking::Sender<ConnectedResponseWithAddr, Recycler>>, senders: Vec<Sender<(CanonicalSocketAddr, ConnectedResponse)>>,
to_any_last_index_picked: usize, to_any_last_index_picked: usize,
} }
impl ConnectedResponseSender { impl ConnectedResponseSender {
pub fn new( pub fn new(senders: Vec<Sender<(CanonicalSocketAddr, ConnectedResponse)>>) -> Self {
senders: Vec<thingbuf::mpsc::blocking::Sender<ConnectedResponseWithAddr, Recycler>>,
) -> Self {
Self { Self {
senders, senders,
to_any_last_index_picked: 0, to_any_last_index_picked: 0,
} }
} }
pub fn try_send_ref_to( pub fn try_send_to(
&self, &self,
index: SocketWorkerIndex, index: SocketWorkerIndex,
) -> Result<SendRef<ConnectedResponseWithAddr>, thingbuf::mpsc::errors::TrySendError> { addr: CanonicalSocketAddr,
self.senders[index.0].try_send_ref() response: ConnectedResponse,
) -> Result<(), TrySendError<(CanonicalSocketAddr, ConnectedResponse)>> {
self.senders[index.0].try_send((addr, response))
} }
pub fn send_ref_to( pub fn send_to(
&self, &self,
index: SocketWorkerIndex, index: SocketWorkerIndex,
) -> Result<SendRef<ConnectedResponseWithAddr>, thingbuf::mpsc::errors::Closed> { addr: CanonicalSocketAddr,
self.senders[index.0].send_ref() response: ConnectedResponse,
) -> Result<(), SendError<(CanonicalSocketAddr, ConnectedResponse)>> {
self.senders[index.0].send((addr, response))
} }
pub fn send_ref_to_any( pub fn send_to_any(
&mut self, &mut self,
) -> Result<SendRef<ConnectedResponseWithAddr>, thingbuf::mpsc::errors::Closed> { addr: CanonicalSocketAddr,
response: ConnectedResponse,
) -> Result<(), SendError<(CanonicalSocketAddr, ConnectedResponse)>> {
let start = self.to_any_last_index_picked + 1; let start = self.to_any_last_index_picked + 1;
let mut message = Some((addr, response));
for i in (start..start + self.senders.len()).map(|i| i % self.senders.len()) { for i in (start..start + self.senders.len()).map(|i| i % self.senders.len()) {
if let Ok(sender) = self.senders[i].try_send_ref() { match self.senders[i].try_send(message.take().unwrap()) {
Ok(()) => {
self.to_any_last_index_picked = i; self.to_any_last_index_picked = i;
return Ok(sender); return Ok(());
}
Err(TrySendError::Full(msg)) => {
message = Some(msg);
}
Err(TrySendError::Disconnected(_)) => {
panic!("ConnectedResponseReceiver disconnected");
}
} }
} }
let (addr, response) = message.unwrap();
self.to_any_last_index_picked = start % self.senders.len(); self.to_any_last_index_picked = start % self.senders.len();
self.send_ref_to(SocketWorkerIndex(self.to_any_last_index_picked)) self.send_to(
SocketWorkerIndex(self.to_any_last_index_picked),
addr,
response,
)
} }
} }
pub type ConnectedResponseReceiver = pub type ConnectedResponseReceiver = Receiver<(CanonicalSocketAddr, ConnectedResponse)>;
thingbuf::mpsc::blocking::Receiver<ConnectedResponseWithAddr, Recycler>;
#[derive(PartialEq, Eq, Hash, Clone, Copy, Debug)] #[derive(PartialEq, Eq, Hash, Clone, Copy, Debug)]
pub enum PeerStatus { pub enum PeerStatus {
@ -287,7 +211,7 @@ impl Statistics {
} }
fn create_atomic_usize_vec(len: usize) -> Vec<AtomicUsize> { fn create_atomic_usize_vec(len: usize) -> Vec<AtomicUsize> {
::std::iter::repeat_with(|| AtomicUsize::default()) ::std::iter::repeat_with(AtomicUsize::default)
.take(len) .take(len)
.collect() .collect()
} }

46
crates/udp/src/lib.rs
View file

@ -3,7 +3,6 @@ pub mod config;
pub mod workers; pub mod workers;
use std::collections::BTreeMap; use std::collections::BTreeMap;
use std::mem::size_of;
use std::thread::Builder; use std::thread::Builder;
use std::time::Duration; use std::time::Duration;
@ -16,16 +15,14 @@ use aquatic_common::access_list::update_access_list;
#[cfg(feature = "cpu-pinning")] #[cfg(feature = "cpu-pinning")]
use aquatic_common::cpu_pinning::{pin_current_if_configured_to, WorkerIndex}; use aquatic_common::cpu_pinning::{pin_current_if_configured_to, WorkerIndex};
use aquatic_common::privileges::PrivilegeDropper; use aquatic_common::privileges::PrivilegeDropper;
use aquatic_common::{CanonicalSocketAddr, PanicSentinelWatcher, ServerStartInstant}; use aquatic_common::{PanicSentinelWatcher, ServerStartInstant};
use common::{ use common::{
ConnectedRequestSender, ConnectedResponseSender, Recycler, SocketWorkerIndex, State, ConnectedRequestSender, ConnectedResponseSender, SocketWorkerIndex, State, SwarmWorkerIndex,
SwarmWorkerIndex,
}; };
use config::Config; use config::Config;
use workers::socket::ConnectionValidator; use workers::socket::ConnectionValidator;
use workers::swarm::SwarmWorker;
use crate::common::{ConnectedRequest, ConnectedResponseWithAddr};
pub const APP_NAME: &str = "aquatic_udp: UDP BitTorrent tracker"; pub const APP_NAME: &str = "aquatic_udp: UDP BitTorrent tracker";
pub const APP_VERSION: &str = env!("CARGO_PKG_VERSION"); pub const APP_VERSION: &str = env!("CARGO_PKG_VERSION");
@ -33,11 +30,6 @@ pub const APP_VERSION: &str = env!("CARGO_PKG_VERSION");
pub fn run(config: Config) -> ::anyhow::Result<()> { pub fn run(config: Config) -> ::anyhow::Result<()> {
let mut signals = Signals::new([SIGUSR1, SIGTERM])?; let mut signals = Signals::new([SIGUSR1, SIGTERM])?;
::log::info!(
"Estimated max channel memory use: {:.02} MB",
est_max_total_channel_memory(&config)
);
let state = State::new(config.swarm_workers); let state = State::new(config.swarm_workers);
let connection_validator = ConnectionValidator::new(&config)?; let connection_validator = ConnectionValidator::new(&config)?;
let (sentinel_watcher, sentinel) = PanicSentinelWatcher::create_with_sentinel(); let (sentinel_watcher, sentinel) = PanicSentinelWatcher::create_with_sentinel();
@ -56,19 +48,14 @@ pub fn run(config: Config) -> ::anyhow::Result<()> {
let server_start_instant = ServerStartInstant::new(); let server_start_instant = ServerStartInstant::new();
for i in 0..config.swarm_workers { for i in 0..config.swarm_workers {
let (request_sender, request_receiver) = if config.worker_channel_size == 0 { let (request_sender, request_receiver) = bounded(config.worker_channel_size);
unbounded()
} else {
bounded(config.worker_channel_size)
};
request_senders.push(request_sender); request_senders.push(request_sender);
request_receivers.insert(i, request_receiver); request_receivers.insert(i, request_receiver);
} }
for i in 0..config.socket_workers { for i in 0..config.socket_workers {
let (response_sender, response_receiver) = let (response_sender, response_receiver) = bounded(config.worker_channel_size);
thingbuf::mpsc::blocking::with_recycle(config.worker_channel_size, Recycler);
response_senders.push(response_sender); response_senders.push(response_sender);
response_receivers.insert(i, response_receiver); response_receivers.insert(i, response_receiver);
@ -93,16 +80,18 @@ pub fn run(config: Config) -> ::anyhow::Result<()> {
WorkerIndex::SwarmWorker(i), WorkerIndex::SwarmWorker(i),
); );
workers::swarm::run_swarm_worker( let mut worker = SwarmWorker {
sentinel, _sentinel: sentinel,
config, config,
state, state,
server_start_instant, server_start_instant,
request_receiver, request_receiver,
response_sender, response_sender,
statistics_sender, statistics_sender,
SwarmWorkerIndex(i), worker_index: SwarmWorkerIndex(i),
) };
worker.run();
}) })
.with_context(|| "spawn swarm worker")?; .with_context(|| "spawn swarm worker")?;
} }
@ -214,16 +203,3 @@ pub fn run(config: Config) -> ::anyhow::Result<()> {
Ok(()) Ok(())
} }
fn est_max_total_channel_memory(config: &Config) -> f64 {
let request_channel_max_size = config.swarm_workers
* config.worker_channel_size
* (size_of::<SocketWorkerIndex>()
+ size_of::<ConnectedRequest>()
+ size_of::<CanonicalSocketAddr>());
let response_channel_max_size = config.socket_workers
* config.worker_channel_size
* ConnectedResponseWithAddr::estimated_max_size(&config);
(request_channel_max_size as u64 + response_channel_max_size as u64) as f64 / (1024.0 * 1024.0)
}

56
crates/udp/src/workers/socket/mio.rs
View file

@ -1,4 +1,3 @@
use std::borrow::Cow;
use std::io::{Cursor, ErrorKind}; use std::io::{Cursor, ErrorKind};
use std::sync::atomic::Ordering; use std::sync::atomic::Ordering;
use std::time::{Duration, Instant}; use std::time::{Duration, Instant};
@ -42,7 +41,7 @@ pub struct SocketWorker {
server_start_instant: ServerStartInstant, server_start_instant: ServerStartInstant,
pending_scrape_responses: PendingScrapeResponseSlab, pending_scrape_responses: PendingScrapeResponseSlab,
socket: UdpSocket, socket: UdpSocket,
opt_resend_buffer: Option<Vec<(Response, CanonicalSocketAddr)>>, opt_resend_buffer: Option<Vec<(CanonicalSocketAddr, Response)>>,
buffer: [u8; BUFFER_SIZE], buffer: [u8; BUFFER_SIZE],
polling_mode: PollMode, polling_mode: PollMode,
/// Storage for requests that couldn't be sent to swarm worker because channel was full /// Storage for requests that couldn't be sent to swarm worker because channel was full
@ -50,6 +49,7 @@ pub struct SocketWorker {
} }
impl SocketWorker { impl SocketWorker {
#[allow(clippy::too_many_arguments)]
pub fn run( pub fn run(
_sentinel: PanicSentinel, _sentinel: PanicSentinel,
shared_state: State, shared_state: State,
@ -133,14 +133,14 @@ impl SocketWorker {
// If resend buffer is enabled, send any responses in it // If resend buffer is enabled, send any responses in it
if let Some(resend_buffer) = self.opt_resend_buffer.as_mut() { if let Some(resend_buffer) = self.opt_resend_buffer.as_mut() {
for (response, addr) in resend_buffer.drain(..) { for (addr, response) in resend_buffer.drain(..) {
Self::send_response( Self::send_response(
&self.config, &self.config,
&self.shared_state, &self.shared_state,
&mut self.socket, &mut self.socket,
&mut self.buffer, &mut self.buffer,
&mut None, &mut None,
response.into(), response,
addr, addr,
); );
} }
@ -206,7 +206,7 @@ impl SocketWorker {
if let Err(HandleRequestError::RequestChannelFull(failed_requests)) = if let Err(HandleRequestError::RequestChannelFull(failed_requests)) =
self.handle_request(pending_scrape_valid_until, request, src) self.handle_request(pending_scrape_valid_until, request, src)
{ {
self.pending_requests.extend(failed_requests.into_iter()); self.pending_requests.extend(failed_requests);
self.polling_mode = PollMode::SkipReceiving; self.polling_mode = PollMode::SkipReceiving;
break; break;
@ -235,7 +235,7 @@ impl SocketWorker {
&mut self.socket, &mut self.socket,
&mut self.buffer, &mut self.buffer,
&mut self.opt_resend_buffer, &mut self.opt_resend_buffer,
CowResponse::Error(Cow::Owned(response)), Response::Error(response),
src, src,
); );
} }
@ -310,7 +310,7 @@ impl SocketWorker {
&mut self.socket, &mut self.socket,
&mut self.buffer, &mut self.buffer,
&mut self.opt_resend_buffer, &mut self.opt_resend_buffer,
CowResponse::Connect(Cow::Owned(response)), Response::Connect(response),
src, src,
); );
@ -346,7 +346,7 @@ impl SocketWorker {
&mut self.socket, &mut self.socket,
&mut self.buffer, &mut self.buffer,
&mut self.opt_resend_buffer, &mut self.opt_resend_buffer,
CowResponse::Error(Cow::Owned(response)), Response::Error(response),
src, src,
); );
@ -392,30 +392,20 @@ impl SocketWorker {
} }
fn handle_swarm_worker_responses(&mut self) { fn handle_swarm_worker_responses(&mut self) {
loop { for (addr, response) in self.response_receiver.try_iter() {
let recv_ref = if let Ok(recv_ref) = self.response_receiver.try_recv_ref() { let response = match response {
recv_ref ConnectedResponse::Scrape(response) => {
} else {
break;
};
let response = match recv_ref.kind {
ConnectedResponseKind::Scrape => {
if let Some(r) = self if let Some(r) = self
.pending_scrape_responses .pending_scrape_responses
.add_and_get_finished(&recv_ref.scrape) .add_and_get_finished(&response)
{ {
CowResponse::Scrape(Cow::Owned(r)) Response::Scrape(r)
} else { } else {
continue; continue;
} }
} }
ConnectedResponseKind::AnnounceIpv4 => { ConnectedResponse::AnnounceIpv4(r) => Response::AnnounceIpv4(r),
CowResponse::AnnounceIpv4(Cow::Borrowed(&recv_ref.announce_ipv4)) ConnectedResponse::AnnounceIpv6(r) => Response::AnnounceIpv6(r),
}
ConnectedResponseKind::AnnounceIpv6 => {
CowResponse::AnnounceIpv6(Cow::Borrowed(&recv_ref.announce_ipv6))
}
}; };
Self::send_response( Self::send_response(
@ -425,7 +415,7 @@ impl SocketWorker {
&mut self.buffer, &mut self.buffer,
&mut self.opt_resend_buffer, &mut self.opt_resend_buffer,
response, response,
recv_ref.addr, addr,
); );
} }
} }
@ -435,8 +425,8 @@ impl SocketWorker {
shared_state: &State, shared_state: &State,
socket: &mut UdpSocket, socket: &mut UdpSocket,
buffer: &mut [u8], buffer: &mut [u8],
opt_resend_buffer: &mut Option<Vec<(Response, CanonicalSocketAddr)>>, opt_resend_buffer: &mut Option<Vec<(CanonicalSocketAddr, Response)>>,
response: CowResponse, response: Response,
canonical_addr: CanonicalSocketAddr, canonical_addr: CanonicalSocketAddr,
) { ) {
let mut buffer = Cursor::new(&mut buffer[..]); let mut buffer = Cursor::new(&mut buffer[..]);
@ -478,18 +468,18 @@ impl SocketWorker {
}; };
match response { match response {
CowResponse::Connect(_) => { Response::Connect(_) => {
stats.responses_sent_connect.fetch_add(1, Ordering::Relaxed); stats.responses_sent_connect.fetch_add(1, Ordering::Relaxed);
} }
CowResponse::AnnounceIpv4(_) | CowResponse::AnnounceIpv6(_) => { Response::AnnounceIpv4(_) | Response::AnnounceIpv6(_) => {
stats stats
.responses_sent_announce .responses_sent_announce
.fetch_add(1, Ordering::Relaxed); .fetch_add(1, Ordering::Relaxed);
} }
CowResponse::Scrape(_) => { Response::Scrape(_) => {
stats.responses_sent_scrape.fetch_add(1, Ordering::Relaxed); stats.responses_sent_scrape.fetch_add(1, Ordering::Relaxed);
} }
CowResponse::Error(_) => { Response::Error(_) => {
stats.responses_sent_error.fetch_add(1, Ordering::Relaxed); stats.responses_sent_error.fetch_add(1, Ordering::Relaxed);
} }
} }
@ -503,7 +493,7 @@ impl SocketWorker {
if resend_buffer.len() < config.network.resend_buffer_max_len { if resend_buffer.len() < config.network.resend_buffer_max_len {
::log::debug!("Adding response to resend queue, since sending it to {} failed with: {:#}", addr, err); ::log::debug!("Adding response to resend queue, since sending it to {} failed with: {:#}", addr, err);
resend_buffer.push((response.into_owned(), canonical_addr)); resend_buffer.push((canonical_addr, response));
} else { } else {
::log::warn!("Response resend buffer full, dropping response"); ::log::warn!("Response resend buffer full, dropping response");
} }

1
crates/udp/src/workers/socket/mod.rs
View file

@ -36,6 +36,7 @@ const EXTRA_PACKET_SIZE_IPV4: usize = 8 + 18 + 20 + 8;
/// - 8 bit udp header /// - 8 bit udp header
const EXTRA_PACKET_SIZE_IPV6: usize = 8 + 18 + 40 + 8; const EXTRA_PACKET_SIZE_IPV6: usize = 8 + 18 + 40 + 8;
#[allow(clippy::too_many_arguments)]
pub fn run_socket_worker( pub fn run_socket_worker(
sentinel: PanicSentinel, sentinel: PanicSentinel,
shared_state: State, shared_state: State,

9
crates/udp/src/workers/socket/storage.rs
View file

@ -44,7 +44,7 @@ impl PendingScrapeResponseSlab {
for (i, info_hash) in request.info_hashes.into_iter().enumerate() { for (i, info_hash) in request.info_hashes.into_iter().enumerate() {
let split_request = split_requests let split_request = split_requests
.entry(SwarmWorkerIndex::from_info_hash(&config, info_hash)) .entry(SwarmWorkerIndex::from_info_hash(config, info_hash))
.or_insert_with(|| PendingScrapeRequest { .or_insert_with(|| PendingScrapeRequest {
slab_key, slab_key,
info_hashes: BTreeMap::new(), info_hashes: BTreeMap::new(),
@ -130,9 +130,10 @@ mod tests {
return TestResult::discard(); return TestResult::discard();
} }
let mut config = Config::default(); let config = Config {
swarm_workers: swarm_workers as usize,
config.swarm_workers = swarm_workers as usize; ..Default::default()
};
let valid_until = ValidUntil::new(ServerStartInstant::new(), 1); let valid_until = ValidUntil::new(ServerStartInstant::new(), 1);

57
crates/udp/src/workers/socket/uring/mod.rs
View file

@ -2,7 +2,6 @@ mod buf_ring;
mod recv_helper; mod recv_helper;
mod send_buffers; mod send_buffers;
use std::borrow::Cow;
use std::cell::RefCell; use std::cell::RefCell;
use std::collections::VecDeque; use std::collections::VecDeque;
use std::net::UdpSocket; use std::net::UdpSocket;
@ -97,6 +96,7 @@ pub struct SocketWorker {
} }
impl SocketWorker { impl SocketWorker {
#[allow(clippy::too_many_arguments)]
pub fn run( pub fn run(
_sentinel: PanicSentinel, _sentinel: PanicSentinel,
shared_state: State, shared_state: State,
@ -137,7 +137,7 @@ impl SocketWorker {
.build() .build()
.unwrap(); .unwrap();
let recv_sqe = recv_helper.create_entry(buf_ring.bgid().try_into().unwrap()); let recv_sqe = recv_helper.create_entry(buf_ring.bgid());
// This timeout enables regular updates of pending_scrape_valid_until // This timeout enables regular updates of pending_scrape_valid_until
// and wakes the main loop to send any pending responses in the case // and wakes the main loop to send any pending responses in the case
@ -210,15 +210,14 @@ impl SocketWorker {
// Enqueue local responses // Enqueue local responses
for _ in 0..sq_space { for _ in 0..sq_space {
if let Some((response, addr)) = self.local_responses.pop_front() { if let Some((response, addr)) = self.local_responses.pop_front() {
match self.send_buffers.prepare_entry(response.into(), addr) { match self.send_buffers.prepare_entry(response, addr) {
Ok(entry) => { Ok(entry) => {
unsafe { ring.submission().push(&entry).unwrap() }; unsafe { ring.submission().push(&entry).unwrap() };
num_send_added += 1; num_send_added += 1;
} }
Err(send_buffers::Error::NoBuffers(response)) => { Err(send_buffers::Error::NoBuffers(response)) => {
self.local_responses self.local_responses.push_front((response, addr));
.push_front((response.into_owned(), addr));
break; break;
} }
@ -233,40 +232,32 @@ impl SocketWorker {
// Enqueue swarm worker responses // Enqueue swarm worker responses
for _ in 0..(sq_space - num_send_added) { for _ in 0..(sq_space - num_send_added) {
let recv_ref = if let Ok(recv_ref) = self.response_receiver.try_recv_ref() { let (addr, response) = if let Ok(r) = self.response_receiver.try_recv() {
recv_ref r
} else { } else {
break; break;
}; };
let response = match recv_ref.kind { let response = match response {
ConnectedResponseKind::AnnounceIpv4 => { ConnectedResponse::AnnounceIpv4(r) => Response::AnnounceIpv4(r),
CowResponse::AnnounceIpv4(Cow::Borrowed(&recv_ref.announce_ipv4)) ConnectedResponse::AnnounceIpv6(r) => Response::AnnounceIpv6(r),
} ConnectedResponse::Scrape(r) => {
ConnectedResponseKind::AnnounceIpv6 => { if let Some(r) = self.pending_scrape_responses.add_and_get_finished(&r) {
CowResponse::AnnounceIpv6(Cow::Borrowed(&recv_ref.announce_ipv6)) Response::Scrape(r)
}
ConnectedResponseKind::Scrape => {
if let Some(response) = self
.pending_scrape_responses
.add_and_get_finished(&recv_ref.scrape)
{
CowResponse::Scrape(Cow::Owned(response))
} else { } else {
continue; continue;
} }
} }
}; };
match self.send_buffers.prepare_entry(response, recv_ref.addr) { match self.send_buffers.prepare_entry(response, addr) {
Ok(entry) => { Ok(entry) => {
unsafe { ring.submission().push(&entry).unwrap() }; unsafe { ring.submission().push(&entry).unwrap() };
num_send_added += 1; num_send_added += 1;
} }
Err(send_buffers::Error::NoBuffers(response)) => { Err(send_buffers::Error::NoBuffers(response)) => {
self.local_responses self.local_responses.push_back((response, addr));
.push_back((response.into_owned(), recv_ref.addr));
break; break;
} }
@ -481,11 +472,11 @@ impl SocketWorker {
let worker_index = let worker_index =
SwarmWorkerIndex::from_info_hash(&self.config, request.info_hash); SwarmWorkerIndex::from_info_hash(&self.config, request.info_hash);
if let Err(_) = self.request_sender.try_send_to( if self
worker_index, .request_sender
ConnectedRequest::Announce(request), .try_send_to(worker_index, ConnectedRequest::Announce(request), src)
src, .is_err()
) { {
::log::warn!("request sender full, dropping request"); ::log::warn!("request sender full, dropping request");
} }
} else { } else {
@ -510,11 +501,11 @@ impl SocketWorker {
); );
for (swarm_worker_index, request) in split_requests { for (swarm_worker_index, request) in split_requests {
if let Err(_) = self.request_sender.try_send_to( if self
swarm_worker_index, .request_sender
ConnectedRequest::Scrape(request), .try_send_to(swarm_worker_index, ConnectedRequest::Scrape(request), src)
src, .is_err()
) { {
::log::warn!("request sender full, dropping request"); ::log::warn!("request sender full, dropping request");
} }
} }

1
crates/udp/src/workers/socket/uring/recv_helper.rs
View file

@ -11,6 +11,7 @@ use crate::config::Config;
use super::{SOCKET_IDENTIFIER, USER_DATA_RECV}; use super::{SOCKET_IDENTIFIER, USER_DATA_RECV};
#[allow(clippy::enum_variant_names)]
pub enum Error { pub enum Error {
RecvMsgParseError, RecvMsgParseError,
RecvMsgTruncated, RecvMsgTruncated,

25
crates/udp/src/workers/socket/uring/send_buffers.rs
View file

@ -6,14 +6,15 @@ use std::{
}; };
use aquatic_common::CanonicalSocketAddr; use aquatic_common::CanonicalSocketAddr;
use aquatic_udp_protocol::Response;
use io_uring::opcode::SendMsg; use io_uring::opcode::SendMsg;
use crate::{common::CowResponse, config::Config}; use crate::config::Config;
use super::{RESPONSE_BUF_LEN, SOCKET_IDENTIFIER}; use super::{RESPONSE_BUF_LEN, SOCKET_IDENTIFIER};
pub enum Error<'a> { pub enum Error {
NoBuffers(CowResponse<'a>), NoBuffers(Response),
SerializationFailed(std::io::Error), SerializationFailed(std::io::Error),
} }
@ -57,11 +58,11 @@ impl SendBuffers {
self.likely_next_free_index = 0; self.likely_next_free_index = 0;
} }
pub fn prepare_entry<'a>( pub fn prepare_entry(
&mut self, &mut self,
response: CowResponse<'a>, response: Response,
addr: CanonicalSocketAddr, addr: CanonicalSocketAddr,
) -> Result<io_uring::squeue::Entry, Error<'a>> { ) -> Result<io_uring::squeue::Entry, Error> {
let index = if let Some(index) = self.next_free_index() { let index = if let Some(index) = self.next_free_index() {
index index
} else { } else {
@ -163,7 +164,7 @@ impl SendBuffer {
fn prepare_entry( fn prepare_entry(
&mut self, &mut self,
response: CowResponse, response: Response,
addr: CanonicalSocketAddr, addr: CanonicalSocketAddr,
socket_is_ipv4: bool, socket_is_ipv4: bool,
metadata: &mut SendBufferMetadata, metadata: &mut SendBufferMetadata,
@ -237,12 +238,12 @@ pub enum ResponseType {
} }
impl ResponseType { impl ResponseType {
fn from_response(response: &CowResponse) -> Self { fn from_response(response: &Response) -> Self {
match response { match response {
CowResponse::Connect(_) => Self::Connect, Response::Connect(_) => Self::Connect,
CowResponse::AnnounceIpv4(_) | CowResponse::AnnounceIpv6(_) => Self::Announce, Response::AnnounceIpv4(_) | Response::AnnounceIpv6(_) => Self::Announce,
CowResponse::Scrape(_) => Self::Scrape, Response::Scrape(_) => Self::Scrape,
CowResponse::Error(_) => Self::Error, Response::Error(_) => Self::Error,
} }
} }
} }

6
crates/udp/src/workers/socket/validator.rs
View file

@ -59,8 +59,8 @@ impl ConnectionValidator {
let mut connection_id_bytes = [0u8; 8]; let mut connection_id_bytes = [0u8; 8];
(&mut connection_id_bytes[..4]).copy_from_slice(&elapsed); connection_id_bytes[..4].copy_from_slice(&elapsed);
(&mut connection_id_bytes[4..]).copy_from_slice(&hash); connection_id_bytes[4..].copy_from_slice(&hash);
ConnectionId::new(i64::from_ne_bytes(connection_id_bytes)) ConnectionId::new(i64::from_ne_bytes(connection_id_bytes))
} }
@ -78,7 +78,7 @@ impl ConnectionValidator {
return false; return false;
} }
let tracker_elapsed = u64::from(self.start_time.elapsed().as_secs()); let tracker_elapsed = self.start_time.elapsed().as_secs();
let client_elapsed = u64::from(u32::from_ne_bytes(elapsed)); let client_elapsed = u64::from(u32::from_ne_bytes(elapsed));
let client_expiration_time = client_elapsed + self.max_connection_age; let client_expiration_time = client_elapsed + self.max_connection_age;

145
crates/udp/src/workers/statistics/collector.rs
View file

@ -9,6 +9,18 @@ use serde::Serialize;
use crate::common::Statistics; use crate::common::Statistics;
use crate::config::Config; use crate::config::Config;
#[cfg(feature = "prometheus")]
macro_rules! set_peer_histogram_gauge {
($ip_version:ident, $data:expr, $type_label:expr) => {
::metrics::gauge!(
"aquatic_peers_per_torrent",
"type" => $type_label,
"ip_version" => $ip_version.clone(),
)
.set($data as f64);
};
}
pub struct StatisticsCollector { pub struct StatisticsCollector {
shared: Arc<Statistics>, shared: Arc<Statistics>,
last_update: Instant, last_update: Instant,
@ -79,59 +91,65 @@ impl StatisticsCollector {
if config.statistics.run_prometheus_endpoint { if config.statistics.run_prometheus_endpoint {
::metrics::counter!( ::metrics::counter!(
"aquatic_requests_total", "aquatic_requests_total",
requests_received.try_into().unwrap(),
"ip_version" => self.ip_version.clone(), "ip_version" => self.ip_version.clone(),
); )
.increment(requests_received.try_into().unwrap());
::metrics::counter!( ::metrics::counter!(
"aquatic_responses_total", "aquatic_responses_total",
responses_sent_connect.try_into().unwrap(),
"type" => "connect", "type" => "connect",
"ip_version" => self.ip_version.clone(), "ip_version" => self.ip_version.clone(),
); )
.increment(responses_sent_connect.try_into().unwrap());
::metrics::counter!( ::metrics::counter!(
"aquatic_responses_total", "aquatic_responses_total",
responses_sent_announce.try_into().unwrap(),
"type" => "announce", "type" => "announce",
"ip_version" => self.ip_version.clone(), "ip_version" => self.ip_version.clone(),
); )
.increment(responses_sent_announce.try_into().unwrap());
::metrics::counter!( ::metrics::counter!(
"aquatic_responses_total", "aquatic_responses_total",
responses_sent_scrape.try_into().unwrap(),
"type" => "scrape", "type" => "scrape",
"ip_version" => self.ip_version.clone(), "ip_version" => self.ip_version.clone(),
); )
.increment(responses_sent_scrape.try_into().unwrap());
::metrics::counter!( ::metrics::counter!(
"aquatic_responses_total", "aquatic_responses_total",
responses_sent_error.try_into().unwrap(),
"type" => "error", "type" => "error",
"ip_version" => self.ip_version.clone(), "ip_version" => self.ip_version.clone(),
); )
.increment(responses_sent_error.try_into().unwrap());
::metrics::counter!( ::metrics::counter!(
"aquatic_rx_bytes", "aquatic_rx_bytes",
bytes_received.try_into().unwrap(),
"ip_version" => self.ip_version.clone(), "ip_version" => self.ip_version.clone(),
); )
.increment(bytes_received.try_into().unwrap());
::metrics::counter!( ::metrics::counter!(
"aquatic_tx_bytes", "aquatic_tx_bytes",
bytes_sent.try_into().unwrap(),
"ip_version" => self.ip_version.clone(), "ip_version" => self.ip_version.clone(),
); )
.increment(bytes_sent.try_into().unwrap());
for (worker_index, n) in num_torrents_by_worker.iter().copied().enumerate() { for (worker_index, n) in num_torrents_by_worker.iter().copied().enumerate() {
::metrics::gauge!( ::metrics::gauge!(
"aquatic_torrents", "aquatic_torrents",
n as f64,
"ip_version" => self.ip_version.clone(), "ip_version" => self.ip_version.clone(),
"worker_index" => worker_index.to_string(), "worker_index" => worker_index.to_string(),
); )
.set(n as f64);
} }
for (worker_index, n) in num_peers_by_worker.iter().copied().enumerate() { for (worker_index, n) in num_peers_by_worker.iter().copied().enumerate() {
::metrics::gauge!( ::metrics::gauge!(
"aquatic_peers", "aquatic_peers",
n as f64,
"ip_version" => self.ip_version.clone(), "ip_version" => self.ip_version.clone(),
"worker_index" => worker_index.to_string(), "worker_index" => worker_index.to_string(),
); )
.set(n as f64);
} }
if config.statistics.torrent_peer_histograms { if config.statistics.torrent_peer_histograms {
@ -236,83 +254,18 @@ impl PeerHistogramStatistics {
#[cfg(feature = "prometheus")] #[cfg(feature = "prometheus")]
fn update_metrics(&self, ip_version: String) { fn update_metrics(&self, ip_version: String) {
::metrics::gauge!( set_peer_histogram_gauge!(ip_version, self.min, "min");
"aquatic_peers_per_torrent", set_peer_histogram_gauge!(ip_version, self.p10, "p10");
self.min as f64, set_peer_histogram_gauge!(ip_version, self.p20, "p20");
"type" => "max", set_peer_histogram_gauge!(ip_version, self.p30, "p30");
"ip_version" => ip_version.clone(), set_peer_histogram_gauge!(ip_version, self.p40, "p40");
); set_peer_histogram_gauge!(ip_version, self.p50, "p50");
::metrics::gauge!( set_peer_histogram_gauge!(ip_version, self.p60, "p60");
"aquatic_peers_per_torrent", set_peer_histogram_gauge!(ip_version, self.p70, "p70");
self.p10 as f64, set_peer_histogram_gauge!(ip_version, self.p80, "p80");
"type" => "p10", set_peer_histogram_gauge!(ip_version, self.p90, "p90");
"ip_version" => ip_version.clone(), set_peer_histogram_gauge!(ip_version, self.p99, "p99");
); set_peer_histogram_gauge!(ip_version, self.p999, "p999");
::metrics::gauge!( set_peer_histogram_gauge!(ip_version, self.max, "max");
"aquatic_peers_per_torrent",
self.p20 as f64,
"type" => "p20",
"ip_version" => ip_version.clone(),
);
::metrics::gauge!(
"aquatic_peers_per_torrent",
self.p30 as f64,
"type" => "p30",
"ip_version" => ip_version.clone(),
);
::metrics::gauge!(
"aquatic_peers_per_torrent",
self.p40 as f64,
"type" => "p40",
"ip_version" => ip_version.clone(),
);
::metrics::gauge!(
"aquatic_peers_per_torrent",
self.p50 as f64,
"type" => "p50",
"ip_version" => ip_version.clone(),
);
::metrics::gauge!(
"aquatic_peers_per_torrent",
self.p60 as f64,
"type" => "p60",
"ip_version" => ip_version.clone(),
);
::metrics::gauge!(
"aquatic_peers_per_torrent",
self.p70 as f64,
"type" => "p70",
"ip_version" => ip_version.clone(),
);
::metrics::gauge!(
"aquatic_peers_per_torrent",
self.p80 as f64,
"type" => "p80",
"ip_version" => ip_version.clone(),
);
::metrics::gauge!(
"aquatic_peers_per_torrent",
self.p90 as f64,
"type" => "p90",
"ip_version" => ip_version.clone(),
);
::metrics::gauge!(
"aquatic_peers_per_torrent",
self.p99 as f64,
"type" => "p99",
"ip_version" => ip_version.clone(),
);
::metrics::gauge!(
"aquatic_peers_per_torrent",
self.p999 as f64,
"type" => "p99.9",
"ip_version" => ip_version.clone(),
);
::metrics::gauge!(
"aquatic_peers_per_torrent",
self.max as f64,
"type" => "max",
"ip_version" => ip_version.clone(),
);
} }
} }

8
crates/udp/src/workers/statistics/mod.rs
View file

@ -152,9 +152,9 @@ pub fn run_statistics_worker(
for (prefix, count) in prefixes { for (prefix, count) in prefixes {
::metrics::gauge!( ::metrics::gauge!(
"aquatic_peer_id_prefixes", "aquatic_peer_id_prefixes",
count as f64,
"prefix_hex" => prefix.to_string(), "prefix_hex" => prefix.to_string(),
); )
.set(count as f64);
} }
} }
@ -169,9 +169,9 @@ pub fn run_statistics_worker(
if config.statistics.run_prometheus_endpoint { if config.statistics.run_prometheus_endpoint {
::metrics::gauge!( ::metrics::gauge!(
"aquatic_peer_clients", "aquatic_peer_clients",
count as f64,
"client" => client.to_string(), "client" => client.to_string(),
); )
.set(count as f64);
} }
} }

118
crates/udp/src/workers/swarm/mod.rs
View file

@ -17,24 +17,28 @@ use crate::config::Config;
use storage::TorrentMaps; use storage::TorrentMaps;
pub fn run_swarm_worker( pub struct SwarmWorker {
_sentinel: PanicSentinel, pub _sentinel: PanicSentinel,
config: Config, pub config: Config,
state: State, pub state: State,
server_start_instant: ServerStartInstant, pub server_start_instant: ServerStartInstant,
request_receiver: Receiver<(SocketWorkerIndex, ConnectedRequest, CanonicalSocketAddr)>, pub request_receiver: Receiver<(SocketWorkerIndex, ConnectedRequest, CanonicalSocketAddr)>,
mut response_sender: ConnectedResponseSender, pub response_sender: ConnectedResponseSender,
statistics_sender: Sender<StatisticsMessage>, pub statistics_sender: Sender<StatisticsMessage>,
worker_index: SwarmWorkerIndex, pub worker_index: SwarmWorkerIndex,
) { }
impl SwarmWorker {
pub fn run(&mut self) {
let mut torrents = TorrentMaps::default(); let mut torrents = TorrentMaps::default();
let mut rng = SmallRng::from_entropy(); let mut rng = SmallRng::from_entropy();
let timeout = Duration::from_millis(config.request_channel_recv_timeout_ms); let timeout = Duration::from_millis(self.config.request_channel_recv_timeout_ms);
let mut peer_valid_until = ValidUntil::new(server_start_instant, config.cleaning.max_peer_age); let mut peer_valid_until =
ValidUntil::new(self.server_start_instant, self.config.cleaning.max_peer_age);
let cleaning_interval = Duration::from_secs(config.cleaning.torrent_cleaning_interval); let cleaning_interval = Duration::from_secs(self.config.cleaning.torrent_cleaning_interval);
let statistics_update_interval = Duration::from_secs(config.statistics.interval); let statistics_update_interval = Duration::from_secs(self.config.statistics.interval);
let mut last_cleaning = Instant::now(); let mut last_cleaning = Instant::now();
let mut last_statistics_update = Instant::now(); let mut last_statistics_update = Instant::now();
@ -42,77 +46,63 @@ pub fn run_swarm_worker(
let mut iter_counter = 0usize; let mut iter_counter = 0usize;
loop { loop {
if let Ok((sender_index, request, src)) = request_receiver.recv_timeout(timeout) { if let Ok((sender_index, request, src)) = self.request_receiver.recv_timeout(timeout) {
// It is OK to block here as long as we don't also do blocking // It is OK to block here as long as we don't also do blocking
// sends in socket workers (doing both could cause a deadlock) // sends in socket workers (doing both could cause a deadlock)
match (request, src.get().ip()) { match (request, src.get().ip()) {
(ConnectedRequest::Announce(request), IpAddr::V4(ip)) => { (ConnectedRequest::Announce(request), IpAddr::V4(ip)) => {
// It doesn't matter which socket worker receives announce responses let response = torrents
let mut send_ref = response_sender
.send_ref_to_any()
.expect("swarm response channel is closed");
send_ref.addr = src;
send_ref.kind = ConnectedResponseKind::AnnounceIpv4;
torrents
.ipv4 .ipv4
.0 .0
.entry(request.info_hash) .entry(request.info_hash)
.or_default() .or_default()
.announce( .announce(
&config, &self.config,
&statistics_sender, &self.statistics_sender,
&mut rng, &mut rng,
&request, &request,
ip.into(), ip.into(),
peer_valid_until, peer_valid_until,
&mut send_ref.announce_ipv4,
); );
// It doesn't matter which socket worker receives announce responses
self.response_sender
.send_to_any(src, ConnectedResponse::AnnounceIpv4(response))
.expect("swarm response channel is closed");
} }
(ConnectedRequest::Announce(request), IpAddr::V6(ip)) => { (ConnectedRequest::Announce(request), IpAddr::V6(ip)) => {
// It doesn't matter which socket worker receives announce responses let response = torrents
let mut send_ref = response_sender
.send_ref_to_any()
.expect("swarm response channel is closed");
send_ref.addr = src;
send_ref.kind = ConnectedResponseKind::AnnounceIpv6;
torrents
.ipv6 .ipv6
.0 .0
.entry(request.info_hash) .entry(request.info_hash)
.or_default() .or_default()
.announce( .announce(
&config, &self.config,
&statistics_sender, &self.statistics_sender,
&mut rng, &mut rng,
&request, &request,
ip.into(), ip.into(),
peer_valid_until, peer_valid_until,
&mut send_ref.announce_ipv6,
); );
// It doesn't matter which socket worker receives announce responses
self.response_sender
.send_to_any(src, ConnectedResponse::AnnounceIpv6(response))
.expect("swarm response channel is closed");
} }
(ConnectedRequest::Scrape(request), IpAddr::V4(_)) => { (ConnectedRequest::Scrape(request), IpAddr::V4(_)) => {
let mut send_ref = response_sender let response = torrents.ipv4.scrape(request);
.send_ref_to(sender_index)
self.response_sender
.send_to(sender_index, src, ConnectedResponse::Scrape(response))
.expect("swarm response channel is closed"); .expect("swarm response channel is closed");
send_ref.addr = src;
send_ref.kind = ConnectedResponseKind::Scrape;
torrents.ipv4.scrape(request, &mut send_ref.scrape);
} }
(ConnectedRequest::Scrape(request), IpAddr::V6(_)) => { (ConnectedRequest::Scrape(request), IpAddr::V6(_)) => {
let mut send_ref = response_sender let response = torrents.ipv6.scrape(request);
.send_ref_to(sender_index)
self.response_sender
.send_to(sender_index, src, ConnectedResponse::Scrape(response))
.expect("swarm response channel is closed"); .expect("swarm response channel is closed");
send_ref.addr = src;
send_ref.kind = ConnectedResponseKind::Scrape;
torrents.ipv6.scrape(request, &mut send_ref.scrape);
} }
}; };
} }
@ -121,26 +111,27 @@ pub fn run_swarm_worker(
if iter_counter % 128 == 0 { if iter_counter % 128 == 0 {
let now = Instant::now(); let now = Instant::now();
peer_valid_until = ValidUntil::new(server_start_instant, config.cleaning.max_peer_age); peer_valid_until =
ValidUntil::new(self.server_start_instant, self.config.cleaning.max_peer_age);
if now > last_cleaning + cleaning_interval { if now > last_cleaning + cleaning_interval {
torrents.clean_and_update_statistics( torrents.clean_and_update_statistics(
&config, &self.config,
&state, &self.state,
&statistics_sender, &self.statistics_sender,
&state.access_list, &self.state.access_list,
server_start_instant, self.server_start_instant,
worker_index, self.worker_index,
); );
last_cleaning = now; last_cleaning = now;
} }
if config.statistics.active() if self.config.statistics.active()
&& now > last_statistics_update + statistics_update_interval && now > last_statistics_update + statistics_update_interval
{ {
state.statistics_ipv4.torrents[worker_index.0] self.state.statistics_ipv4.torrents[self.worker_index.0]
.store(torrents.ipv4.num_torrents(), Ordering::Release); .store(torrents.ipv4.num_torrents(), Ordering::Release);
state.statistics_ipv6.torrents[worker_index.0] self.state.statistics_ipv6.torrents[self.worker_index.0]
.store(torrents.ipv6.num_torrents(), Ordering::Release); .store(torrents.ipv6.num_torrents(), Ordering::Release);
last_statistics_update = now; last_statistics_update = now;
@ -149,4 +140,5 @@ pub fn run_swarm_worker(
iter_counter = iter_counter.wrapping_add(1); iter_counter = iter_counter.wrapping_add(1);
} }
}
} }

423
crates/udp/src/workers/swarm/storage.rs
View file

@ -10,6 +10,7 @@ use aquatic_common::{
}; };
use aquatic_udp_protocol::*; use aquatic_udp_protocol::*;
use arrayvec::ArrayVec;
use crossbeam_channel::Sender; use crossbeam_channel::Sender;
use hdrhistogram::Histogram; use hdrhistogram::Histogram;
use rand::prelude::SmallRng; use rand::prelude::SmallRng;
@ -18,6 +19,8 @@ use rand::Rng;
use crate::common::*; use crate::common::*;
use crate::config::Config; use crate::config::Config;
const SMALL_PEER_MAP_CAPACITY: usize = 2;
pub struct TorrentMaps { pub struct TorrentMaps {
pub ipv4: TorrentMap<Ipv4AddrBytes>, pub ipv4: TorrentMap<Ipv4AddrBytes>,
pub ipv6: TorrentMap<Ipv6AddrBytes>, pub ipv6: TorrentMap<Ipv6AddrBytes>,
@ -76,20 +79,29 @@ impl TorrentMaps {
pub struct TorrentMap<I: Ip>(pub IndexMap<InfoHash, TorrentData<I>>); pub struct TorrentMap<I: Ip>(pub IndexMap<InfoHash, TorrentData<I>>);
impl<I: Ip> TorrentMap<I> { impl<I: Ip> TorrentMap<I> {
pub fn scrape(&mut self, request: PendingScrapeRequest, response: &mut PendingScrapeResponse) { pub fn scrape(&mut self, request: PendingScrapeRequest) -> PendingScrapeResponse {
response.slab_key = request.slab_key; let torrent_stats = request
.info_hashes
let torrent_stats = request.info_hashes.into_iter().map(|(i, info_hash)| { .into_iter()
.map(|(i, info_hash)| {
let stats = self let stats = self
.0 .0
.get(&info_hash) .get(&info_hash)
.map(|torrent_data| torrent_data.scrape_statistics()) .map(|torrent_data| torrent_data.scrape_statistics())
.unwrap_or_else(|| create_torrent_scrape_statistics(0, 0)); .unwrap_or_else(|| TorrentScrapeStatistics {
seeders: NumberOfPeers::new(0),
(i, stats) leechers: NumberOfPeers::new(0),
completed: NumberOfDownloads::new(0),
}); });
response.torrent_stats.extend(torrent_stats); (i, stats)
})
.collect();
PendingScrapeResponse {
slab_key: request.slab_key,
torrent_stats,
}
} }
/// Remove forbidden or inactive torrents, reclaim space and return number of remaining peers /// Remove forbidden or inactive torrents, reclaim space and return number of remaining peers
fn clean_and_get_statistics( fn clean_and_get_statistics(
@ -100,7 +112,7 @@ impl<I: Ip> TorrentMap<I> {
access_list_mode: AccessListMode, access_list_mode: AccessListMode,
now: SecondsSinceServerStart, now: SecondsSinceServerStart,
) -> (usize, Option<Histogram<u64>>) { ) -> (usize, Option<Histogram<u64>>) {
let mut num_peers = 0; let mut total_num_peers = 0;
let mut opt_histogram: Option<Histogram<u64>> = if config.statistics.torrent_peer_histograms let mut opt_histogram: Option<Histogram<u64>> = if config.statistics.torrent_peer_histograms
{ {
@ -124,17 +136,27 @@ impl<I: Ip> TorrentMap<I> {
return false; return false;
} }
torrent.clean(config, statistics_sender, now); let num_peers = match torrent {
TorrentData::Small(peer_map) => {
peer_map.clean_and_get_num_peers(config, statistics_sender, now)
}
TorrentData::Large(peer_map) => {
let num_peers =
peer_map.clean_and_get_num_peers(config, statistics_sender, now);
num_peers += torrent.peers.len(); if let Some(peer_map) = peer_map.try_shrink() {
*torrent = TorrentData::Small(peer_map);
}
num_peers
}
};
total_num_peers += num_peers;
match opt_histogram { match opt_histogram {
Some(ref mut histogram) if torrent.peers.len() != 0 => { Some(ref mut histogram) if num_peers > 0 => {
let n = torrent let n = num_peers.try_into().expect("Couldn't fit usize into u64");
.peers
.len()
.try_into()
.expect("Couldn't fit usize into u64");
if let Err(err) = histogram.record(n) { if let Err(err) = histogram.record(n) {
::log::error!("Couldn't record {} to histogram: {:#}", n, err); ::log::error!("Couldn't record {} to histogram: {:#}", n, err);
@ -143,12 +165,12 @@ impl<I: Ip> TorrentMap<I> {
_ => (), _ => (),
} }
!torrent.peers.is_empty() num_peers > 0
}); });
self.0.shrink_to_fit(); self.0.shrink_to_fit();
(num_peers, opt_histogram) (total_num_peers, opt_histogram)
} }
pub fn num_torrents(&self) -> usize { pub fn num_torrents(&self) -> usize {
@ -156,9 +178,9 @@ impl<I: Ip> TorrentMap<I> {
} }
} }
pub struct TorrentData<I: Ip> { pub enum TorrentData<I: Ip> {
peers: IndexMap<ResponsePeer<I>, Peer>, Small(SmallPeerMap<I>),
num_seeders: usize, Large(LargePeerMap<I>),
} }
impl<I: Ip> TorrentData<I> { impl<I: Ip> TorrentData<I> {
@ -170,8 +192,7 @@ impl<I: Ip> TorrentData<I> {
request: &AnnounceRequest, request: &AnnounceRequest,
ip_address: I, ip_address: I,
valid_until: ValidUntil, valid_until: ValidUntil,
response: &mut AnnounceResponse<I>, ) -> AnnounceResponse<I> {
) {
let max_num_peers_to_take: usize = if request.peers_wanted.0.get() <= 0 { let max_num_peers_to_take: usize = if request.peers_wanted.0.get() <= 0 {
config.protocol.max_response_peers config.protocol.max_response_peers
} else { } else {
@ -189,60 +210,77 @@ impl<I: Ip> TorrentData<I> {
port: request.port, port: request.port,
}; };
let opt_removed_peer = self.peers.remove(&peer_map_key);
if let Some(Peer {
is_seeder: true, ..
}) = opt_removed_peer
{
self.num_seeders -= 1;
}
// Create the response before inserting the peer. This means that we // Create the response before inserting the peer. This means that we
// don't have to filter it out from the response peers, and that the // don't have to filter it out from the response peers, and that the
// reported number of seeders/leechers will not include it // reported number of seeders/leechers will not include it
let (response, opt_removed_peer) = match self {
Self::Small(peer_map) => {
let opt_removed_peer = peer_map.remove(&peer_map_key);
response.fixed = AnnounceResponseFixedData { let (seeders, leechers) = peer_map.num_seeders_leechers();
let response = AnnounceResponse {
fixed: AnnounceResponseFixedData {
transaction_id: request.transaction_id, transaction_id: request.transaction_id,
announce_interval: AnnounceInterval::new(config.protocol.peer_announce_interval), announce_interval: AnnounceInterval::new(
leechers: NumberOfPeers::new(self.num_leechers().try_into().unwrap_or(i32::MAX)), config.protocol.peer_announce_interval,
seeders: NumberOfPeers::new(self.num_seeders().try_into().unwrap_or(i32::MAX)), ),
leechers: NumberOfPeers::new(leechers.try_into().unwrap_or(i32::MAX)),
seeders: NumberOfPeers::new(seeders.try_into().unwrap_or(i32::MAX)),
},
peers: peer_map.extract_response_peers(max_num_peers_to_take),
}; };
extract_response_peers( // Convert peer map to large variant if it is full and
rng, // announcing peer is not stopped and will therefore be
&self.peers, // inserted
max_num_peers_to_take, if peer_map.is_full() && status != PeerStatus::Stopped {
|k, _| *k, *self = Self::Large(peer_map.to_large());
&mut response.peers, }
);
(response, opt_removed_peer)
}
Self::Large(peer_map) => {
let opt_removed_peer = peer_map.remove_peer(&peer_map_key);
let (seeders, leechers) = peer_map.num_seeders_leechers();
let response = AnnounceResponse {
fixed: AnnounceResponseFixedData {
transaction_id: request.transaction_id,
announce_interval: AnnounceInterval::new(
config.protocol.peer_announce_interval,
),
leechers: NumberOfPeers::new(leechers.try_into().unwrap_or(i32::MAX)),
seeders: NumberOfPeers::new(seeders.try_into().unwrap_or(i32::MAX)),
},
peers: peer_map.extract_response_peers(rng, max_num_peers_to_take),
};
// Try shrinking the map if announcing peer is stopped and
// will therefore not be inserted
if status == PeerStatus::Stopped {
if let Some(peer_map) = peer_map.try_shrink() {
*self = Self::Small(peer_map);
}
}
(response, opt_removed_peer)
}
};
match status { match status {
PeerStatus::Leeching => { PeerStatus::Leeching | PeerStatus::Seeding => {
let peer = Peer { let peer = Peer {
peer_id: request.peer_id, peer_id: request.peer_id,
is_seeder: false, is_seeder: status == PeerStatus::Seeding,
valid_until, valid_until,
}; };
self.peers.insert(peer_map_key, peer); match self {
Self::Small(peer_map) => peer_map.insert(peer_map_key, peer),
if config.statistics.peer_clients && opt_removed_peer.is_none() { Self::Large(peer_map) => peer_map.insert(peer_map_key, peer),
statistics_sender
.try_send(StatisticsMessage::PeerAdded(request.peer_id))
.expect("statistics channel should be unbounded");
} }
}
PeerStatus::Seeding => {
let peer = Peer {
peer_id: request.peer_id,
is_seeder: true,
valid_until,
};
self.peers.insert(peer_map_key, peer);
self.num_seeders += 1;
if config.statistics.peer_clients && opt_removed_peer.is_none() { if config.statistics.peer_clients && opt_removed_peer.is_none() {
statistics_sender statistics_sender
@ -258,94 +296,147 @@ impl<I: Ip> TorrentData<I> {
} }
} }
}; };
}
pub fn num_leechers(&self) -> usize { response
self.peers.len() - self.num_seeders
}
pub fn num_seeders(&self) -> usize {
self.num_seeders
} }
pub fn scrape_statistics(&self) -> TorrentScrapeStatistics { pub fn scrape_statistics(&self) -> TorrentScrapeStatistics {
create_torrent_scrape_statistics( let (seeders, leechers) = match self {
self.num_seeders.try_into().unwrap_or(i32::MAX), Self::Small(peer_map) => peer_map.num_seeders_leechers(),
self.num_leechers().try_into().unwrap_or(i32::MAX), Self::Large(peer_map) => peer_map.num_seeders_leechers(),
) };
}
/// Remove inactive peers and reclaim space TorrentScrapeStatistics {
fn clean( seeders: NumberOfPeers::new(seeders.try_into().unwrap_or(i32::MAX)),
&mut self, leechers: NumberOfPeers::new(leechers.try_into().unwrap_or(i32::MAX)),
config: &Config, completed: NumberOfDownloads::new(0),
statistics_sender: &Sender<StatisticsMessage>,
now: SecondsSinceServerStart,
) {
self.peers.retain(|_, peer| {
let keep = peer.valid_until.valid(now);
if !keep {
if peer.is_seeder {
self.num_seeders -= 1;
}
if config.statistics.peer_clients {
if let Err(_) =
statistics_sender.try_send(StatisticsMessage::PeerRemoved(peer.peer_id))
{
// Should never happen in practice
::log::error!("Couldn't send StatisticsMessage::PeerRemoved");
}
}
}
keep
});
if !self.peers.is_empty() {
self.peers.shrink_to_fit();
} }
} }
} }
impl<I: Ip> Default for TorrentData<I> { impl<I: Ip> Default for TorrentData<I> {
fn default() -> Self { fn default() -> Self {
Self { Self::Small(SmallPeerMap(ArrayVec::default()))
peers: Default::default(),
num_seeders: 0,
}
} }
} }
#[derive(Clone, Debug)] /// Store torrents with up to two peers without an extra heap allocation
struct Peer { ///
peer_id: PeerId, /// On public open trackers, this is likely to be the majority of torrents.
is_seeder: bool, #[derive(Default, Debug)]
valid_until: ValidUntil, pub struct SmallPeerMap<I: Ip>(ArrayVec<(ResponsePeer<I>, Peer), SMALL_PEER_MAP_CAPACITY>);
impl<I: Ip> SmallPeerMap<I> {
fn is_full(&self) -> bool {
self.0.is_full()
}
fn num_seeders_leechers(&self) -> (usize, usize) {
let seeders = self.0.iter().filter(|(_, p)| p.is_seeder).count();
let leechers = self.0.len() - seeders;
(seeders, leechers)
}
fn insert(&mut self, key: ResponsePeer<I>, peer: Peer) {
self.0.push((key, peer));
}
fn remove(&mut self, key: &ResponsePeer<I>) -> Option<Peer> {
for (i, (k, _)) in self.0.iter().enumerate() {
if k == key {
return Some(self.0.remove(i).1);
}
}
None
}
fn extract_response_peers(&self, max_num_peers_to_take: usize) -> Vec<ResponsePeer<I>> {
Vec::from_iter(self.0.iter().take(max_num_peers_to_take).map(|(k, _)| *k))
}
fn clean_and_get_num_peers(
&mut self,
config: &Config,
statistics_sender: &Sender<StatisticsMessage>,
now: SecondsSinceServerStart,
) -> usize {
self.0.retain(|(_, peer)| {
let keep = peer.valid_until.valid(now);
if !keep
&& config.statistics.peer_clients
&& statistics_sender
.try_send(StatisticsMessage::PeerRemoved(peer.peer_id))
.is_err()
{
// Should never happen in practice
::log::error!("Couldn't send StatisticsMessage::PeerRemoved");
}
keep
});
self.0.len()
}
fn to_large(&self) -> LargePeerMap<I> {
let (num_seeders, _) = self.num_seeders_leechers();
let peers = self.0.iter().copied().collect();
LargePeerMap { peers, num_seeders }
}
} }
/// Extract response peers #[derive(Default)]
/// pub struct LargePeerMap<I: Ip> {
/// If there are more peers in map than `max_num_peers_to_take`, do a random peers: IndexMap<ResponsePeer<I>, Peer>,
/// selection of peers from first and second halves of map in order to avoid num_seeders: usize,
/// returning too homogeneous peers. }
///
/// Does NOT filter out announcing peer. impl<I: Ip> LargePeerMap<I> {
#[inline] fn num_seeders_leechers(&self) -> (usize, usize) {
pub fn extract_response_peers<K, V, R, F>( (self.num_seeders, self.peers.len() - self.num_seeders)
}
fn insert(&mut self, key: ResponsePeer<I>, peer: Peer) {
if peer.is_seeder {
self.num_seeders += 1;
}
self.peers.insert(key, peer);
}
fn remove_peer(&mut self, key: &ResponsePeer<I>) -> Option<Peer> {
let opt_removed_peer = self.peers.remove(key);
if let Some(Peer {
is_seeder: true, ..
}) = opt_removed_peer
{
self.num_seeders -= 1;
}
opt_removed_peer
}
/// Extract response peers
///
/// If there are more peers in map than `max_num_peers_to_take`, do a random
/// selection of peers from first and second halves of map in order to avoid
/// returning too homogeneous peers.
///
/// Does NOT filter out announcing peer.
pub fn extract_response_peers(
&self,
rng: &mut impl Rng, rng: &mut impl Rng,
peer_map: &IndexMap<K, V>,
max_num_peers_to_take: usize, max_num_peers_to_take: usize,
peer_conversion_function: F, ) -> Vec<ResponsePeer<I>> {
peers: &mut Vec<R>, if self.peers.len() <= max_num_peers_to_take {
) where self.peers.keys().copied().collect()
K: Eq + ::std::hash::Hash,
F: Fn(&K, &V) -> R,
{
if peer_map.len() <= max_num_peers_to_take {
peers.extend(peer_map.iter().map(|(k, v)| peer_conversion_function(k, v)));
} else { } else {
let middle_index = peer_map.len() / 2; let middle_index = self.peers.len() / 2;
let num_to_take_per_half = max_num_peers_to_take / 2; let num_to_take_per_half = max_num_peers_to_take / 2;
let offset_half_one = { let offset_half_one = {
@ -356,7 +447,7 @@ pub fn extract_response_peers<K, V, R, F>(
}; };
let offset_half_two = { let offset_half_two = {
let from = middle_index; let from = middle_index;
let to = usize::max(middle_index + 1, peer_map.len() - num_to_take_per_half); let to = usize::max(middle_index + 1, self.peers.len() - num_to_take_per_half);
rng.gen_range(from..to) rng.gen_range(from..to)
}; };
@ -364,20 +455,64 @@ pub fn extract_response_peers<K, V, R, F>(
let end_half_one = offset_half_one + num_to_take_per_half; let end_half_one = offset_half_one + num_to_take_per_half;
let end_half_two = offset_half_two + num_to_take_per_half; let end_half_two = offset_half_two + num_to_take_per_half;
if let Some(slice) = peer_map.get_range(offset_half_one..end_half_one) { let mut peers = Vec::with_capacity(max_num_peers_to_take);
peers.extend(slice.iter().map(|(k, v)| peer_conversion_function(k, v)));
if let Some(slice) = self.peers.get_range(offset_half_one..end_half_one) {
peers.extend(slice.keys());
} }
if let Some(slice) = peer_map.get_range(offset_half_two..end_half_two) { if let Some(slice) = self.peers.get_range(offset_half_two..end_half_two) {
peers.extend(slice.iter().map(|(k, v)| peer_conversion_function(k, v))); peers.extend(slice.keys());
} }
peers
}
}
fn clean_and_get_num_peers(
&mut self,
config: &Config,
statistics_sender: &Sender<StatisticsMessage>,
now: SecondsSinceServerStart,
) -> usize {
self.peers.retain(|_, peer| {
let keep = peer.valid_until.valid(now);
if !keep {
if peer.is_seeder {
self.num_seeders -= 1;
}
if config.statistics.peer_clients
&& statistics_sender
.try_send(StatisticsMessage::PeerRemoved(peer.peer_id))
.is_err()
{
// Should never happen in practice
::log::error!("Couldn't send StatisticsMessage::PeerRemoved");
}
}
keep
});
if !self.peers.is_empty() {
self.peers.shrink_to_fit();
}
self.peers.len()
}
fn try_shrink(&mut self) -> Option<SmallPeerMap<I>> {
(self.peers.len() <= SMALL_PEER_MAP_CAPACITY).then(|| {
SmallPeerMap(ArrayVec::from_iter(
self.peers.iter().map(|(k, v)| (*k, *v)),
))
})
} }
} }
#[inline(always)] #[derive(Clone, Copy, Debug)]
fn create_torrent_scrape_statistics(seeders: i32, leechers: i32) -> TorrentScrapeStatistics { struct Peer {
TorrentScrapeStatistics { peer_id: PeerId,
seeders: NumberOfPeers::new(seeders), is_seeder: bool,
completed: NumberOfDownloads::new(0), // No implementation planned valid_until: ValidUntil,
leechers: NumberOfPeers::new(leechers),
}
} }

10
crates/udp/tests/common/mod.rs
View file

@ -29,7 +29,7 @@ pub fn connect(socket: &UdpSocket, tracker_addr: SocketAddr) -> anyhow::Result<C
transaction_id: TransactionId::new(0), transaction_id: TransactionId::new(0),
}); });
let response = request_and_response(&socket, tracker_addr, request)?; let response = request_and_response(socket, tracker_addr, request)?;
if let Response::Connect(response) = response { if let Response::Connect(response) = response {
Ok(response.connection_id) Ok(response.connection_id)
@ -69,7 +69,7 @@ pub fn announce(
port: Port::new(peer_port), port: Port::new(peer_port),
}); });
Ok(request_and_response(&socket, tracker_addr, request)?) request_and_response(socket, tracker_addr, request)
} }
pub fn scrape( pub fn scrape(
@ -84,12 +84,12 @@ pub fn scrape(
info_hashes, info_hashes,
}); });
let response = request_and_response(&socket, tracker_addr, request)?; let response = request_and_response(socket, tracker_addr, request)?;
if let Response::Scrape(response) = response { if let Response::Scrape(response) = response {
Ok(response) Ok(response)
} else { } else {
return Err(anyhow::anyhow!("not scrape response: {:?}", response)); Err(anyhow::anyhow!("not scrape response: {:?}", response))
} }
} }
@ -119,6 +119,6 @@ pub fn request_and_response(
.recv_from(&mut buffer) .recv_from(&mut buffer)
.with_context(|| "recv response")?; .with_context(|| "recv response")?;
Ok(Response::from_bytes(&buffer[..bytes_read], true).with_context(|| "parse response")?) Response::from_bytes(&buffer[..bytes_read], true).with_context(|| "parse response")
} }
} }

4
crates/udp_load_test/src/lib.rs
View file

@ -63,12 +63,10 @@ pub fn run(config: Config) -> ::anyhow::Result<()> {
let ip = if config.server_address.is_ipv6() { let ip = if config.server_address.is_ipv6() {
Ipv6Addr::LOCALHOST.into() Ipv6Addr::LOCALHOST.into()
} else { } else if config.network.multiple_client_ipv4s {
if config.network.multiple_client_ipv4s {
Ipv4Addr::new(127, 0, 0, 1 + i).into() Ipv4Addr::new(127, 0, 0, 1 + i).into()
} else { } else {
Ipv4Addr::LOCALHOST.into() Ipv4Addr::LOCALHOST.into()
}
}; };
let addr = SocketAddr::new(ip, port); let addr = SocketAddr::new(ip, port);

24
crates/udp_protocol/src/common.rs
View file

@ -104,15 +104,15 @@ pub struct Ipv4AddrBytes(pub [u8; 4]);
impl Ip for Ipv4AddrBytes {} impl Ip for Ipv4AddrBytes {}
impl Into<Ipv4Addr> for Ipv4AddrBytes { impl From<Ipv4AddrBytes> for Ipv4Addr {
fn into(self) -> Ipv4Addr { fn from(val: Ipv4AddrBytes) -> Self {
Ipv4Addr::from(self.0) Ipv4Addr::from(val.0)
} }
} }
impl Into<Ipv4AddrBytes> for Ipv4Addr { impl From<Ipv4Addr> for Ipv4AddrBytes {
fn into(self) -> Ipv4AddrBytes { fn from(val: Ipv4Addr) -> Self {
Ipv4AddrBytes(self.octets()) Ipv4AddrBytes(val.octets())
} }
} }
@ -122,15 +122,15 @@ pub struct Ipv6AddrBytes(pub [u8; 16]);
impl Ip for Ipv6AddrBytes {} impl Ip for Ipv6AddrBytes {}
impl Into<Ipv6Addr> for Ipv6AddrBytes { impl From<Ipv6AddrBytes> for Ipv6Addr {
fn into(self) -> Ipv6Addr { fn from(val: Ipv6AddrBytes) -> Self {
Ipv6Addr::from(self.0) Ipv6Addr::from(val.0)
} }
} }
impl Into<Ipv6AddrBytes> for Ipv6Addr { impl From<Ipv6Addr> for Ipv6AddrBytes {
fn into(self) -> Ipv6AddrBytes { fn from(val: Ipv6Addr) -> Self {
Ipv6AddrBytes(self.octets()) Ipv6AddrBytes(val.octets())
} }
} }

2
crates/udp_protocol/src/response.rs
View file

@ -85,7 +85,7 @@ impl Response {
// Error // Error
3 => { 3 => {
let transaction_id = read_i32_ne(&mut bytes).map(TransactionId)?; let transaction_id = read_i32_ne(&mut bytes).map(TransactionId)?;
let message = String::from_utf8_lossy(&bytes).into_owned().into(); let message = String::from_utf8_lossy(bytes).into_owned().into();
Ok((ErrorResponse { Ok((ErrorResponse {
transaction_id, transaction_id,

10
crates/ws/src/common.rs
View file

@ -57,12 +57,12 @@ pub struct OutMessageMeta {
pub pending_scrape_id: Option<PendingScrapeId>, pub pending_scrape_id: Option<PendingScrapeId>,
} }
impl Into<OutMessageMeta> for InMessageMeta { impl From<InMessageMeta> for OutMessageMeta {
fn into(self) -> OutMessageMeta { fn from(val: InMessageMeta) -> Self {
OutMessageMeta { OutMessageMeta {
out_message_consumer_id: self.out_message_consumer_id, out_message_consumer_id: val.out_message_consumer_id,
connection_id: self.connection_id, connection_id: val.connection_id,
pending_scrape_id: self.pending_scrape_id, pending_scrape_id: val.pending_scrape_id,
} }
} }
} }

13
crates/ws/src/workers/socket/connection.rs
View file

@ -41,6 +41,9 @@ use crate::workers::socket::calculate_in_message_consumer_index;
#[cfg(feature = "metrics")] #[cfg(feature = "metrics")]
use crate::workers::socket::{ip_version_to_metrics_str, WORKER_INDEX}; use crate::workers::socket::{ip_version_to_metrics_str, WORKER_INDEX};
/// Optional second tuple field is for peer id hex representation
type PeerClientGauge = (Gauge, Option<Gauge>);
pub struct ConnectionRunner { pub struct ConnectionRunner {
pub config: Rc<Config>, pub config: Rc<Config>,
pub access_list: Arc<AccessListArcSwap>, pub access_list: Arc<AccessListArcSwap>,
@ -77,7 +80,7 @@ impl ConnectionRunner {
clean_up_data.before_open(); clean_up_data.before_open();
let config = self.config.clone(); let config = self.config.clone();
let connection_id = self.connection_id.clone(); let connection_id = self.connection_id;
race( race(
async { async {
@ -283,6 +286,8 @@ impl<S: futures::AsyncRead + futures::AsyncWrite + Unpin> ConnectionReader<S> {
} }
} }
// Silence RefCell lint due to false positives
#[allow(clippy::await_holding_refcell_ref)]
async fn handle_announce_request(&mut self, request: AnnounceRequest) -> anyhow::Result<()> { async fn handle_announce_request(&mut self, request: AnnounceRequest) -> anyhow::Result<()> {
#[cfg(feature = "metrics")] #[cfg(feature = "metrics")]
self.total_announce_requests_counter.increment(1); self.total_announce_requests_counter.increment(1);
@ -485,6 +490,8 @@ struct ConnectionWriter<S> {
} }
impl<S: futures::AsyncRead + futures::AsyncWrite + Unpin> ConnectionWriter<S> { impl<S: futures::AsyncRead + futures::AsyncWrite + Unpin> ConnectionWriter<S> {
// Silence RefCell lint due to false positives
#[allow(clippy::await_holding_refcell_ref)]
async fn run_out_message_loop(&mut self) -> anyhow::Result<()> { async fn run_out_message_loop(&mut self) -> anyhow::Result<()> {
loop { loop {
let (meta, out_message) = self.out_message_receiver.recv().await.ok_or_else(|| { let (meta, out_message) = self.out_message_receiver.recv().await.ok_or_else(|| {
@ -590,7 +597,7 @@ impl<S: futures::AsyncRead + futures::AsyncWrite + Unpin> ConnectionWriter<S> {
struct ConnectionCleanupData { struct ConnectionCleanupData {
announced_info_hashes: Rc<RefCell<HashMap<InfoHash, PeerId>>>, announced_info_hashes: Rc<RefCell<HashMap<InfoHash, PeerId>>>,
ip_version: IpVersion, ip_version: IpVersion,
opt_peer_client: Rc<RefCell<Option<(Gauge, Option<Gauge>)>>>, opt_peer_client: Rc<RefCell<Option<PeerClientGauge>>>,
#[cfg(feature = "metrics")] #[cfg(feature = "metrics")]
active_connections_gauge: Gauge, active_connections_gauge: Gauge,
} }
@ -608,7 +615,7 @@ impl ConnectionCleanupData {
let mut announced_info_hashes = HashMap::new(); let mut announced_info_hashes = HashMap::new();
for (info_hash, peer_id) in self.announced_info_hashes.take().into_iter() { for (info_hash, peer_id) in self.announced_info_hashes.take().into_iter() {
let consumer_index = calculate_in_message_consumer_index(&config, info_hash); let consumer_index = calculate_in_message_consumer_index(config, info_hash);
announced_info_hashes announced_info_hashes
.entry(consumer_index) .entry(consumer_index)

1
crates/ws/src/workers/socket/mod.rs
View file

@ -48,6 +48,7 @@ struct ConnectionHandle {
valid_until_after_tls_update: Option<ValidUntil>, valid_until_after_tls_update: Option<ValidUntil>,
} }
#[allow(clippy::too_many_arguments)]
pub async fn run_socket_worker( pub async fn run_socket_worker(
_sentinel: PanicSentinel, _sentinel: PanicSentinel,
config: Config, config: Config,

5
crates/ws/src/workers/swarm/mod.rs
View file

@ -24,6 +24,7 @@ use self::storage::TorrentMaps;
#[cfg(feature = "metrics")] #[cfg(feature = "metrics")]
thread_local! { static WORKER_INDEX: ::std::cell::Cell<usize> = Default::default() } thread_local! { static WORKER_INDEX: ::std::cell::Cell<usize> = Default::default() }
#[allow(clippy::too_many_arguments)]
pub async fn run_swarm_worker( pub async fn run_swarm_worker(
_sentinel: PanicSentinel, _sentinel: PanicSentinel,
config: Config, config: Config,
@ -140,7 +141,7 @@ async fn handle_request_stream<S>(
match in_message { match in_message {
InMessage::AnnounceRequest(request) => { InMessage::AnnounceRequest(request) => {
torrents.borrow_mut().handle_announce_request( torrents.borrow_mut().handle_announce_request(
&config, config,
&mut rng.borrow_mut(), &mut rng.borrow_mut(),
&mut out_messages, &mut out_messages,
server_start_instant, server_start_instant,
@ -150,7 +151,7 @@ async fn handle_request_stream<S>(
} }
InMessage::ScrapeRequest(request) => torrents InMessage::ScrapeRequest(request) => torrents
.borrow_mut() .borrow_mut()
.handle_scrape_request(&config, &mut out_messages, meta, request), .handle_scrape_request(config, &mut out_messages, meta, request),
}; };
for (meta, out_message) in out_messages { for (meta, out_message) in out_messages {

10
crates/ws/src/workers/swarm/storage.rs
View file

@ -248,7 +248,11 @@ impl TorrentMaps {
regarding_offer_id: offer_id, regarding_offer_id: offer_id,
}; };
if let Some(_) = answer_receiver.expecting_answers.remove(&expecting_answer) { if answer_receiver
.expecting_answers
.remove(&expecting_answer)
.is_some()
{
let answer_out_message = AnswerOutMessage { let answer_out_message = AnswerOutMessage {
action: AnnounceAction::Announce, action: AnnounceAction::Announce,
peer_id: request.peer_id, peer_id: request.peer_id,
@ -426,15 +430,13 @@ impl TorrentMaps {
#[cfg(feature = "metrics")] #[cfg(feature = "metrics")]
self.peers_gauge_ipv4.decrement(1.0); self.peers_gauge_ipv4.decrement(1.0);
} }
} else { } else if let Some(torrent_data) = self.ipv6.get_mut(&info_hash) {
if let Some(torrent_data) = self.ipv6.get_mut(&info_hash) {
torrent_data.remove_peer(peer_id); torrent_data.remove_peer(peer_id);
#[cfg(feature = "metrics")] #[cfg(feature = "metrics")]
self.peers_gauge_ipv6.decrement(1.0); self.peers_gauge_ipv6.decrement(1.0);
} }
} }
}
} }
struct TorrentData { struct TorrentData {

26
crates/ws_load_test/src/network.rs
View file

@ -149,6 +149,19 @@ impl Connection {
} }
async fn send_message(&mut self) -> anyhow::Result<()> { async fn send_message(&mut self) -> anyhow::Result<()> {
let request = self.create_request();
self.stream.send(request.to_ws_message()).await?;
self.load_test_state
.statistics
.requests
.fetch_add(1, Ordering::Relaxed);
Ok(())
}
fn create_request(&mut self) -> InMessage {
let mut rng = self.rng.borrow_mut(); let mut rng = self.rng.borrow_mut();
let request = match random_request_type(&self.config, &mut *rng) { let request = match random_request_type(&self.config, &mut *rng) {
@ -226,18 +239,9 @@ impl Connection {
} }
}; };
drop(rng);
self.can_send_answer = None; self.can_send_answer = None;
self.stream.send(request.to_ws_message()).await?; request
self.load_test_state
.statistics
.requests
.fetch_add(1, Ordering::Relaxed);
Ok(())
} }
async fn read_message(&mut self) -> anyhow::Result<()> { async fn read_message(&mut self) -> anyhow::Result<()> {
@ -312,7 +316,7 @@ pub fn random_request_type(config: &Config, rng: &mut impl Rng) -> RequestType {
let items = [RequestType::Announce, RequestType::Scrape]; let items = [RequestType::Announce, RequestType::Scrape];
let dist = WeightedIndex::new(&weights).expect("random request weighted index"); let dist = WeightedIndex::new(weights).expect("random request weighted index");
items[dist.sample(rng)] items[dist.sample(rng)]
} }

2
crates/ws_protocol/src/common.rs
View file

@ -156,7 +156,7 @@ mod tests {
assert!(bytes.len() == 20); assert!(bytes.len() == 20);
arr.copy_from_slice(&bytes[..]); arr.copy_from_slice(bytes);
InfoHash(arr) InfoHash(arr)
} }

8
crates/ws_protocol/src/lib.rs
View file

@ -270,7 +270,7 @@ mod tests {
assert!(bytes.len() == 20); assert!(bytes.len() == 20);
arr.copy_from_slice(&bytes[..]); arr.copy_from_slice(bytes);
InfoHash(arr) InfoHash(arr)
} }
@ -370,10 +370,6 @@ mod tests {
::simd_json::serde::from_str(&mut json).unwrap() ::simd_json::serde::from_str(&mut json).unwrap()
}; };
let success = info_hashes == deserialized; info_hashes == deserialized
if !success {}
success
} }
} }