diff --git a/Cargo.lock b/Cargo.lock index 89f5e0e..5eaf471 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -946,9 +946,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.5" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d39cd93900197114fa1fcb7ae84ca742095eed9442088988ae74fa744e930e77" +checksum = "9be70c98951c83b8d2f8f60d7065fa6d5146873094452a1008da8c2f1e4205ad" dependencies = [ "cfg-if", "js-sys", @@ -1231,10 +1231,11 @@ dependencies = [ [[package]] name = "lock_api" -version = "0.4.6" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88943dd7ef4a2e5a4bfa2753aaab3013e34ce2533d1996fb18ef591e315e2b3b" +checksum = "327fa5b6a6940e4699ec49a9beae1ea4845c6bab9314e4f84ac68742139d8c53" dependencies = [ + "autocfg", "scopeguard", ] @@ -1533,9 +1534,9 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "pkg-config" -version = "0.3.24" +version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58893f751c9b0412871a09abd62ecd2a00298c6c83befa223ef98c52aef40cbe" +checksum = "1df8c4ec4b0627e53bdf214615ad287367e482558cf84b109250b37464dc03ae" [[package]] name = "plotters" @@ -1614,9 +1615,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.16" +version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4af2ec4714533fcdf07e886f17025ace8b997b9ce51204ee69b6da831c3da57" +checksum = "632d02bff7f874a36f33ea8bb416cd484b90cc66c1194b1a1110d067a7013f58" dependencies = [ "proc-macro2", ] @@ -1808,9 +1809,9 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.6" +version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4a3381e03edd24287172047536f20cabde766e2cd3e65e6b00fb3af51c4f38d" +checksum = "d65bd28f48be7196d222d95b9243287f48d27aca604e08497513019ff0502cc4" [[package]] name = "serde" @@ -1954,11 +1955,13 @@ checksum = "f2dd574626839106c320a323308629dcb1acfc96e32a8cba364ddc61ac23ee83" [[package]] name = "smartstring" -version = "1.0.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea958ad90cacc8ece7f238fde3671e1b350ee1741964edf2a22fd16f60224163" +checksum = "3fb72c633efbaa2dd666986505016c32c3044395ceaf881518399d2f4127ee29" dependencies = [ + "autocfg", "static_assertions", + "version_check", ] [[package]] @@ -2005,9 +2008,9 @@ checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" [[package]] name = "syn" -version = "1.0.89" +version = "1.0.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea297be220d52398dcc07ce15a209fce436d361735ac1db700cab3b6cdfb9f54" +checksum = "704df27628939572cd88d33f171cd6f896f4eaca85252c6e0a72d8d8287ee86f" dependencies = [ "proc-macro2", "quote", diff --git a/README.md b/README.md index 4abe1cf..112a486 100644 --- a/README.md +++ b/README.md @@ -214,6 +214,12 @@ IPv4 and IPv6 peers are tracked separately. `aquatic_ws` has not been tested as much as `aquatic_udp` but likely works fine. +#### Performance + +![WebTorrent tracker throughput comparison](./documents/aquatic-ws-load-test-illustration-2022-03-29.png) + +More details are available [here](./documents/aquatic-ws-load-test-2022-03-29.pdf). + ## Load testing There are load test binaries for all protocols. They use a CLI structure diff --git a/TODO.md b/TODO.md index f211d7c..24186ca 100644 --- a/TODO.md +++ b/TODO.md @@ -11,6 +11,7 @@ * Run cargo-deny in CI * aquatic_ws + * remove peer from all torrent maps when connection is closed * RES memory still high after traffic stops, even if torrent maps and connection slabs go down to 0 len and capacity * replacing indexmap_amortized / simd_json with equivalents doesn't help * SinkExt::send maybe doesn't wake up properly? diff --git a/documents/aquatic-udp-load-test-2020-04-19.pdf b/documents/aquatic-udp-load-test-2020-04-19.pdf deleted file mode 100644 index 54dd19f..0000000 Binary files a/documents/aquatic-udp-load-test-2020-04-19.pdf and /dev/null differ diff --git a/documents/aquatic-udp-load-test-2021-08-19.pdf b/documents/aquatic-udp-load-test-2021-08-19.pdf deleted file mode 100644 index 43a0bf6..0000000 Binary files a/documents/aquatic-udp-load-test-2021-08-19.pdf and /dev/null differ diff --git a/documents/aquatic-udp-load-test-2021-11-08.pdf b/documents/aquatic-udp-load-test-2021-11-08.pdf deleted file mode 100644 index d3c25d5..0000000 Binary files a/documents/aquatic-udp-load-test-2021-11-08.pdf and /dev/null differ diff --git a/documents/aquatic-udp-load-test-illustration-2021-11-08.png b/documents/aquatic-udp-load-test-illustration-2021-11-08.png deleted file mode 100644 index a016b96..0000000 Binary files a/documents/aquatic-udp-load-test-illustration-2021-11-08.png and /dev/null differ diff --git a/documents/aquatic-ws-load-test-2021-08-18.pdf b/documents/aquatic-ws-load-test-2021-08-18.pdf deleted file mode 100644 index 9f6b990..0000000 Binary files a/documents/aquatic-ws-load-test-2021-08-18.pdf and /dev/null differ diff --git a/documents/aquatic-ws-load-test-2022-03-29.pdf b/documents/aquatic-ws-load-test-2022-03-29.pdf new file mode 100644 index 0000000..26c09c4 Binary files /dev/null and b/documents/aquatic-ws-load-test-2022-03-29.pdf differ diff --git a/documents/aquatic-ws-load-test-illustration-2022-03-29.png b/documents/aquatic-ws-load-test-illustration-2022-03-29.png new file mode 100644 index 0000000..578d7d5 Binary files /dev/null and b/documents/aquatic-ws-load-test-illustration-2022-03-29.png differ diff --git a/documents/testing-aquatic-ws.md b/documents/testing-aquatic-ws.md deleted file mode 100644 index 7f58da4..0000000 --- a/documents/testing-aquatic-ws.md +++ /dev/null @@ -1,129 +0,0 @@ -# Working procedure for testing file transfer with aquatic_ws - -- On VPS, create identity (using real certificate), run tracker with TLS -- On VPS, create torrent with external url as announce. Edit file and put - external url not only as announce, but in announce list too. -- On VPS, disallow traffic to other trackers by adding them to /etc/hosts - or maybe with firewall, since webtorrent-hybrid adds its own trackers - willy-nilly. To get a list of the tracker urls which are actually used, - the node application under heading "Seed application" can be used as a - starting point. -- I opened the listening port in the VPS firewall too (this might not be - necessary if running both clients on the VPS, see below) -- On VPS, run webtorrent-hybrid download --keep-seeding ./abc.torrent -- On desktop/non-VPS computer, fetch torrent file, run - webtorrent-hybrid download ./abc.torrent - I actually got it to work running this client on the VPS too. - -## Seed application - -```js -// Start webtorrent seeder from data file, create torrent, write it to file, -// output info - -var WebTorrent = require('webtorrent-hybrid') -var fs = require('fs') - -// WebTorrent seems to use same peer id for different -// clients in some cases (I don't know how) -peerId = "ae61b6f4a5be4ada48333891512db5e90347d736" -announceUrl = 'ws://127.0.0.1:3000' -dataFile = './files-seed/ws-ipv4' -torrentFile = './torrents/ws-ipv4.torrent' - -function createSeeder(){ - console.log('creating seeder..') - - var seeder = new WebTorrent({ dht: false, webSeeds: false, peerId: peerId }) - seeder.on('error', function(err) { - console.log('seeder error: ' + err) - }) - - var addOpts = { - announceList: [[announceUrl]], - announce: [announceUrl], - private: true - } - - seeder.seed(dataFile, addOpts, function(torrent){ - console.log("seeding") - // Log torrent info, including actually used trackers - console.log(torrent) - - fs.writeFile(torrentFile, torrent.torrentFile, function(err){ - if (err){ - console.log(err) - } - }) - - torrent.on('warning', function(err){ - console.log(err) - }); - - torrent.on('error', function(err){ - console.log(err) - }); - - torrent.on('download', function(bytes){ - console.log('downloaded bytes: ' + bytes) - }); - - torrent.on('upload', function(bytes){ - console.log('uploaded bytes: ' + bytes) - }); - - torrent.on('wire', function(wire, addr){ - console.log('connected to peer with addr: ' + addr) - }); - - torrent.on('noPeers', function(announceType){ - console.log('no peers received with announce type: ' + announceType) - }) - - torrent.on('done', function(){ - console.log('done') - }); - - }) -} - -createSeeder() -``` - -## Simplifications to procedure that might work - -- using fake certificate and routing certificate url to localhost in - /etc/hosts, meaning all of this could maybe be run locally/in Docker (but I - think sdp negotiations tend to fail in that case..) - -## Issues with Docker implementation - -- webtorrent-hybrid adds its own trackers when opening torrents, even if they - have been removed from file! The really robust way to get around this would - be to block all outgoing traffic with e.g. iptables before starting tests, - but I couldn't get it to work - -## Notes on testing locally - -Official tracker does not successfully handle file transfer on localhost -on my machine between two instances of the official client (webtorrent-hybrid), -probably due to sdp negotiation issues. This was with plain `ws` protocol. - -## Possibly useful collection of commands - -```sh -npm install -g webtorrent-hybrid -npm install -g bittorrent-tracker # Reference tracker - -bittorrent-tracker --ws -p 3000 & # Reference tracker - -mkdir files-seed files-leech torrents - -webtorrent create files-seed/ws-ipv4 --announce "wss://127.0.0.1:3000" > torrents/ws-ipv4.torrent - -cd files-seed -webtorrent-hybrid seed torrents/ws-ipv4.torrent --keep-seeding -q & - -cd ../files-leech -webtorrent-hybrid download torrents/ws-ipv4.torrent -q & -``` \ No newline at end of file diff --git a/scripts/gen-tls.sh b/scripts/gen-tls.sh index 12a57a6..db8a19b 100755 --- a/scripts/gen-tls.sh +++ b/scripts/gen-tls.sh @@ -1,18 +1,17 @@ #/bin/bash +# Generate self-signed TLS cert and private key for local testing set -e -mkdir -p tmp/tls +TLS_DIR="./tmp/tls" -cd tmp/tls +mkdir -p "$TLS_DIR" +cd "$TLS_DIR" openssl ecparam -genkey -name prime256v1 -out key.pem openssl req -new -sha256 -key key.pem -out csr.csr -subj "/C=GB/ST=Test/L=Test/O=Test/OU=Test/CN=example.com" openssl req -x509 -sha256 -nodes -days 365 -key key.pem -in csr.csr -out cert.crt - -sudo cp cert.crt /usr/local/share/ca-certificates/snakeoil.crt -sudo update-ca-certificates - openssl pkcs8 -in key.pem -topk8 -nocrypt -out key.pk8 -# openssl pkcs12 -export -passout "pass:p" -out identity.pfx -inkey key.pem -in cert.crt \ No newline at end of file +echo "tls_certificate_path = \"$TLS_DIR/cert.crt\"" +echo "tls_private_key_path = \"$TLS_DIR/key.pk8\""