From c0e0adbbae8449e923b41c3f408259b61cd4f9ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20Frosteg=C3=A5rd?= Date: Wed, 10 Feb 2021 21:27:32 +0100 Subject: [PATCH 01/22] Add wss functionality for file transfer CI --- .../test-transfer-http-udp/entrypoint.sh | 175 --------- .../Dockerfile | 0 .../action.yml | 6 +- .github/actions/test-transfer/entrypoint.sh | 346 ++++++++++++++++++ ...ransfer-http-udp.yml => test-transfer.yml} | 8 +- 5 files changed, 354 insertions(+), 181 deletions(-) delete mode 100755 .github/actions/test-transfer-http-udp/entrypoint.sh rename .github/actions/{test-transfer-http-udp => test-transfer}/Dockerfile (100%) rename .github/actions/{test-transfer-http-udp => test-transfer}/action.yml (65%) create mode 100755 .github/actions/test-transfer/entrypoint.sh rename .github/workflows/{test-transfer-http-udp.yml => test-transfer.yml} (67%) diff --git a/.github/actions/test-transfer-http-udp/entrypoint.sh b/.github/actions/test-transfer-http-udp/entrypoint.sh deleted file mode 100755 index 3618bed..0000000 --- a/.github/actions/test-transfer-http-udp/entrypoint.sh +++ /dev/null @@ -1,175 +0,0 @@ -#!/bin/bash -# -# Test that file transfers work with aquatic_http (with and without TLS) -# and aquatic_udp. -# -# IPv6 is unfortunately disabled by default in Docker -# (see sysctl net.ipv6.conf.lo.disable_ipv6) - -set -e - -# Install programs and build dependencies - -if command -v sudo; then - SUDO="sudo " -else - SUDO="" -fi - -$SUDO apt-get update -$SUDO apt-get install -y cmake libssl-dev screen rtorrent mktorrent ssl-cert ca-certificates - -rtorrent -h - -# Clone repository if necessary, go to repository directory - -if [[ -z "${GITHUB_WORKSPACE}" ]]; then - cd "$HOME" - - git clone https://github.com/greatest-ape/aquatic.git - - cd aquatic -else - cd "$GITHUB_WORKSPACE" -fi - -# Setup bogus TLS certificate - -$SUDO echo "127.0.0.1 example.com" >> /etc/hosts - -openssl ecparam -genkey -name prime256v1 -out key.pem -openssl req -new -sha256 -key key.pem -out csr.csr -subj "/C=GB/ST=Test/L=Test/O=Test/OU=Test/CN=example.com" -openssl req -x509 -sha256 -nodes -days 365 -key key.pem -in csr.csr -out cert.crt - -$SUDO cp cert.crt /usr/local/share/ca-certificates/snakeoil.crt -$SUDO update-ca-certificates - -openssl pkcs12 -export -passout "pass:p" -out identity.pfx -inkey key.pem -in cert.crt - -# Build and start tracker - -cargo build --bin aquatic - -echo "log_level = 'debug' - -[network] -address = '127.0.0.1:3000'" > http.toml -./target/debug/aquatic http -c http.toml > "$HOME/http.log" 2>&1 & - -echo "log_level = 'debug' - -[network] -address = '127.0.0.1:3001' -use_tls = true -tls_pkcs12_path = './identity.pfx' -tls_pkcs12_password = 'p' -" > tls.toml -./target/debug/aquatic http -c tls.toml > "$HOME/tls.log" 2>&1 & - -echo "[network] -address = '127.0.0.1:3000'" > udp.toml -./target/debug/aquatic udp -c udp.toml > "$HOME/udp.log" 2>&1 & - -# Setup directories - -cd "$HOME" - -mkdir seed -mkdir leech -mkdir torrents - -# Create torrents - -echo "http-test-ipv4" > seed/http-test-ipv4 -echo "tls-test-ipv4" > seed/tls-test-ipv4 -echo "udp-test-ipv4" > seed/udp-test-ipv4 - -mktorrent -p -o "torrents/http-ipv4.torrent" -a "http://127.0.0.1:3000/announce" "seed/http-test-ipv4" -mktorrent -p -o "torrents/tls-ipv4.torrent" -a "https://example.com:3001/announce" "seed/tls-test-ipv4" -mktorrent -p -o "torrents/udp-ipv4.torrent" -a "udp://127.0.0.1:3000" "seed/udp-test-ipv4" - -cp -r torrents torrents-seed -cp -r torrents torrents-leech - -# Start seeding client - -echo "directory.default.set = $HOME/seed -schedule2 = watch_directory,5,5,load.start=$HOME/torrents-seed/*.torrent" > ~/.rtorrent.rc - -echo "Starting seeding client" -screen -dmS rtorrent-seed rtorrent - -sleep 10 # Give seeding rtorrent time to load its config file - -# Start leeching client - -echo "directory.default.set = $HOME/leech -schedule2 = watch_directory,5,5,load.start=$HOME/torrents-leech/*.torrent" > ~/.rtorrent.rc - -echo "Starting leeching client.." -screen -dmS rtorrent-leech rtorrent - -# Check for completion - -HTTP_IPv4="Failed" -TLS_IPv4="Failed" -UDP_IPv4="Failed" - -i="0" - -echo "Watching for finished files.." - -while [ $i -lt 300 ] -do - if test -f "leech/http-test-ipv4"; then - if grep -q "http-test-ipv4" "leech/http-test-ipv4"; then - HTTP_IPv4="Ok" - fi - fi - if test -f "leech/tls-test-ipv4"; then - if grep -q "tls-test-ipv4" "leech/tls-test-ipv4"; then - TLS_IPv4="Ok" - fi - fi - if test -f "leech/udp-test-ipv4"; then - if grep -q "udp-test-ipv4" "leech/udp-test-ipv4"; then - UDP_IPv4="Ok" - fi - fi - - if [ "$HTTP_IPv4" = "Ok" ] && [ "$TLS_IPv4" = "Ok" ] && [ "$UDP_IPv4" = "Ok" ]; then - break - fi - - sleep 1 - - i=$[$i+1] -done - -echo "Waited for $i seconds" - -echo "::set-output name=http_ipv4::$HTTP_IPv4" -echo "::set-output name=http_tls_ipv4::$TLS_IPv4" -echo "::set-output name=udp_ipv4::$UDP_IPv4" - -echo "" -echo "# --- HTTP log --- #" -cat "http.log" - -echo "" -echo "# --- HTTP over TLS log --- #" -cat "tls.log" - -echo "" -echo "# --- UDP log --- #" -cat "udp.log" - -echo "" -echo "# --- Test results --- #" -echo "HTTP (IPv4): $HTTP_IPv4" -echo "HTTP over TLS (IPv4): $TLS_IPv4" -echo "UDP (IPv4): $UDP_IPv4" - -if [ "$HTTP_IPv4" != "Ok" ] || [ "$TLS_IPv4" != "Ok" ] || [ "$UDP_IPv4" != "Ok" ]; then - exit 1 -fi \ No newline at end of file diff --git a/.github/actions/test-transfer-http-udp/Dockerfile b/.github/actions/test-transfer/Dockerfile similarity index 100% rename from .github/actions/test-transfer-http-udp/Dockerfile rename to .github/actions/test-transfer/Dockerfile diff --git a/.github/actions/test-transfer-http-udp/action.yml b/.github/actions/test-transfer/action.yml similarity index 65% rename from .github/actions/test-transfer-http-udp/action.yml rename to .github/actions/test-transfer/action.yml index 9bf3fa2..2520a97 100644 --- a/.github/actions/test-transfer-http-udp/action.yml +++ b/.github/actions/test-transfer/action.yml @@ -1,5 +1,5 @@ -name: 'test-transfer-http-udp' -description: 'test aquatic http and udp file transfer' +name: 'test-transfer' +description: 'test aquatic file transfer' outputs: http_ipv4: description: 'HTTP IPv4 status' @@ -7,6 +7,8 @@ outputs: description: 'HTTP IPv4 over TLS status' udp_ipv4: description: 'UDP IPv4 status' + wss_ipv4: + description: 'WSS IPv4 status' runs: using: 'docker' image: 'Dockerfile' \ No newline at end of file diff --git a/.github/actions/test-transfer/entrypoint.sh b/.github/actions/test-transfer/entrypoint.sh new file mode 100755 index 0000000..5cd5eed --- /dev/null +++ b/.github/actions/test-transfer/entrypoint.sh @@ -0,0 +1,346 @@ +#!/bin/bash +# +# Test that file transfers work with aquatic_http (with and without TLS) +# aquatic_udp and experimentally aquatic_ws (with TLS). +# +# IPv6 is unfortunately disabled by default in Docker +# (see sysctl net.ipv6.conf.lo.disable_ipv6) +# +# When testing locally, use: +# 1. docker build -t aquatic ./path/to/Dockerfile +# 2. docker run aquatic +# 3. On failure, run `docker rmi aquatic -f` and go back to step 1 + +set -e + +# Install programs and build dependencies + +if command -v sudo; then + SUDO="sudo " +else + SUDO="" +fi + +$SUDO apt-get update +$SUDO apt-get install -y cmake libssl-dev screen rtorrent mktorrent ssl-cert ca-certificates curl + +$SUDO curl -sL https://deb.nodesource.com/setup_15.x | bash - +$SUDO apt-get install nodejs -y + +rtorrent -h + +# Clone repository if necessary, go to repository directory + +if [[ -z "${GITHUB_WORKSPACE}" ]]; then + cd "$HOME" + + git clone https://github.com/greatest-ape/aquatic.git + + cd aquatic +else + cd "$GITHUB_WORKSPACE" +fi + +# Setup bogus TLS certificate + +$SUDO echo "127.0.0.1 example.com" >> /etc/hosts + +openssl ecparam -genkey -name prime256v1 -out key.pem +openssl req -new -sha256 -key key.pem -out csr.csr -subj "/C=GB/ST=Test/L=Test/O=Test/OU=Test/CN=example.com" +openssl req -x509 -sha256 -nodes -days 365 -key key.pem -in csr.csr -out cert.crt + +$SUDO cp cert.crt /usr/local/share/ca-certificates/snakeoil.crt +$SUDO update-ca-certificates + +openssl pkcs12 -export -passout "pass:p" -out identity.pfx -inkey key.pem -in cert.crt + +# Build and start tracker + +cargo build --bin aquatic + +echo "log_level = 'debug' + +[network] +address = '127.0.0.1:3000'" > http.toml +./target/debug/aquatic http -c http.toml > "$HOME/http.log" 2>&1 & + +echo "log_level = 'debug' + +[network] +address = '127.0.0.1:3001' +use_tls = true +tls_pkcs12_path = './identity.pfx' +tls_pkcs12_password = 'p' +" > tls.toml +./target/debug/aquatic http -c tls.toml > "$HOME/tls.log" 2>&1 & + +echo "[network] +address = '127.0.0.1:3000'" > udp.toml +./target/debug/aquatic udp -c udp.toml > "$HOME/udp.log" 2>&1 & + +echo "log_level = 'trace' + +[network] +address = '127.0.0.1:3002' +use_tls = true +tls_pkcs12_path = './identity.pfx' +tls_pkcs12_password = 'p' +" > ws.toml +./target/debug/aquatic ws -c ws.toml > "$HOME/wss.log" 2>&1 & + +# Setup directories + +cd "$HOME" + +mkdir seed +mkdir leech +mkdir torrents + +# Create torrents + +echo "http-test-ipv4" > seed/http-test-ipv4 +echo "tls-test-ipv4" > seed/tls-test-ipv4 +echo "udp-test-ipv4" > seed/udp-test-ipv4 +echo "wss-test-ipv4" > seed/wss-test-ipv4 + +mktorrent -p -o "torrents/http-ipv4.torrent" -a "http://127.0.0.1:3000/announce" "seed/http-test-ipv4" +mktorrent -p -o "torrents/tls-ipv4.torrent" -a "https://example.com:3001/announce" "seed/tls-test-ipv4" +mktorrent -p -o "torrents/udp-ipv4.torrent" -a "udp://127.0.0.1:3000" "seed/udp-test-ipv4" + +cp -r torrents torrents-seed +cp -r torrents torrents-leech + +# Setup wss seeding client + +# Seems to fix webtorrent-hybrid install error. +# Will likely be fixed in later versions of webtorrent-hybrid. +npm install @mapbox/node-pre-gyp + +npm install webtorrent-hybrid@4.0.3 + +echo " +// Start webtorrent seeder from data file, create torrent, write it to file, +// output info + +var WebTorrent = require('webtorrent-hybrid') +var fs = require('fs') + +// WebTorrent seems to use same peer id for different +// clients in some cases (I don't know how) +peerId = 'ae61b6f4a5be4ada48333891512db5e90347d736' +announceUrl = 'wss://example.com:3002' +dataFile = './seed/wss-test-ipv4' +torrentFile = './torrents/wss-ipv4.torrent' +trackerFile = './wss-trackers.txt' + +function createSeeder(){ + console.log('creating seeder..') + + var seeder = new WebTorrent({ dht: false, webSeeds: false, peerId: peerId }) + seeder.on('error', function(err) { + console.log('seeder error: ' + err) + }) + + var addOpts = { + announceList: [[announceUrl]], + announce: [announceUrl], + private: true + } + + seeder.seed(dataFile, addOpts, function(torrent){ + console.log('seeding') + console.log(torrent.announce) + + torrent.announce.forEach(function(item, index){ + if (item != announceUrl) { + fs.appendFile(trackerFile, '127.0.0.1 '.concat(item.replace(/(^\w+:|^)\/\//, ''), '\n'), function(err){ + if (err){ + console.log(err) + } + }) + } + }); + + fs.writeFile(torrentFile, torrent.torrentFile, function(err){ + if (err){ + console.log(err) + } + }) + + torrent.on('warning', function(err){ + console.log(err) + }); + + torrent.on('error', function(err){ + console.log(err) + }); + + torrent.on('download', function(bytes){ + console.log('downloaded bytes: ' + bytes) + }); + + torrent.on('upload', function(bytes){ + console.log('uploaded bytes: ' + bytes) + }); + + torrent.on('wire', function(wire, addr){ + console.log('connected to peer with addr: ' + addr) + }); + + torrent.on('noPeers', function(announceType){ + console.log('no peers received with announce type: ' + announceType) + }) + + torrent.on('done', function(){ + console.log('done') + }); + + }) +} + +createSeeder() +" > wss-seeder.js + +cat wss-seeder.js + +# Start seeding ws client, create torrent + +echo "Starting seeding ws client" +node wss-seeder.js > "$HOME/wss-seed.log" 2>&1 & + +# Start seeding rtorrent client + +echo "directory.default.set = $HOME/seed +schedule2 = watch_directory,5,5,load.start=$HOME/torrents-seed/*.torrent" > ~/.rtorrent.rc + +echo "Starting seeding rtorrent client" +screen -dmS rtorrent-seed rtorrent + +# Give seeding clients time to write trackers to file, load config files etc + +echo "Waiting for a while" +sleep 30 + +# Forbid access to other trackers used by webtorrent-hybrid + +if test -f "wss-trackers.txt"; then + $SUDO cat wss-trackers.txt >> /etc/hosts + echo "Added the following lines to /etc/hosts:" + cat wss-trackers.txt + echo "" +else + echo "ERROR: WSS TRACKER FILE NOT FOUND. Seeding client log:" + cat "$HOME/wss-seed.log" + exit 1 +fi + +# Start leeching clients + +echo "directory.default.set = $HOME/leech +schedule2 = watch_directory,5,5,load.start=$HOME/torrents-leech/*.torrent" > ~/.rtorrent.rc + +echo "Starting leeching client.." +screen -dmS rtorrent-leech rtorrent + +./node_modules/webtorrent-hybrid/bin/cmd.js download ./torrents/wss-ipv4.torrent -o leech > "$HOME/wss-leech.log" 2>&1 & + +# Check for completion + +HTTP_IPv4="Failed" +TLS_IPv4="Failed" +UDP_IPv4="Failed" +WSS_IPv4="Failed" + +i="0" + +echo "Watching for finished files.." + +while [ $i -lt 300 ] +do + if test -f "leech/http-test-ipv4"; then + if grep -q "http-test-ipv4" "leech/http-test-ipv4"; then + HTTP_IPv4="Ok" + echo "HTTP_IPv4 is Ok" + fi + fi + if test -f "leech/tls-test-ipv4"; then + if grep -q "tls-test-ipv4" "leech/tls-test-ipv4"; then + TLS_IPv4="Ok" + echo "TLS_IPv4 is Ok" + fi + fi + if test -f "leech/udp-test-ipv4"; then + if grep -q "udp-test-ipv4" "leech/udp-test-ipv4"; then + UDP_IPv4="Ok" + echo "UDP_IPv4 is Ok" + fi + fi + if test -f "leech/wss-test-ipv4"; then + if grep -q "wss-test-ipv4" "leech/wss-test-ipv4"; then + WSS_IPv4="Ok" + echo "WSS_IPv4 is Ok" + fi + fi + + if [ "$HTTP_IPv4" = "Ok" ] && [ "$TLS_IPv4" = "Ok" ] && [ "$UDP_IPv4" = "Ok" ] && [ "$WSS_IPv4" = "Ok" ]; then + break + fi + + sleep 1 + + i=$[$i+1] +done + +echo "Waited for $i seconds" + +echo "::set-output name=http_ipv4::$HTTP_IPv4" +echo "::set-output name=http_tls_ipv4::$TLS_IPv4" +echo "::set-output name=udp_ipv4::$UDP_IPv4" +echo "::set-output name=wss_ipv4::$WSS_IPv4" + +echo "" +echo "# --- HTTP log --- #" +cat "http.log" + +sleep 1 + +echo "" +echo "# --- HTTP over TLS log --- #" +cat "tls.log" + +sleep 1 + +echo "" +echo "# --- UDP log --- #" +cat "udp.log" + +sleep 1 + +echo "" +echo "# --- WSS tracker log --- #" +cat "wss.log" + +sleep 1 + +echo "" +echo "# --- WSS seed log --- #" +cat "wss-seed.log" + +sleep 1 + +echo "" +echo "# --- WSS leech log --- #" +cat "wss-leech.log" + +sleep 1 + +echo "" +echo "# --- Test results --- #" +echo "HTTP (IPv4): $HTTP_IPv4" +echo "HTTP over TLS (IPv4): $TLS_IPv4" +echo "UDP (IPv4): $UDP_IPv4" +echo "WSS (IPv4): $WSS_IPv4" + +if [ "$HTTP_IPv4" != "Ok" ] || [ "$TLS_IPv4" != "Ok" ] || [ "$UDP_IPv4" != "Ok" ] || [ "$WSS_IPv4" != "Ok" ]; then + exit 1 +fi \ No newline at end of file diff --git a/.github/workflows/test-transfer-http-udp.yml b/.github/workflows/test-transfer.yml similarity index 67% rename from .github/workflows/test-transfer-http-udp.yml rename to .github/workflows/test-transfer.yml index 956dd63..97a4f91 100644 --- a/.github/workflows/test-transfer-http-udp.yml +++ b/.github/workflows/test-transfer.yml @@ -1,4 +1,4 @@ -name: "Test HTTP and UDP file transfer" +name: "Test HTTP, UDP and WSS file transfer" on: push: @@ -9,10 +9,10 @@ on: jobs: test-transfer-http: runs-on: ubuntu-latest - name: "Test BitTorrent file transfer over HTTP (with and without TLS) and UDP" + name: "Test BitTorrent file transfer over HTTP (with and without TLS), UDP and WSS" steps: - name: Checkout uses: actions/checkout@v2 - name: Test file transfers - uses: ./.github/actions/test-transfer-http-udp - id: test_transfer_udp_http \ No newline at end of file + uses: ./.github/actions/test-transfer + id: test_transfer \ No newline at end of file From de42d2e1b71c45b92826b3a2e8cc4eecc627659b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20Frosteg=C3=A5rd?= Date: Wed, 10 Feb 2021 21:28:59 +0100 Subject: [PATCH 02/22] aquatic_ws: add trace logging of requests --- aquatic_ws/src/lib/handler.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/aquatic_ws/src/lib/handler.rs b/aquatic_ws/src/lib/handler.rs index 1e53fbf..67c485f 100644 --- a/aquatic_ws/src/lib/handler.rs +++ b/aquatic_ws/src/lib/handler.rs @@ -123,6 +123,8 @@ pub fn handle_announce_requests( } } + ::log::trace!("received request from {:?}", request_sender_meta); + // Insert/update/remove peer who sent this request { let peer_status = PeerStatus::from_event_and_bytes_left( @@ -197,6 +199,10 @@ pub fn handle_announce_requests( offer_receiver.connection_meta, OutMessage::Offer(middleman_offer) ); + ::log::trace!( + "sent middleman offer to {:?}", + offer_receiver.connection_meta + ); wake_socket_workers[offer_receiver.connection_meta.worker_index] = true; } } @@ -220,6 +226,10 @@ pub fn handle_announce_requests( answer_receiver.connection_meta, OutMessage::Answer(middleman_answer) ); + ::log::trace!( + "sent middleman answer to {:?}", + answer_receiver.connection_meta + ); wake_socket_workers[answer_receiver.connection_meta.worker_index] = true; } } From d93f1fda7f4338ed175194a4b7cb1eb111a2fb7a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20Frosteg=C3=A5rd?= Date: Wed, 10 Feb 2021 21:50:56 +0100 Subject: [PATCH 03/22] aquatic_ws: add more trace logging --- aquatic_ws/src/lib/network/connection.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/aquatic_ws/src/lib/network/connection.rs b/aquatic_ws/src/lib/network/connection.rs index 3d60949..30685d3 100644 --- a/aquatic_ws/src/lib/network/connection.rs +++ b/aquatic_ws/src/lib/network/connection.rs @@ -153,6 +153,11 @@ impl HandshakeMachine { ) -> (Option>, bool) { match result { Ok(stream) => { + ::log::trace!( + "established tls handshake with peer with addr: {:?}", + stream.get_ref().peer_addr() + ); + (Some(Either::Right(Self::TlsStream(stream))), false) }, Err(native_tls::HandshakeError::WouldBlock(handshake)) => { @@ -174,6 +179,11 @@ impl HandshakeMachine { Ok(mut ws) => { let peer_addr = ws.get_mut().get_peer_addr(); + ::log::trace!( + "established ws handshake with peer with addr: {:?}", + peer_addr + ); + let established_ws = EstablishedWs { ws, peer_addr, From 130bc07f9417e1e3c8984b64ce07a92ed8d8a97d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20Frosteg=C3=A5rd?= Date: Wed, 10 Feb 2021 21:51:07 +0100 Subject: [PATCH 04/22] Transfer CI: don't output status too often --- .github/actions/test-transfer/entrypoint.sh | 24 ++++++++++++++------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/.github/actions/test-transfer/entrypoint.sh b/.github/actions/test-transfer/entrypoint.sh index 5cd5eed..1aa3eaf 100755 --- a/.github/actions/test-transfer/entrypoint.sh +++ b/.github/actions/test-transfer/entrypoint.sh @@ -259,26 +259,34 @@ while [ $i -lt 300 ] do if test -f "leech/http-test-ipv4"; then if grep -q "http-test-ipv4" "leech/http-test-ipv4"; then - HTTP_IPv4="Ok" - echo "HTTP_IPv4 is Ok" + if [ "$HTTP_IPv4" != "Ok" ]; then + HTTP_IPv4="Ok" + echo "HTTP_IPv4 is Ok" + fi fi fi if test -f "leech/tls-test-ipv4"; then if grep -q "tls-test-ipv4" "leech/tls-test-ipv4"; then - TLS_IPv4="Ok" - echo "TLS_IPv4 is Ok" + if [ "$TLS_IPv4" != "Ok" ]; then + TLS_IPv4="Ok" + echo "TLS_IPv4 is Ok" + fi fi fi if test -f "leech/udp-test-ipv4"; then if grep -q "udp-test-ipv4" "leech/udp-test-ipv4"; then - UDP_IPv4="Ok" - echo "UDP_IPv4 is Ok" + if [ "$UDP_IPv4" != "Ok" ]; then + UDP_IPv4="Ok" + echo "UDP_IPv4 is Ok" + fi fi fi if test -f "leech/wss-test-ipv4"; then if grep -q "wss-test-ipv4" "leech/wss-test-ipv4"; then - WSS_IPv4="Ok" - echo "WSS_IPv4 is Ok" + if [ "$WSS_IPv4" != "Ok" ]; then + WSS_IPv4="Ok" + echo "WSS_IPv4 is Ok" + fi fi fi From 1e2cad7e80dcaeff2a8d3d65eff34cf397e656d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20Frosteg=C3=A5rd?= Date: Mon, 16 Aug 2021 19:23:04 +0200 Subject: [PATCH 05/22] Try out using other ws client for CI --- .github/actions/test-transfer/entrypoint.sh | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/actions/test-transfer/entrypoint.sh b/.github/actions/test-transfer/entrypoint.sh index 1aa3eaf..2b43712 100755 --- a/.github/actions/test-transfer/entrypoint.sh +++ b/.github/actions/test-transfer/entrypoint.sh @@ -22,7 +22,9 @@ else fi $SUDO apt-get update -$SUDO apt-get install -y cmake libssl-dev screen rtorrent mktorrent ssl-cert ca-certificates curl +$SUDO apt-get install -y cmake libssl-dev screen rtorrent mktorrent ssl-cert ca-certificates curl golang + +$SUDO go get github.com/anacrolix/torrent/cmd/torrent $SUDO curl -sL https://deb.nodesource.com/setup_15.x | bash - $SUDO apt-get install nodejs -y @@ -242,7 +244,11 @@ schedule2 = watch_directory,5,5,load.start=$HOME/torrents-leech/*.torrent" > ~/. echo "Starting leeching client.." screen -dmS rtorrent-leech rtorrent -./node_modules/webtorrent-hybrid/bin/cmd.js download ./torrents/wss-ipv4.torrent -o leech > "$HOME/wss-leech.log" 2>&1 & +# ./node_modules/webtorrent-hybrid/bin/cmd.js download ./torrents/wss-ipv4.torrent -o leech > "$HOME/wss-leech.log" 2>&1 & + +cd leech +GOPPROF=http torrent download ./torrents/wss-ipv4.torrent > "$HOME/wss-leech.log" 2>&1 & +cd .. # Check for completion From 7b4e00bb79f15168311a20a70ff5963b81cd8ef2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20Frosteg=C3=A5rd?= Date: Mon, 16 Aug 2021 19:35:16 +0200 Subject: [PATCH 06/22] CI: use gotorrent from git; use rust-bullseye --- .github/actions/test-transfer/Dockerfile | 2 +- .github/actions/test-transfer/entrypoint.sh | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/actions/test-transfer/Dockerfile b/.github/actions/test-transfer/Dockerfile index 7e01849..b297bc7 100644 --- a/.github/actions/test-transfer/Dockerfile +++ b/.github/actions/test-transfer/Dockerfile @@ -1,5 +1,5 @@ # Container image that runs your code -FROM rust:latest +FROM rust:bullseye # Copies your code file from your action repository to the filesystem path `/` of the container COPY entrypoint.sh /entrypoint.sh diff --git a/.github/actions/test-transfer/entrypoint.sh b/.github/actions/test-transfer/entrypoint.sh index 2b43712..fbb96ff 100755 --- a/.github/actions/test-transfer/entrypoint.sh +++ b/.github/actions/test-transfer/entrypoint.sh @@ -24,8 +24,6 @@ fi $SUDO apt-get update $SUDO apt-get install -y cmake libssl-dev screen rtorrent mktorrent ssl-cert ca-certificates curl golang -$SUDO go get github.com/anacrolix/torrent/cmd/torrent - $SUDO curl -sL https://deb.nodesource.com/setup_15.x | bash - $SUDO apt-get install nodejs -y @@ -246,8 +244,9 @@ screen -dmS rtorrent-leech rtorrent # ./node_modules/webtorrent-hybrid/bin/cmd.js download ./torrents/wss-ipv4.torrent -o leech > "$HOME/wss-leech.log" 2>&1 & +git clone https://github.com/anacrolix/torrent.git leech cd leech -GOPPROF=http torrent download ./torrents/wss-ipv4.torrent > "$HOME/wss-leech.log" 2>&1 & +GOPPROF=http go run -v ./cmd/torrent download ./torrents/wss-ipv4.torrent > "$HOME/wss-leech.log" 2>&1 & cd .. # Check for completion From 4dd56889998dd0166984753a080ed5241b8fa276 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20Frosteg=C3=A5rd?= Date: Mon, 16 Aug 2021 19:42:22 +0200 Subject: [PATCH 07/22] CI: fix typo --- .github/actions/test-transfer/entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/test-transfer/entrypoint.sh b/.github/actions/test-transfer/entrypoint.sh index fbb96ff..d1cc3aa 100755 --- a/.github/actions/test-transfer/entrypoint.sh +++ b/.github/actions/test-transfer/entrypoint.sh @@ -246,7 +246,7 @@ screen -dmS rtorrent-leech rtorrent git clone https://github.com/anacrolix/torrent.git leech cd leech -GOPPROF=http go run -v ./cmd/torrent download ./torrents/wss-ipv4.torrent > "$HOME/wss-leech.log" 2>&1 & +GOPPROF=http go run -v ./cmd/torrent download ../torrents/wss-ipv4.torrent > "$HOME/wss-leech.log" 2>&1 & cd .. # Check for completion From a268544a1be56b7743d0337fe2feb7037ce8b126 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20Frosteg=C3=A5rd?= Date: Mon, 16 Aug 2021 19:54:02 +0200 Subject: [PATCH 08/22] CI: use debian buster; make go accept TLS CommonNames in certs --- .github/actions/test-transfer/Dockerfile | 2 +- .github/actions/test-transfer/entrypoint.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/test-transfer/Dockerfile b/.github/actions/test-transfer/Dockerfile index b297bc7..7e01849 100644 --- a/.github/actions/test-transfer/Dockerfile +++ b/.github/actions/test-transfer/Dockerfile @@ -1,5 +1,5 @@ # Container image that runs your code -FROM rust:bullseye +FROM rust:latest # Copies your code file from your action repository to the filesystem path `/` of the container COPY entrypoint.sh /entrypoint.sh diff --git a/.github/actions/test-transfer/entrypoint.sh b/.github/actions/test-transfer/entrypoint.sh index d1cc3aa..3a5d95e 100755 --- a/.github/actions/test-transfer/entrypoint.sh +++ b/.github/actions/test-transfer/entrypoint.sh @@ -246,7 +246,7 @@ screen -dmS rtorrent-leech rtorrent git clone https://github.com/anacrolix/torrent.git leech cd leech -GOPPROF=http go run -v ./cmd/torrent download ../torrents/wss-ipv4.torrent > "$HOME/wss-leech.log" 2>&1 & +GOPPROF=http GODEBUG=x509ignoreCN=0 go run -v ./cmd/torrent download ../torrents/wss-ipv4.torrent > "$HOME/wss-leech.log" 2>&1 & cd .. # Check for completion From cf25586bcb95edc7e8915e3320e9d440fb6edfa1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20Frosteg=C3=A5rd?= Date: Mon, 16 Aug 2021 20:05:41 +0200 Subject: [PATCH 09/22] CI: use bullseye again --- .github/actions/test-transfer/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/test-transfer/Dockerfile b/.github/actions/test-transfer/Dockerfile index 7e01849..b297bc7 100644 --- a/.github/actions/test-transfer/Dockerfile +++ b/.github/actions/test-transfer/Dockerfile @@ -1,5 +1,5 @@ # Container image that runs your code -FROM rust:latest +FROM rust:bullseye # Copies your code file from your action repository to the filesystem path `/` of the container COPY entrypoint.sh /entrypoint.sh From 631b2c0494d15255cc06d61b3a7f4e473d299a72 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20Frosteg=C3=A5rd?= Date: Mon, 16 Aug 2021 20:37:24 +0200 Subject: [PATCH 10/22] CI: seed with gotorrent too --- .github/actions/test-transfer/entrypoint.sh | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/.github/actions/test-transfer/entrypoint.sh b/.github/actions/test-transfer/entrypoint.sh index 3a5d95e..5b36825 100755 --- a/.github/actions/test-transfer/entrypoint.sh +++ b/.github/actions/test-transfer/entrypoint.sh @@ -24,6 +24,11 @@ fi $SUDO apt-get update $SUDO apt-get install -y cmake libssl-dev screen rtorrent mktorrent ssl-cert ca-certificates curl golang +git clone https://github.com/anacrolix/torrent.git gotorrent +cd gotorrent +go build -o ../gotorrent ./cmd/torrent +cd .. + $SUDO curl -sL https://deb.nodesource.com/setup_15.x | bash - $SUDO apt-get install nodejs -y @@ -234,6 +239,12 @@ else exit 1 fi +# Start seeding ws client 2 + +cd seed +GOPPROF=http GODEBUG=x509ignoreCN=0 ../gotorrent download --seed ../torrents/wss-ipv4.torrent > "$HOME/wss-seed2.log" 2>&1 & +cd .. + # Start leeching clients echo "directory.default.set = $HOME/leech @@ -244,9 +255,8 @@ screen -dmS rtorrent-leech rtorrent # ./node_modules/webtorrent-hybrid/bin/cmd.js download ./torrents/wss-ipv4.torrent -o leech > "$HOME/wss-leech.log" 2>&1 & -git clone https://github.com/anacrolix/torrent.git leech cd leech -GOPPROF=http GODEBUG=x509ignoreCN=0 go run -v ./cmd/torrent download ../torrents/wss-ipv4.torrent > "$HOME/wss-leech.log" 2>&1 & +GOPPROF=http GODEBUG=x509ignoreCN=0 ../gotorrent download ../torrents/wss-ipv4.torrent > "$HOME/wss-leech.log" 2>&1 & cd .. # Check for completion @@ -341,6 +351,12 @@ cat "wss-seed.log" sleep 1 +echo "" +echo "# --- WSS seed log 2 --- #" +cat "wss-seed2.log" + +sleep 1 + echo "" echo "# --- WSS leech log --- #" cat "wss-leech.log" From fad3e1ea0df81ee99fac6708d56867579d663553 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20Frosteg=C3=A5rd?= Date: Mon, 16 Aug 2021 20:44:49 +0200 Subject: [PATCH 11/22] CI: wait max 60 seconds --- .github/actions/test-transfer/entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/test-transfer/entrypoint.sh b/.github/actions/test-transfer/entrypoint.sh index 5b36825..61f061e 100755 --- a/.github/actions/test-transfer/entrypoint.sh +++ b/.github/actions/test-transfer/entrypoint.sh @@ -270,7 +270,7 @@ i="0" echo "Watching for finished files.." -while [ $i -lt 300 ] +while [ $i -lt 60 ] do if test -f "leech/http-test-ipv4"; then if grep -q "http-test-ipv4" "leech/http-test-ipv4"; then From 015f31b8cc598d5ac9276d3582c8ac476585fcb6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20Frosteg=C3=A5rd?= Date: Mon, 16 Aug 2021 20:50:30 +0200 Subject: [PATCH 12/22] CI: fix gotorrent install path --- .github/actions/test-transfer/entrypoint.sh | 7 ++++--- Cargo.lock | 2 -- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/actions/test-transfer/entrypoint.sh b/.github/actions/test-transfer/entrypoint.sh index 61f061e..5652ecd 100755 --- a/.github/actions/test-transfer/entrypoint.sh +++ b/.github/actions/test-transfer/entrypoint.sh @@ -26,8 +26,9 @@ $SUDO apt-get install -y cmake libssl-dev screen rtorrent mktorrent ssl-cert ca- git clone https://github.com/anacrolix/torrent.git gotorrent cd gotorrent -go build -o ../gotorrent ./cmd/torrent +go build -o $HOME/gotorrent ./cmd/torrent cd .. +file $HOME/gotorrent $SUDO curl -sL https://deb.nodesource.com/setup_15.x | bash - $SUDO apt-get install nodejs -y @@ -242,7 +243,7 @@ fi # Start seeding ws client 2 cd seed -GOPPROF=http GODEBUG=x509ignoreCN=0 ../gotorrent download --seed ../torrents/wss-ipv4.torrent > "$HOME/wss-seed2.log" 2>&1 & +GOPPROF=http GODEBUG=x509ignoreCN=0 $HOME/gotorrent download --seed ../torrents/wss-ipv4.torrent > "$HOME/wss-seed2.log" 2>&1 & cd .. # Start leeching clients @@ -256,7 +257,7 @@ screen -dmS rtorrent-leech rtorrent # ./node_modules/webtorrent-hybrid/bin/cmd.js download ./torrents/wss-ipv4.torrent -o leech > "$HOME/wss-leech.log" 2>&1 & cd leech -GOPPROF=http GODEBUG=x509ignoreCN=0 ../gotorrent download ../torrents/wss-ipv4.torrent > "$HOME/wss-leech.log" 2>&1 & +GOPPROF=http GODEBUG=x509ignoreCN=0 $HOME/gotorrent download ../torrents/wss-ipv4.torrent > "$HOME/wss-leech.log" 2>&1 & cd .. # Check for completion diff --git a/Cargo.lock b/Cargo.lock index 6d63746..19829c4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,7 +1,5 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -version = 3 - [[package]] name = "addr2line" version = "0.16.0" From aae274156e93ec867d646754ae371b2fbdf5b1de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20Frosteg=C3=A5rd?= Date: Mon, 16 Aug 2021 20:51:20 +0200 Subject: [PATCH 13/22] CI: use node 16, use latest webtorrent-hybrid --- .github/actions/test-transfer/entrypoint.sh | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/.github/actions/test-transfer/entrypoint.sh b/.github/actions/test-transfer/entrypoint.sh index 5652ecd..0a2d7a3 100755 --- a/.github/actions/test-transfer/entrypoint.sh +++ b/.github/actions/test-transfer/entrypoint.sh @@ -30,7 +30,7 @@ go build -o $HOME/gotorrent ./cmd/torrent cd .. file $HOME/gotorrent -$SUDO curl -sL https://deb.nodesource.com/setup_15.x | bash - +$SUDO curl -sL https://deb.nodesource.com/setup_16.x | bash - $SUDO apt-get install nodejs -y rtorrent -h @@ -118,11 +118,7 @@ cp -r torrents torrents-leech # Setup wss seeding client -# Seems to fix webtorrent-hybrid install error. -# Will likely be fixed in later versions of webtorrent-hybrid. -npm install @mapbox/node-pre-gyp - -npm install webtorrent-hybrid@4.0.3 +npm install webtorrent-hybrid echo " // Start webtorrent seeder from data file, create torrent, write it to file, From 28d37721b7f5f9e74523f8647227eb2259a629a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20Frosteg=C3=A5rd?= Date: Mon, 16 Aug 2021 20:56:06 +0200 Subject: [PATCH 14/22] CI: install node-pre-gyp again.. --- .github/actions/test-transfer/entrypoint.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/actions/test-transfer/entrypoint.sh b/.github/actions/test-transfer/entrypoint.sh index 0a2d7a3..0f034f6 100755 --- a/.github/actions/test-transfer/entrypoint.sh +++ b/.github/actions/test-transfer/entrypoint.sh @@ -118,7 +118,11 @@ cp -r torrents torrents-leech # Setup wss seeding client -npm install webtorrent-hybrid +# Seems to fix webtorrent-hybrid install error. +# Will likely be fixed in later versions of webtorrent-hybrid. +npm install @mapbox/node-pre-gyp + +npm install webtorrent-hybrid@4.0.3 echo " // Start webtorrent seeder from data file, create torrent, write it to file, From 9971ef8a857ad1decee4f2262daa65a94c0442d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20Frosteg=C3=A5rd?= Date: Mon, 16 Aug 2021 21:04:18 +0200 Subject: [PATCH 15/22] CI: use other port for gotorrent client 2 --- .github/actions/test-transfer/entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/test-transfer/entrypoint.sh b/.github/actions/test-transfer/entrypoint.sh index 0f034f6..8d95a8b 100755 --- a/.github/actions/test-transfer/entrypoint.sh +++ b/.github/actions/test-transfer/entrypoint.sh @@ -257,7 +257,7 @@ screen -dmS rtorrent-leech rtorrent # ./node_modules/webtorrent-hybrid/bin/cmd.js download ./torrents/wss-ipv4.torrent -o leech > "$HOME/wss-leech.log" 2>&1 & cd leech -GOPPROF=http GODEBUG=x509ignoreCN=0 $HOME/gotorrent download ../torrents/wss-ipv4.torrent > "$HOME/wss-leech.log" 2>&1 & +GOPPROF=http GODEBUG=x509ignoreCN=0 $HOME/gotorrent download --addr ":43000" ../torrents/wss-ipv4.torrent > "$HOME/wss-leech.log" 2>&1 & cd .. # Check for completion From 898926cd6c8bab938f7bfb8e21c6fad81991a1ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20Frosteg=C3=A5rd?= Date: Mon, 16 Aug 2021 21:16:42 +0200 Subject: [PATCH 16/22] CI: gotorrent: add more flags --- .github/actions/test-transfer/entrypoint.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/actions/test-transfer/entrypoint.sh b/.github/actions/test-transfer/entrypoint.sh index 8d95a8b..b44bfc4 100755 --- a/.github/actions/test-transfer/entrypoint.sh +++ b/.github/actions/test-transfer/entrypoint.sh @@ -240,10 +240,10 @@ else exit 1 fi -# Start seeding ws client 2 +# Start seeding ws client 2, which is the one that actually does uploading cd seed -GOPPROF=http GODEBUG=x509ignoreCN=0 $HOME/gotorrent download --seed ../torrents/wss-ipv4.torrent > "$HOME/wss-seed2.log" 2>&1 & +GOPPROF=http GODEBUG=x509ignoreCN=0 $HOME/gotorrent download --dht=false --tcppeers=false --utppeers=false --pex=false --stats --seed ../torrents/wss-ipv4.torrent > "$HOME/wss-seed2.log" 2>&1 & cd .. # Start leeching clients @@ -257,7 +257,7 @@ screen -dmS rtorrent-leech rtorrent # ./node_modules/webtorrent-hybrid/bin/cmd.js download ./torrents/wss-ipv4.torrent -o leech > "$HOME/wss-leech.log" 2>&1 & cd leech -GOPPROF=http GODEBUG=x509ignoreCN=0 $HOME/gotorrent download --addr ":43000" ../torrents/wss-ipv4.torrent > "$HOME/wss-leech.log" 2>&1 & +GOPPROF=http GODEBUG=x509ignoreCN=0 $HOME/gotorrent download --dht=false --tcppeers=false --utppeers=false --pex=false --stats --addr ":43000" ../torrents/wss-ipv4.torrent > "$HOME/wss-leech.log" 2>&1 & cd .. # Check for completion From bc0441c43574bc833bd8278a4dca45fb74708d55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20Frosteg=C3=A5rd?= Date: Mon, 16 Aug 2021 21:33:54 +0200 Subject: [PATCH 17/22] CI: remove node, just use gotorrent --- .github/actions/test-transfer/entrypoint.sh | 135 +------------------- 1 file changed, 7 insertions(+), 128 deletions(-) diff --git a/.github/actions/test-transfer/entrypoint.sh b/.github/actions/test-transfer/entrypoint.sh index b44bfc4..396f616 100755 --- a/.github/actions/test-transfer/entrypoint.sh +++ b/.github/actions/test-transfer/entrypoint.sh @@ -30,11 +30,6 @@ go build -o $HOME/gotorrent ./cmd/torrent cd .. file $HOME/gotorrent -$SUDO curl -sL https://deb.nodesource.com/setup_16.x | bash - -$SUDO apt-get install nodejs -y - -rtorrent -h - # Clone repository if necessary, go to repository directory if [[ -z "${GITHUB_WORKSPACE}" ]]; then @@ -112,107 +107,17 @@ echo "wss-test-ipv4" > seed/wss-test-ipv4 mktorrent -p -o "torrents/http-ipv4.torrent" -a "http://127.0.0.1:3000/announce" "seed/http-test-ipv4" mktorrent -p -o "torrents/tls-ipv4.torrent" -a "https://example.com:3001/announce" "seed/tls-test-ipv4" mktorrent -p -o "torrents/udp-ipv4.torrent" -a "udp://127.0.0.1:3000" "seed/udp-test-ipv4" +mktorrent -p -o "torrents/wss-ipv4.torrent" -a "wss://example.com:3002" "seed/wss-test-ipv4" cp -r torrents torrents-seed cp -r torrents torrents-leech # Setup wss seeding client -# Seems to fix webtorrent-hybrid install error. -# Will likely be fixed in later versions of webtorrent-hybrid. -npm install @mapbox/node-pre-gyp - -npm install webtorrent-hybrid@4.0.3 - -echo " -// Start webtorrent seeder from data file, create torrent, write it to file, -// output info - -var WebTorrent = require('webtorrent-hybrid') -var fs = require('fs') - -// WebTorrent seems to use same peer id for different -// clients in some cases (I don't know how) -peerId = 'ae61b6f4a5be4ada48333891512db5e90347d736' -announceUrl = 'wss://example.com:3002' -dataFile = './seed/wss-test-ipv4' -torrentFile = './torrents/wss-ipv4.torrent' -trackerFile = './wss-trackers.txt' - -function createSeeder(){ - console.log('creating seeder..') - - var seeder = new WebTorrent({ dht: false, webSeeds: false, peerId: peerId }) - seeder.on('error', function(err) { - console.log('seeder error: ' + err) - }) - - var addOpts = { - announceList: [[announceUrl]], - announce: [announceUrl], - private: true - } - - seeder.seed(dataFile, addOpts, function(torrent){ - console.log('seeding') - console.log(torrent.announce) - - torrent.announce.forEach(function(item, index){ - if (item != announceUrl) { - fs.appendFile(trackerFile, '127.0.0.1 '.concat(item.replace(/(^\w+:|^)\/\//, ''), '\n'), function(err){ - if (err){ - console.log(err) - } - }) - } - }); - - fs.writeFile(torrentFile, torrent.torrentFile, function(err){ - if (err){ - console.log(err) - } - }) - - torrent.on('warning', function(err){ - console.log(err) - }); - - torrent.on('error', function(err){ - console.log(err) - }); - - torrent.on('download', function(bytes){ - console.log('downloaded bytes: ' + bytes) - }); - - torrent.on('upload', function(bytes){ - console.log('uploaded bytes: ' + bytes) - }); - - torrent.on('wire', function(wire, addr){ - console.log('connected to peer with addr: ' + addr) - }); - - torrent.on('noPeers', function(announceType){ - console.log('no peers received with announce type: ' + announceType) - }) - - torrent.on('done', function(){ - console.log('done') - }); - - }) -} - -createSeeder() -" > wss-seeder.js - -cat wss-seeder.js - -# Start seeding ws client, create torrent - -echo "Starting seeding ws client" -node wss-seeder.js > "$HOME/wss-seed.log" 2>&1 & +echo "Starting seeding wss client" +cd seed +GOPPROF=http GODEBUG=x509ignoreCN=0 $HOME/gotorrent download --dht=false --tcppeers=false --utppeers=false --pex=false --stats --seed ../torrents/wss-ipv4.torrent > "$HOME/wss-seed.log" 2>&1 & +cd .. # Start seeding rtorrent client @@ -222,30 +127,11 @@ schedule2 = watch_directory,5,5,load.start=$HOME/torrents-seed/*.torrent" > ~/.r echo "Starting seeding rtorrent client" screen -dmS rtorrent-seed rtorrent -# Give seeding clients time to write trackers to file, load config files etc +# Give seeding clients time to load config files etc echo "Waiting for a while" sleep 30 -# Forbid access to other trackers used by webtorrent-hybrid - -if test -f "wss-trackers.txt"; then - $SUDO cat wss-trackers.txt >> /etc/hosts - echo "Added the following lines to /etc/hosts:" - cat wss-trackers.txt - echo "" -else - echo "ERROR: WSS TRACKER FILE NOT FOUND. Seeding client log:" - cat "$HOME/wss-seed.log" - exit 1 -fi - -# Start seeding ws client 2, which is the one that actually does uploading - -cd seed -GOPPROF=http GODEBUG=x509ignoreCN=0 $HOME/gotorrent download --dht=false --tcppeers=false --utppeers=false --pex=false --stats --seed ../torrents/wss-ipv4.torrent > "$HOME/wss-seed2.log" 2>&1 & -cd .. - # Start leeching clients echo "directory.default.set = $HOME/leech @@ -254,8 +140,7 @@ schedule2 = watch_directory,5,5,load.start=$HOME/torrents-leech/*.torrent" > ~/. echo "Starting leeching client.." screen -dmS rtorrent-leech rtorrent -# ./node_modules/webtorrent-hybrid/bin/cmd.js download ./torrents/wss-ipv4.torrent -o leech > "$HOME/wss-leech.log" 2>&1 & - +echo "Starting leeching wss client" cd leech GOPPROF=http GODEBUG=x509ignoreCN=0 $HOME/gotorrent download --dht=false --tcppeers=false --utppeers=false --pex=false --stats --addr ":43000" ../torrents/wss-ipv4.torrent > "$HOME/wss-leech.log" 2>&1 & cd .. @@ -352,12 +237,6 @@ cat "wss-seed.log" sleep 1 -echo "" -echo "# --- WSS seed log 2 --- #" -cat "wss-seed2.log" - -sleep 1 - echo "" echo "# --- WSS leech log --- #" cat "wss-leech.log" From 81dca0fb50275ac5c0ad0262001466c274e05d35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20Frosteg=C3=A5rd?= Date: Mon, 16 Aug 2021 21:37:57 +0200 Subject: [PATCH 18/22] CI: use specific gotorrent commit --- .github/actions/test-transfer/entrypoint.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/actions/test-transfer/entrypoint.sh b/.github/actions/test-transfer/entrypoint.sh index 396f616..77fd0ad 100755 --- a/.github/actions/test-transfer/entrypoint.sh +++ b/.github/actions/test-transfer/entrypoint.sh @@ -26,6 +26,7 @@ $SUDO apt-get install -y cmake libssl-dev screen rtorrent mktorrent ssl-cert ca- git clone https://github.com/anacrolix/torrent.git gotorrent cd gotorrent +git checkout 16176b762e4a840fc5dfe3b1dfd2d6fa853b68d7 go build -o $HOME/gotorrent ./cmd/torrent cd .. file $HOME/gotorrent From fae0692f907de55a931b0b44af5573340045c601 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20Frosteg=C3=A5rd?= Date: Mon, 16 Aug 2021 23:22:56 +0200 Subject: [PATCH 19/22] Upgrade tungstenite --- Cargo.lock | 15 ++------------- aquatic_ws/Cargo.toml | 2 +- aquatic_ws/src/lib/network/connection.rs | 6 +++--- aquatic_ws_load_test/Cargo.toml | 2 +- aquatic_ws_protocol/Cargo.toml | 2 +- 5 files changed, 8 insertions(+), 19 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 19829c4..e66e4cd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -790,15 +790,6 @@ dependencies = [ "regex", ] -[[package]] -name = "input_buffer" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f97967975f448f1a7ddb12b0bc41069d09ed6a1c161a92687e057325db35d413" -dependencies = [ - "bytes", -] - [[package]] name = "instant" version = "0.1.10" @@ -1673,18 +1664,16 @@ dependencies = [ [[package]] name = "tungstenite" -version = "0.13.0" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fe8dada8c1a3aeca77d6b51a4f1314e0f4b8e438b7b1b71e3ddaca8080e4093" +checksum = "983d40747bce878d2fb67d910dcb8bd3eca2b2358540c3cc1b98c027407a3ae3" dependencies = [ "base64", "byteorder", "bytes", "http", "httparse", - "input_buffer", "log", - "native-tls", "rand", "sha-1", "thiserror", diff --git a/aquatic_ws/Cargo.toml b/aquatic_ws/Cargo.toml index b5597e8..1087208 100644 --- a/aquatic_ws/Cargo.toml +++ b/aquatic_ws/Cargo.toml @@ -34,7 +34,7 @@ privdrop = "0.5" rand = { version = "0.8", features = ["small_rng"] } serde = { version = "1", features = ["derive"] } socket2 = { version = "0.4.1", features = ["all"] } -tungstenite = "0.13" +tungstenite = "0.15" [dev-dependencies] quickcheck = "1.0" diff --git a/aquatic_ws/src/lib/network/connection.rs b/aquatic_ws/src/lib/network/connection.rs index a8a8d5c..fbaf299 100644 --- a/aquatic_ws/src/lib/network/connection.rs +++ b/aquatic_ws/src/lib/network/connection.rs @@ -9,7 +9,7 @@ use mio::{Poll, Token}; use native_tls::{MidHandshakeTlsStream, TlsAcceptor, TlsStream}; use tungstenite::handshake::{server::NoCallback, HandshakeError, MidHandshake}; use tungstenite::protocol::WebSocketConfig; -use tungstenite::server::ServerHandshake; +use tungstenite::ServerHandshake; use tungstenite::WebSocket; use crate::common::*; @@ -111,7 +111,7 @@ impl HandshakeMachine { if let Some(tls_acceptor) = opt_tls_acceptor { Self::handle_tls_handshake_result(tls_acceptor.accept(stream)) } else { - let handshake_result = ::tungstenite::server::accept_with_config( + let handshake_result = ::tungstenite::accept_with_config( Stream::TcpStream(stream), Some(ws_config), ); @@ -120,7 +120,7 @@ impl HandshakeMachine { } } HandshakeMachine::TlsStream(stream) => { - let handshake_result = ::tungstenite::server::accept(Stream::TlsStream(stream)); + let handshake_result = ::tungstenite::accept(Stream::TlsStream(stream)); Self::handle_ws_handshake_result(handshake_result) } diff --git a/aquatic_ws_load_test/Cargo.toml b/aquatic_ws_load_test/Cargo.toml index 54a5923..abf14ab 100644 --- a/aquatic_ws_load_test/Cargo.toml +++ b/aquatic_ws_load_test/Cargo.toml @@ -21,7 +21,7 @@ rand_distr = "0.4" serde = { version = "1", features = ["derive"] } serde_json = "1" slab = "0.4" -tungstenite = "0.13" +tungstenite = "0.15" [dev-dependencies] quickcheck = "1.0" diff --git a/aquatic_ws_protocol/Cargo.toml b/aquatic_ws_protocol/Cargo.toml index 5817ffc..1830725 100644 --- a/aquatic_ws_protocol/Cargo.toml +++ b/aquatic_ws_protocol/Cargo.toml @@ -22,7 +22,7 @@ hashbrown = { version = "0.11.2", features = ["serde"] } serde = { version = "1", features = ["derive"] } serde_json = "1" simd-json = { version = "0.4.7", features = ["allow-non-simd"] } -tungstenite = "0.13" +tungstenite = "0.15" [dev-dependencies] criterion = "0.3" From a121a58a8d23e5ee48bdc12d479c34002e26a68b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20Frosteg=C3=A5rd?= Date: Mon, 16 Aug 2021 23:23:12 +0200 Subject: [PATCH 20/22] Run cargo update --- Cargo.lock | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e66e4cd..aadb41a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,5 +1,7 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. +version = 3 + [[package]] name = "addr2line" version = "0.16.0" @@ -335,9 +337,9 @@ dependencies = [ [[package]] name = "bitflags" -version = "1.3.1" +version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2da1976d75adbe5fbc88130ecd119529cf1cc6a93ae1546d8696ee66f0d21af1" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "block-buffer" From 83d450e014122ece182cc707d233e91900bb89ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20Frosteg=C3=A5rd?= Date: Tue, 17 Aug 2021 11:39:08 +0200 Subject: [PATCH 21/22] README: remove section about lack of wss file transfer CI --- README.md | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/README.md b/README.md index 86ef704..1406376 100644 --- a/README.md +++ b/README.md @@ -149,11 +149,7 @@ exceptions: For information about running over TLS, please refer to the TLS subsection of the `aquatic_http` section above. -`aquatic_ws` is experimental software. Connections are established -successfully when using `aquatic_ws_load_test`, but so far, I haven't been able -to implement CI for testing if aquatic_ws works as the tracker for a full -file transfer session between two real-world clients. One reason for this -is the general lack of high-quality WebTorrent clients. +`aquatic_ws` is experimental software. ## Load testing From a21f8dbc6e47610caa6694fcaf5cbd6af5633f70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20Frosteg=C3=A5rd?= Date: Tue, 17 Aug 2021 11:46:06 +0200 Subject: [PATCH 22/22] README: reword architecture overview section --- README.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 1406376..a0a99bf 100644 --- a/README.md +++ b/README.md @@ -17,10 +17,12 @@ Distributed under Apache 2.0 license (details in `LICENSE` file.) ## Technical overview of tracker design One or more socket workers open sockets, read and parse requests from peers and -send them through channels to request workers. They in turn go through the -requests, update internal state as appropriate and generate responses, which -are sent back to the socket workers, which serialize them and send them to -peers. This design means little waiting for locks on internal state occurs, +send them through channels to request workers. The request workers go through +the requests, update shared internal tracker state as appropriate and generate +responses that are sent back to the socket workers. The responses are then +serialized and sent back to the peers. + +This design means little waiting for locks on internal state occurs, while network work can be efficiently distributed over multiple threads, making use of SO_REUSEPORT setting.