From ad7e46478878c9bb341fe42eb7a790562c4db3a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20Frosteg=C3=A5rd?= Date: Tue, 26 Oct 2021 15:26:06 +0200 Subject: [PATCH 01/59] aquatic_http: split into mio and glommio modules --- Cargo.lock | 3 + aquatic_http/Cargo.toml | 22 ++- aquatic_http/src/lib/glommio/mod.rs | 10 ++ aquatic_http/src/lib/lib.rs | 155 ++---------------- aquatic_http/src/lib/{ => mio}/common.rs | 0 aquatic_http/src/lib/{ => mio}/handler.rs | 2 +- aquatic_http/src/lib/mio/mod.rs | 150 +++++++++++++++++ .../src/lib/{ => mio}/network/connection.rs | 2 +- aquatic_http/src/lib/{ => mio}/network/mod.rs | 2 +- .../src/lib/{ => mio}/network/stream.rs | 0 .../src/lib/{ => mio}/network/utils.rs | 0 aquatic_http/src/lib/{ => mio}/tasks.rs | 3 +- 12 files changed, 196 insertions(+), 153 deletions(-) create mode 100644 aquatic_http/src/lib/glommio/mod.rs rename aquatic_http/src/lib/{ => mio}/common.rs (100%) rename aquatic_http/src/lib/{ => mio}/handler.rs (99%) create mode 100644 aquatic_http/src/lib/mio/mod.rs rename aquatic_http/src/lib/{ => mio}/network/connection.rs (99%) rename aquatic_http/src/lib/{ => mio}/network/mod.rs (99%) rename aquatic_http/src/lib/{ => mio}/network/stream.rs (100%) rename aquatic_http/src/lib/{ => mio}/network/utils.rs (100%) rename aquatic_http/src/lib/{ => mio}/tasks.rs (97%) diff --git a/Cargo.lock b/Cargo.lock index c83d439..607e122 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -91,8 +91,11 @@ dependencies = [ "aquatic_cli_helpers", "aquatic_common", "aquatic_http_protocol", + "cfg-if", "crossbeam-channel", "either", + "futures-lite", + "glommio", "hashbrown 0.11.2", "histogram", "indexmap", diff --git a/aquatic_http/Cargo.toml b/aquatic_http/Cargo.toml index f604413..092bd5d 100644 --- a/aquatic_http/Cargo.toml +++ b/aquatic_http/Cargo.toml @@ -15,28 +15,40 @@ path = "src/lib/lib.rs" name = "aquatic_http" path = "src/bin/main.rs" +[features] +default = ["with-mio"] +with-glommio = ["glommio", "futures-lite"] +with-mio = ["crossbeam-channel", "histogram", "mio", "native-tls", "socket2"] + [dependencies] anyhow = "1" aquatic_cli_helpers = "0.1.0" aquatic_common = "0.1.0" aquatic_http_protocol = "0.1.0" -crossbeam-channel = "0.5" +cfg-if = "1" either = "1" hashbrown = "0.11.2" -histogram = "0.6" indexmap = "1" itoa = "0.4" log = "0.4" mimalloc = { version = "0.1", default-features = false } memchr = "2" -mio = { version = "0.7", features = ["tcp", "os-poll", "os-util"] } -native-tls = "0.2" parking_lot = "0.11" privdrop = "0.5" rand = { version = "0.8", features = ["small_rng"] } serde = { version = "1", features = ["derive"] } smartstring = "0.2" -socket2 = { version = "0.4.1", features = ["all"] } + +# mio +crossbeam-channel = { version = "0.5", optional = true } +histogram = { version = "0.6", optional = true } +mio = { version = "0.7", features = ["tcp", "os-poll", "os-util"], optional = true } +native-tls = { version = "0.2", optional = true } +socket2 = { version = "0.4.1", features = ["all"], optional = true } + +# glommio +glommio = { git = "https://github.com/DataDog/glommio.git", rev = "4e6b14772da2f4325271fbcf12d24cf91ed466e5", optional = true } +futures-lite = { version = "1", optional = true } [dev-dependencies] quickcheck = "1.0" diff --git a/aquatic_http/src/lib/glommio/mod.rs b/aquatic_http/src/lib/glommio/mod.rs new file mode 100644 index 0000000..0f2955f --- /dev/null +++ b/aquatic_http/src/lib/glommio/mod.rs @@ -0,0 +1,10 @@ +use glommio::prelude::*; + +use crate::config::Config; + +pub fn run( + config: Config, +) -> anyhow::Result<()> { + + Ok(()) +} \ No newline at end of file diff --git a/aquatic_http/src/lib/lib.rs b/aquatic_http/src/lib/lib.rs index 3f96ef2..ee6f8e8 100644 --- a/aquatic_http/src/lib/lib.rs +++ b/aquatic_http/src/lib/lib.rs @@ -1,153 +1,20 @@ -use std::sync::Arc; -use std::thread::Builder; -use std::time::Duration; +use cfg_if::cfg_if; -use anyhow::Context; -use mio::{Poll, Waker}; -use parking_lot::Mutex; -use privdrop::PrivDrop; - -pub mod common; pub mod config; -pub mod handler; -pub mod network; -pub mod tasks; -use common::*; -use config::Config; -use network::utils::create_tls_acceptor; +#[cfg(feature = "with-mio")] +pub mod mio; +#[cfg(all(feature = "with-glommio", target_os = "linux"))] +pub mod glommio; pub const APP_NAME: &str = "aquatic_http: HTTP/TLS BitTorrent tracker"; -pub fn run(config: Config) -> anyhow::Result<()> { - let state = State::default(); - - tasks::update_access_list(&config, &state); - - start_workers(config.clone(), state.clone())?; - - loop { - ::std::thread::sleep(Duration::from_secs(config.cleaning.interval)); - - tasks::update_access_list(&config, &state); - - state - .torrent_maps - .lock() - .clean(&config, &state.access_list.load_full()); - } -} - -pub fn start_workers(config: Config, state: State) -> anyhow::Result<()> { - let opt_tls_acceptor = create_tls_acceptor(&config.network.tls)?; - - let (request_channel_sender, request_channel_receiver) = ::crossbeam_channel::unbounded(); - - let mut out_message_senders = Vec::new(); - let mut wakers = Vec::new(); - - let socket_worker_statuses: SocketWorkerStatuses = { - let mut statuses = Vec::new(); - - for _ in 0..config.socket_workers { - statuses.push(None); - } - - Arc::new(Mutex::new(statuses)) - }; - - for i in 0..config.socket_workers { - let config = config.clone(); - let state = state.clone(); - let socket_worker_statuses = socket_worker_statuses.clone(); - let request_channel_sender = request_channel_sender.clone(); - let opt_tls_acceptor = opt_tls_acceptor.clone(); - let poll = Poll::new().expect("create poll"); - let waker = Arc::new(Waker::new(poll.registry(), CHANNEL_TOKEN).expect("create waker")); - - let (response_channel_sender, response_channel_receiver) = ::crossbeam_channel::unbounded(); - - out_message_senders.push(response_channel_sender); - wakers.push(waker); - - Builder::new() - .name(format!("socket-{:02}", i + 1)) - .spawn(move || { - network::run_socket_worker( - config, - state, - i, - socket_worker_statuses, - request_channel_sender, - response_channel_receiver, - opt_tls_acceptor, - poll, - ); - })?; - } - - // Wait for socket worker statuses. On error from any, quit program. - // On success from all, drop privileges if corresponding setting is set - // and continue program. - loop { - ::std::thread::sleep(::std::time::Duration::from_millis(10)); - - if let Some(statuses) = socket_worker_statuses.try_lock() { - for opt_status in statuses.iter() { - if let Some(Err(err)) = opt_status { - return Err(::anyhow::anyhow!(err.to_owned())); - } - } - - if statuses.iter().all(Option::is_some) { - if config.privileges.drop_privileges { - PrivDrop::default() - .chroot(config.privileges.chroot_path.clone()) - .user(config.privileges.user.clone()) - .apply() - .context("Couldn't drop root privileges")?; - } - - break; - } +pub fn run(config: config::Config) -> ::anyhow::Result<()> { + cfg_if! { + if #[cfg(all(feature = "with-glommio", target_os = "linux"))] { + glommio::run(config) + } else { + mio::run(config) } } - - let response_channel_sender = ResponseChannelSender::new(out_message_senders); - - for i in 0..config.request_workers { - let config = config.clone(); - let state = state.clone(); - let request_channel_receiver = request_channel_receiver.clone(); - let response_channel_sender = response_channel_sender.clone(); - let wakers = wakers.clone(); - - Builder::new() - .name(format!("request-{:02}", i + 1)) - .spawn(move || { - handler::run_request_worker( - config, - state, - request_channel_receiver, - response_channel_sender, - wakers, - ); - })?; - } - - if config.statistics.interval != 0 { - let state = state.clone(); - let config = config.clone(); - - Builder::new() - .name("statistics".to_string()) - .spawn(move || loop { - ::std::thread::sleep(Duration::from_secs(config.statistics.interval)); - - tasks::print_statistics(&state); - }) - .expect("spawn statistics thread"); - } - - Ok(()) } diff --git a/aquatic_http/src/lib/common.rs b/aquatic_http/src/lib/mio/common.rs similarity index 100% rename from aquatic_http/src/lib/common.rs rename to aquatic_http/src/lib/mio/common.rs diff --git a/aquatic_http/src/lib/handler.rs b/aquatic_http/src/lib/mio/handler.rs similarity index 99% rename from aquatic_http/src/lib/handler.rs rename to aquatic_http/src/lib/mio/handler.rs index cd823d8..c54df07 100644 --- a/aquatic_http/src/lib/handler.rs +++ b/aquatic_http/src/lib/mio/handler.rs @@ -13,8 +13,8 @@ use aquatic_common::extract_response_peers; use aquatic_http_protocol::request::*; use aquatic_http_protocol::response::*; -use crate::common::*; use crate::config::Config; +use super::common::*; pub fn run_request_worker( config: Config, diff --git a/aquatic_http/src/lib/mio/mod.rs b/aquatic_http/src/lib/mio/mod.rs new file mode 100644 index 0000000..ea61019 --- /dev/null +++ b/aquatic_http/src/lib/mio/mod.rs @@ -0,0 +1,150 @@ +use std::sync::Arc; +use std::thread::Builder; +use std::time::Duration; + +use anyhow::Context; +use mio::{Poll, Waker}; +use parking_lot::Mutex; +use privdrop::PrivDrop; + +pub mod common; +pub mod handler; +pub mod network; +pub mod tasks; + +use crate::config::Config; +use common::*; +use network::utils::create_tls_acceptor; + +pub fn run(config: Config) -> anyhow::Result<()> { + let state = State::default(); + + tasks::update_access_list(&config, &state); + + start_workers(config.clone(), state.clone())?; + + loop { + ::std::thread::sleep(Duration::from_secs(config.cleaning.interval)); + + tasks::update_access_list(&config, &state); + + state + .torrent_maps + .lock() + .clean(&config, &state.access_list.load_full()); + } +} + +pub fn start_workers(config: Config, state: State) -> anyhow::Result<()> { + let opt_tls_acceptor = create_tls_acceptor(&config.network.tls)?; + + let (request_channel_sender, request_channel_receiver) = ::crossbeam_channel::unbounded(); + + let mut out_message_senders = Vec::new(); + let mut wakers = Vec::new(); + + let socket_worker_statuses: SocketWorkerStatuses = { + let mut statuses = Vec::new(); + + for _ in 0..config.socket_workers { + statuses.push(None); + } + + Arc::new(Mutex::new(statuses)) + }; + + for i in 0..config.socket_workers { + let config = config.clone(); + let state = state.clone(); + let socket_worker_statuses = socket_worker_statuses.clone(); + let request_channel_sender = request_channel_sender.clone(); + let opt_tls_acceptor = opt_tls_acceptor.clone(); + let poll = Poll::new().expect("create poll"); + let waker = Arc::new(Waker::new(poll.registry(), CHANNEL_TOKEN).expect("create waker")); + + let (response_channel_sender, response_channel_receiver) = ::crossbeam_channel::unbounded(); + + out_message_senders.push(response_channel_sender); + wakers.push(waker); + + Builder::new() + .name(format!("socket-{:02}", i + 1)) + .spawn(move || { + network::run_socket_worker( + config, + state, + i, + socket_worker_statuses, + request_channel_sender, + response_channel_receiver, + opt_tls_acceptor, + poll, + ); + })?; + } + + // Wait for socket worker statuses. On error from any, quit program. + // On success from all, drop privileges if corresponding setting is set + // and continue program. + loop { + ::std::thread::sleep(::std::time::Duration::from_millis(10)); + + if let Some(statuses) = socket_worker_statuses.try_lock() { + for opt_status in statuses.iter() { + if let Some(Err(err)) = opt_status { + return Err(::anyhow::anyhow!(err.to_owned())); + } + } + + if statuses.iter().all(Option::is_some) { + if config.privileges.drop_privileges { + PrivDrop::default() + .chroot(config.privileges.chroot_path.clone()) + .user(config.privileges.user.clone()) + .apply() + .context("Couldn't drop root privileges")?; + } + + break; + } + } + } + + let response_channel_sender = ResponseChannelSender::new(out_message_senders); + + for i in 0..config.request_workers { + let config = config.clone(); + let state = state.clone(); + let request_channel_receiver = request_channel_receiver.clone(); + let response_channel_sender = response_channel_sender.clone(); + let wakers = wakers.clone(); + + Builder::new() + .name(format!("request-{:02}", i + 1)) + .spawn(move || { + handler::run_request_worker( + config, + state, + request_channel_receiver, + response_channel_sender, + wakers, + ); + })?; + } + + if config.statistics.interval != 0 { + let state = state.clone(); + let config = config.clone(); + + Builder::new() + .name("statistics".to_string()) + .spawn(move || loop { + ::std::thread::sleep(Duration::from_secs(config.statistics.interval)); + + tasks::print_statistics(&state); + }) + .expect("spawn statistics thread"); + } + + Ok(()) +} diff --git a/aquatic_http/src/lib/network/connection.rs b/aquatic_http/src/lib/mio/network/connection.rs similarity index 99% rename from aquatic_http/src/lib/network/connection.rs rename to aquatic_http/src/lib/mio/network/connection.rs index 4324dfc..57fee71 100644 --- a/aquatic_http/src/lib/network/connection.rs +++ b/aquatic_http/src/lib/mio/network/connection.rs @@ -10,7 +10,7 @@ use native_tls::{MidHandshakeTlsStream, TlsAcceptor}; use aquatic_http_protocol::request::{Request, RequestParseError}; -use crate::common::*; +use crate::mio::common::*; use super::stream::Stream; diff --git a/aquatic_http/src/lib/network/mod.rs b/aquatic_http/src/lib/mio/network/mod.rs similarity index 99% rename from aquatic_http/src/lib/network/mod.rs rename to aquatic_http/src/lib/mio/network/mod.rs index dd7d889..ed0a552 100644 --- a/aquatic_http/src/lib/network/mod.rs +++ b/aquatic_http/src/lib/mio/network/mod.rs @@ -13,7 +13,7 @@ use native_tls::TlsAcceptor; use aquatic_http_protocol::response::*; -use crate::common::*; +use crate::mio::common::*; use crate::config::Config; pub mod connection; diff --git a/aquatic_http/src/lib/network/stream.rs b/aquatic_http/src/lib/mio/network/stream.rs similarity index 100% rename from aquatic_http/src/lib/network/stream.rs rename to aquatic_http/src/lib/mio/network/stream.rs diff --git a/aquatic_http/src/lib/network/utils.rs b/aquatic_http/src/lib/mio/network/utils.rs similarity index 100% rename from aquatic_http/src/lib/network/utils.rs rename to aquatic_http/src/lib/mio/network/utils.rs diff --git a/aquatic_http/src/lib/tasks.rs b/aquatic_http/src/lib/mio/tasks.rs similarity index 97% rename from aquatic_http/src/lib/tasks.rs rename to aquatic_http/src/lib/mio/tasks.rs index 341a434..67c9158 100644 --- a/aquatic_http/src/lib/tasks.rs +++ b/aquatic_http/src/lib/mio/tasks.rs @@ -2,7 +2,8 @@ use histogram::Histogram; use aquatic_common::access_list::{AccessListMode, AccessListQuery}; -use crate::{common::*, config::Config}; +use crate::config::Config; +use super::common::*; pub fn update_access_list(config: &Config, state: &State) { match config.access_list.mode { From 34bc4046b76a9e2c513ec875a5e20c1d4201ba50 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20Frosteg=C3=A5rd?= Date: Tue, 26 Oct 2021 16:26:37 +0200 Subject: [PATCH 02/59] WIP: aquatic_http glommio impl --- Cargo.lock | 70 ++++++++++++++ aquatic_http/Cargo.toml | 6 +- aquatic_http/src/lib/glommio/mod.rs | 2 + aquatic_http/src/lib/glommio/network.rs | 119 ++++++++++++++++++++++++ 4 files changed, 195 insertions(+), 2 deletions(-) create mode 100644 aquatic_http/src/lib/glommio/network.rs diff --git a/Cargo.lock b/Cargo.lock index 607e122..b604d6e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -110,6 +110,8 @@ dependencies = [ "quickcheck", "quickcheck_macros", "rand", + "rustls", + "rustls-pemfile", "serde", "smartstring", "socket2 0.4.2", @@ -1545,6 +1547,21 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "ring" +version = "0.16.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" +dependencies = [ + "cc", + "libc", + "once_cell", + "spin", + "untrusted", + "web-sys", + "winapi 0.3.9", +] + [[package]] name = "rlimit" version = "0.6.2" @@ -1569,6 +1586,27 @@ dependencies = [ "semver", ] +[[package]] +name = "rustls" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b5ac6078ca424dc1d3ae2328526a76787fecc7f8011f520e3276730e711fc95" +dependencies = [ + "log", + "ring", + "sct", + "webpki", +] + +[[package]] +name = "rustls-pemfile" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5eebeaeb360c87bfb72e84abdb3447159c0eaececf1bef2aecd65a8be949d1c9" +dependencies = [ + "base64", +] + [[package]] name = "ryu" version = "1.0.5" @@ -1606,6 +1644,16 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" +[[package]] +name = "sct" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d53dcdb7c9f8158937a7981b48accfd39a43af418591a5d008c7b22b5e1b7ca4" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "security-framework" version = "2.4.2" @@ -1780,6 +1828,12 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "spin" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" + [[package]] name = "static_assertions" version = "1.1.0" @@ -1999,6 +2053,12 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3" +[[package]] +name = "untrusted" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" + [[package]] name = "url" version = "2.2.2" @@ -2134,6 +2194,16 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "webpki" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f095d78192e208183081cc07bc5515ef55216397af48b873e5edcd72637fa1bd" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "winapi" version = "0.2.8" diff --git a/aquatic_http/Cargo.toml b/aquatic_http/Cargo.toml index 092bd5d..73ff3c6 100644 --- a/aquatic_http/Cargo.toml +++ b/aquatic_http/Cargo.toml @@ -17,7 +17,7 @@ path = "src/bin/main.rs" [features] default = ["with-mio"] -with-glommio = ["glommio", "futures-lite"] +with-glommio = ["glommio", "futures-lite", "rustls", "rustls-pemfile"] with-mio = ["crossbeam-channel", "histogram", "mio", "native-tls", "socket2"] [dependencies] @@ -47,8 +47,10 @@ native-tls = { version = "0.2", optional = true } socket2 = { version = "0.4.1", features = ["all"], optional = true } # glommio -glommio = { git = "https://github.com/DataDog/glommio.git", rev = "4e6b14772da2f4325271fbcf12d24cf91ed466e5", optional = true } futures-lite = { version = "1", optional = true } +glommio = { git = "https://github.com/DataDog/glommio.git", rev = "4e6b14772da2f4325271fbcf12d24cf91ed466e5", optional = true } +rustls = { version = "0.20", optional = true } +rustls-pemfile = { version = "0.2", optional = true } [dev-dependencies] quickcheck = "1.0" diff --git a/aquatic_http/src/lib/glommio/mod.rs b/aquatic_http/src/lib/glommio/mod.rs index 0f2955f..1bf5358 100644 --- a/aquatic_http/src/lib/glommio/mod.rs +++ b/aquatic_http/src/lib/glommio/mod.rs @@ -2,6 +2,8 @@ use glommio::prelude::*; use crate::config::Config; +mod network; + pub fn run( config: Config, ) -> anyhow::Result<()> { diff --git a/aquatic_http/src/lib/glommio/network.rs b/aquatic_http/src/lib/glommio/network.rs new file mode 100644 index 0000000..b1893eb --- /dev/null +++ b/aquatic_http/src/lib/glommio/network.rs @@ -0,0 +1,119 @@ +use std::io::{BufReader, Cursor, Read}; +use std::rc::Rc; +use std::sync::Arc; + +use aquatic_http_protocol::request::Request; +use futures_lite::{AsyncReadExt, StreamExt}; +use glommio::prelude::*; +use glommio::net::{TcpListener, TcpStream}; +use rustls::{IoState, ServerConnection}; + +use crate::config::Config; + +pub async fn run_socket_worker( + config: Config, +) { + let tlsConfig = Arc::new(create_tls_config(&config)); + let config = Rc::new(config); + + let listener = TcpListener::bind(config.network.address).expect("bind socket"); + + let mut incoming = listener.incoming(); + + while let Some(stream) = incoming.next().await { + match stream { + Ok(stream) => { + spawn_local(handle_stream(config.clone(), tlsConfig.clone(), stream)).detach(); + }, + Err(err) => { + ::log::error!("accept connection: {:?}", err); + } + } + + } +} + +async fn handle_stream( + config: Rc, + tlsConfig: Arc, + mut stream: TcpStream, +){ + let mut buf = [0u8; 1024]; + let mut conn = ServerConnection::new(tlsConfig).unwrap(); + + loop { + match stream.read(&mut buf).await { + Ok(ciphertext_bytes_read) => { + let mut cursor = Cursor::new(&buf[..ciphertext_bytes_read]); + + match conn.read_tls(&mut cursor) { + Ok(plaintext_bytes_read) => { + match conn.process_new_packets() { + Ok(_) => { + if ciphertext_bytes_read == 0 && plaintext_bytes_read == 0 { + let mut request_bytes = Vec::new(); + + conn.reader().read_to_end(&mut request_bytes); + + match Request::from_bytes(&request_bytes[..]) { + Ok(request) => { + + }, + Err(err) => { + // TODO: return error response, close connection + } + } + } + // TODO: check for io_state.peer_has_closed + }, + Err(err) => { + // TODO: call write_tls + ::log::info!("conn.process_new_packets: {:?}", err); + + break + } + } + }, + Err(err) => { + ::log::info!("conn.read_tls: {:?}", err); + } + } + }, + Err(err) => { + ::log::info!("stream.read: {:?}", err); + } + } + } +} + +fn create_tls_config( + config: &Config, +) -> rustls::ServerConfig { + let mut certs = Vec::new(); + let mut private_key = None; + + use std::iter; + use rustls_pemfile::{Item, read_one}; + + let pemfile = Vec::new(); + let mut reader = BufReader::new(&pemfile[..]); + + for item in iter::from_fn(|| read_one(&mut reader).transpose()) { + match item.unwrap() { + Item::X509Certificate(cert) => { + certs.push(rustls::Certificate(cert)); + }, + Item::RSAKey(key) | Item::PKCS8Key(key) => { + if private_key.is_none(){ + private_key = Some(rustls::PrivateKey(key)); + } + } + } + } + + rustls::ServerConfig::builder() + .with_safe_defaults() + .with_no_client_auth() + .with_single_cert(certs, private_key.expect("no private key")) + .expect("bad certificate/key") +} \ No newline at end of file From ef10c4f3665dbd80751f31556ac3fb7c171d2d86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20Frosteg=C3=A5rd?= Date: Tue, 26 Oct 2021 17:29:21 +0200 Subject: [PATCH 03/59] WIP: aquatic http glommio --- Cargo.lock | 1 + aquatic_http/Cargo.toml | 3 +- aquatic_http/src/lib/glommio/network.rs | 158 ++++++++++++++++-------- 3 files changed, 112 insertions(+), 50 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b604d6e..d4959b1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -113,6 +113,7 @@ dependencies = [ "rustls", "rustls-pemfile", "serde", + "slab", "smartstring", "socket2 0.4.2", ] diff --git a/aquatic_http/Cargo.toml b/aquatic_http/Cargo.toml index 73ff3c6..97f7bab 100644 --- a/aquatic_http/Cargo.toml +++ b/aquatic_http/Cargo.toml @@ -17,7 +17,7 @@ path = "src/bin/main.rs" [features] default = ["with-mio"] -with-glommio = ["glommio", "futures-lite", "rustls", "rustls-pemfile"] +with-glommio = ["glommio", "futures-lite", "rustls", "rustls-pemfile", "slab"] with-mio = ["crossbeam-channel", "histogram", "mio", "native-tls", "socket2"] [dependencies] @@ -51,6 +51,7 @@ futures-lite = { version = "1", optional = true } glommio = { git = "https://github.com/DataDog/glommio.git", rev = "4e6b14772da2f4325271fbcf12d24cf91ed466e5", optional = true } rustls = { version = "0.20", optional = true } rustls-pemfile = { version = "0.2", optional = true } +slab = { version = "0.4", optional = true } [dev-dependencies] quickcheck = "1.0" diff --git a/aquatic_http/src/lib/glommio/network.rs b/aquatic_http/src/lib/glommio/network.rs index b1893eb..ba04ef8 100644 --- a/aquatic_http/src/lib/glommio/network.rs +++ b/aquatic_http/src/lib/glommio/network.rs @@ -3,27 +3,67 @@ use std::rc::Rc; use std::sync::Arc; use aquatic_http_protocol::request::Request; -use futures_lite::{AsyncReadExt, StreamExt}; +use aquatic_http_protocol::response::Response; +use futures_lite::{AsyncReadExt, AsyncWriteExt, StreamExt}; use glommio::prelude::*; use glommio::net::{TcpListener, TcpStream}; +use glommio::channels::local_channel::{new_bounded, LocalReceiver, LocalSender}; +use glommio::task::JoinHandle; use rustls::{IoState, ServerConnection}; +use slab::Slab; use crate::config::Config; +struct ConnectionReference { + response_sender: LocalSender, + handle: JoinHandle<()>, +} + +struct Connection { + response_receiver: LocalReceiver, + tls: ServerConnection, + stream: TcpStream, + index: usize, +} + pub async fn run_socket_worker( config: Config, ) { - let tlsConfig = Arc::new(create_tls_config(&config)); + let tls_config = Arc::new(create_tls_config(&config)); let config = Rc::new(config); let listener = TcpListener::bind(config.network.address).expect("bind socket"); + let mut connection_slab: Slab = Slab::new(); + let mut incoming = listener.incoming(); while let Some(stream) = incoming.next().await { match stream { Ok(stream) => { - spawn_local(handle_stream(config.clone(), tlsConfig.clone(), stream)).detach(); + let (response_sender, response_receiver) = new_bounded(1); + + let entry = connection_slab.vacant_entry(); + + let conn = Connection { + response_receiver, + tls: ServerConnection::new(tls_config.clone()).unwrap(), + stream, + index: entry.key(), + }; + + async fn handle_stream(mut conn: Connection) { + conn.handle_stream().await; + } + + let handle = spawn_local(handle_stream(conn)).detach(); + + let connection_reference = ConnectionReference { + response_sender, + handle, + }; + + entry.insert(connection_reference); }, Err(err) => { ::log::error!("accept connection: {:?}", err); @@ -33,59 +73,79 @@ pub async fn run_socket_worker( } } -async fn handle_stream( - config: Rc, - tlsConfig: Arc, - mut stream: TcpStream, -){ - let mut buf = [0u8; 1024]; - let mut conn = ServerConnection::new(tlsConfig).unwrap(); +impl Connection { + async fn handle_stream(&mut self){ + loop { + while let Some(response) = self.response_receiver.stream().next().await { + response.write(&mut self.tls.writer()).unwrap(); - loop { - match stream.read(&mut buf).await { - Ok(ciphertext_bytes_read) => { - let mut cursor = Cursor::new(&buf[..ciphertext_bytes_read]); + let mut buf = Vec::new(); + let mut buf = Cursor::new(&mut buf); - match conn.read_tls(&mut cursor) { - Ok(plaintext_bytes_read) => { - match conn.process_new_packets() { - Ok(_) => { - if ciphertext_bytes_read == 0 && plaintext_bytes_read == 0 { - let mut request_bytes = Vec::new(); - - conn.reader().read_to_end(&mut request_bytes); - - match Request::from_bytes(&request_bytes[..]) { - Ok(request) => { - - }, - Err(err) => { - // TODO: return error response, close connection - } - } - } - // TODO: check for io_state.peer_has_closed - }, - Err(err) => { - // TODO: call write_tls - ::log::info!("conn.process_new_packets: {:?}", err); - - break - } - } - }, - Err(err) => { - ::log::info!("conn.read_tls: {:?}", err); - } + while self.tls.wants_write() { + self.tls.write_tls(&mut buf).unwrap(); } - }, - Err(err) => { - ::log::info!("stream.read: {:?}", err); + + self.stream.write_all(&buf.into_inner()).await.unwrap(); } } } -} + async fn handle_stream_handshake(&mut self) { + let mut buf = [0u8; 1024]; + + loop { + match self.stream.read(&mut buf).await { + Ok(ciphertext_bytes_read) => { + let mut cursor = Cursor::new(&buf[..ciphertext_bytes_read]); + + match self.tls.read_tls(&mut cursor) { + Ok(plaintext_bytes_read) => { + match self.tls.process_new_packets() { + Ok(_) => { + if ciphertext_bytes_read == 0 && plaintext_bytes_read == 0 { + let mut request_bytes = Vec::new(); + + self.tls.reader().read_to_end(&mut request_bytes); + + match Request::from_bytes(&request_bytes[..]) { + Ok(request) => { + ::log::info!("request read: {:?}", request); + }, + Err(err) => { + // TODO: send error response, close connection + + ::log::info!("Request::from_bytes: {:?}", err); + + break + } + } + } + // TODO: check for io_state.peer_has_closed + }, + Err(err) => { + // TODO: call write_tls + ::log::info!("conn.process_new_packets: {:?}", err); + + break + } + } + }, + Err(err) => { + ::log::info!("conn.read_tls: {:?}", err); + } + } + }, + Err(err) => { + ::log::info!("stream.read: {:?}", err); + } + } + } + } + + + +} fn create_tls_config( config: &Config, ) -> rustls::ServerConfig { From dcb03f42e7b5114d7543857d0966e453778b0d2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20Frosteg=C3=A5rd?= Date: Tue, 26 Oct 2021 18:18:35 +0200 Subject: [PATCH 04/59] WIP: aquatic http glommio --- aquatic_http/src/lib/glommio/network.rs | 124 ++++++++++++------------ 1 file changed, 63 insertions(+), 61 deletions(-) diff --git a/aquatic_http/src/lib/glommio/network.rs b/aquatic_http/src/lib/glommio/network.rs index ba04ef8..8bc1ff6 100644 --- a/aquatic_http/src/lib/glommio/network.rs +++ b/aquatic_http/src/lib/glommio/network.rs @@ -2,9 +2,11 @@ use std::io::{BufReader, Cursor, Read}; use std::rc::Rc; use std::sync::Arc; -use aquatic_http_protocol::request::Request; +use aquatic_http_protocol::request::{Request, RequestParseError}; use aquatic_http_protocol::response::Response; use futures_lite::{AsyncReadExt, AsyncWriteExt, StreamExt}; +use glommio::channels::channel_mesh::{MeshBuilder, Partial, Role, Senders}; +use glommio::channels::shared_channel::{ConnectedSender, SharedSender}; use glommio::prelude::*; use glommio::net::{TcpListener, TcpStream}; use glommio::channels::local_channel::{new_bounded, LocalReceiver, LocalSender}; @@ -20,19 +22,24 @@ struct ConnectionReference { } struct Connection { + request_senders: Rc>, response_receiver: LocalReceiver, tls: ServerConnection, stream: TcpStream, index: usize, + expects_request: bool, } pub async fn run_socket_worker( config: Config, + request_mesh_builder: MeshBuilder, ) { let tls_config = Arc::new(create_tls_config(&config)); let config = Rc::new(config); let listener = TcpListener::bind(config.network.address).expect("bind socket"); + let (request_senders, _) = request_mesh_builder.join(Role::Producer).await.unwrap(); + let request_senders = Rc::new(request_senders); let mut connection_slab: Slab = Slab::new(); @@ -46,14 +53,18 @@ pub async fn run_socket_worker( let entry = connection_slab.vacant_entry(); let conn = Connection { + request_senders: request_senders.clone(), response_receiver, tls: ServerConnection::new(tls_config.clone()).unwrap(), stream, index: entry.key(), + expects_request: true, }; async fn handle_stream(mut conn: Connection) { - conn.handle_stream().await; + if let Err(err) = conn.handle_stream().await { + ::log::error!("conn.handle_stream() error: {:?}", err); + } } let handle = spawn_local(handle_stream(conn)).detach(); @@ -74,78 +85,69 @@ pub async fn run_socket_worker( } impl Connection { - async fn handle_stream(&mut self){ + async fn handle_stream(&mut self) -> anyhow::Result<()> { loop { - while let Some(response) = self.response_receiver.stream().next().await { - response.write(&mut self.tls.writer()).unwrap(); + self.write_tls().await?; + self.read_tls().await; - let mut buf = Vec::new(); - let mut buf = Cursor::new(&mut buf); + if !self.tls.is_handshaking() { + if self.expects_request { + let request = self.extract_request()?; - while self.tls.wants_write() { - self.tls.write_tls(&mut buf).unwrap(); - } + self.request_senders.try_send_to(0, request); + self.expects_request = false; - self.stream.write_all(&buf.into_inner()).await.unwrap(); - } - } - } + } else if let Some(response) = self.response_receiver.recv().await { + response.write(&mut self.tls.writer())?; - async fn handle_stream_handshake(&mut self) { - let mut buf = [0u8; 1024]; - - loop { - match self.stream.read(&mut buf).await { - Ok(ciphertext_bytes_read) => { - let mut cursor = Cursor::new(&buf[..ciphertext_bytes_read]); - - match self.tls.read_tls(&mut cursor) { - Ok(plaintext_bytes_read) => { - match self.tls.process_new_packets() { - Ok(_) => { - if ciphertext_bytes_read == 0 && plaintext_bytes_read == 0 { - let mut request_bytes = Vec::new(); - - self.tls.reader().read_to_end(&mut request_bytes); - - match Request::from_bytes(&request_bytes[..]) { - Ok(request) => { - ::log::info!("request read: {:?}", request); - }, - Err(err) => { - // TODO: send error response, close connection - - ::log::info!("Request::from_bytes: {:?}", err); - - break - } - } - } - // TODO: check for io_state.peer_has_closed - }, - Err(err) => { - // TODO: call write_tls - ::log::info!("conn.process_new_packets: {:?}", err); - - break - } - } - }, - Err(err) => { - ::log::info!("conn.read_tls: {:?}", err); - } - } - }, - Err(err) => { - ::log::info!("stream.read: {:?}", err); + self.expects_request = true; } } } } + async fn read_tls(&mut self) -> anyhow::Result<()> { + while self.tls.wants_read() { + let mut buf = Vec::new(); + let _ciphertext_bytes_read = self.stream.read_to_end(&mut buf).await?; + let mut cursor = Cursor::new(&buf[..]); + + let _plaintext_bytes_read = self.tls.read_tls(&mut cursor)?; + + let _io_state = self.tls.process_new_packets()?; + } + + Ok(()) + } + + async fn write_tls(&mut self) -> anyhow::Result<()> { + if !self.tls.wants_write() { + return Ok(()); + } + + let mut buf = Vec::new(); + let mut buf = Cursor::new(&mut buf); + + while self.tls.wants_write() { + self.tls.write_tls(&mut buf)?; + } + + self.stream.write_all(&buf.into_inner()).await?; + + Ok(()) + } + + fn extract_request(&mut self) -> anyhow::Result { + let mut request_bytes = Vec::new(); + + self.tls.reader().read_to_end(&mut request_bytes)?; + + Request::from_bytes(&request_bytes[..]).map_err(|err| anyhow::anyhow!("{:?}", err)) + } } + fn create_tls_config( config: &Config, ) -> rustls::ServerConfig { From 03b8f3e5c539bdb45d1b0f34fc0cf90e407e9e8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20Frosteg=C3=A5rd?= Date: Tue, 26 Oct 2021 18:45:09 +0200 Subject: [PATCH 05/59] WIP: aquatic_udp glommio --- aquatic_http/src/lib/glommio/network.rs | 42 +++++++++++++++++++------ 1 file changed, 33 insertions(+), 9 deletions(-) diff --git a/aquatic_http/src/lib/glommio/network.rs b/aquatic_http/src/lib/glommio/network.rs index 8bc1ff6..7f1fcdf 100644 --- a/aquatic_http/src/lib/glommio/network.rs +++ b/aquatic_http/src/lib/glommio/network.rs @@ -1,3 +1,4 @@ +use std::cell::RefCell; use std::io::{BufReader, Cursor, Read}; use std::rc::Rc; use std::sync::Arc; @@ -5,8 +6,8 @@ use std::sync::Arc; use aquatic_http_protocol::request::{Request, RequestParseError}; use aquatic_http_protocol::response::Response; use futures_lite::{AsyncReadExt, AsyncWriteExt, StreamExt}; -use glommio::channels::channel_mesh::{MeshBuilder, Partial, Role, Senders}; -use glommio::channels::shared_channel::{ConnectedSender, SharedSender}; +use glommio::channels::channel_mesh::{MeshBuilder, Partial, Receivers, Role, Senders}; +use glommio::channels::shared_channel::{ConnectedReceiver, ConnectedSender, SharedSender}; use glommio::prelude::*; use glommio::net::{TcpListener, TcpStream}; use glommio::channels::local_channel::{new_bounded, LocalReceiver, LocalSender}; @@ -16,32 +17,43 @@ use slab::Slab; use crate::config::Config; +#[derive(Clone, Copy, Debug)] +pub struct ConnectionId(pub usize); + struct ConnectionReference { response_sender: LocalSender, handle: JoinHandle<()>, } struct Connection { - request_senders: Rc>, + request_senders: Rc>, response_receiver: LocalReceiver, tls: ServerConnection, stream: TcpStream, - index: usize, + index: ConnectionId, expects_request: bool, } pub async fn run_socket_worker( config: Config, - request_mesh_builder: MeshBuilder, + request_mesh_builder: MeshBuilder<(ConnectionId, Request), Partial>, + response_mesh_builder: MeshBuilder<(ConnectionId, Response), Partial>, ) { let tls_config = Arc::new(create_tls_config(&config)); let config = Rc::new(config); let listener = TcpListener::bind(config.network.address).expect("bind socket"); + + let (_, mut response_receivers) = response_mesh_builder.join(Role::Consumer).await.unwrap(); + let (request_senders, _) = request_mesh_builder.join(Role::Producer).await.unwrap(); let request_senders = Rc::new(request_senders); - let mut connection_slab: Slab = Slab::new(); + let connection_slab = Rc::new(RefCell::new(Slab::new())); + + for (_, response_receiver) in response_receivers.streams() { + spawn_local(receive_responses(response_receiver, connection_slab.clone())).detach(); + } let mut incoming = listener.incoming(); @@ -50,14 +62,15 @@ pub async fn run_socket_worker( Ok(stream) => { let (response_sender, response_receiver) = new_bounded(1); - let entry = connection_slab.vacant_entry(); + let mut slab = connection_slab.borrow_mut(); + let entry = slab.vacant_entry(); let conn = Connection { request_senders: request_senders.clone(), response_receiver, tls: ServerConnection::new(tls_config.clone()).unwrap(), stream, - index: entry.key(), + index: ConnectionId(entry.key()), expects_request: true, }; @@ -84,6 +97,17 @@ pub async fn run_socket_worker( } } +async fn receive_responses( + mut response_receiver: ConnectedReceiver<(ConnectionId, Response)>, + connection_references: Rc>>, +) { + while let Some((connection_id, response)) = response_receiver.next().await { + if let Some(reference) = connection_references.borrow().get(connection_id.0) { + reference.response_sender.try_send(response); + } + } +} + impl Connection { async fn handle_stream(&mut self) -> anyhow::Result<()> { loop { @@ -94,7 +118,7 @@ impl Connection { if self.expects_request { let request = self.extract_request()?; - self.request_senders.try_send_to(0, request); + self.request_senders.try_send_to(0, (self.index, request)); self.expects_request = false; } else if let Some(response) = self.response_receiver.recv().await { From 02735ba2ff7d7117bc9f20905dcd6b989d84ade6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20Frosteg=C3=A5rd?= Date: Tue, 26 Oct 2021 19:09:17 +0200 Subject: [PATCH 06/59] aquatic_http: glommio: actually start socket workers --- aquatic_http/src/lib/glommio/mod.rs | 55 ++++++++++++++++++++++++- aquatic_http/src/lib/glommio/network.rs | 3 ++ 2 files changed, 57 insertions(+), 1 deletion(-) diff --git a/aquatic_http/src/lib/glommio/mod.rs b/aquatic_http/src/lib/glommio/mod.rs index 1bf5358..3bb89fb 100644 --- a/aquatic_http/src/lib/glommio/mod.rs +++ b/aquatic_http/src/lib/glommio/mod.rs @@ -1,12 +1,65 @@ -use glommio::prelude::*; +use std::sync::{Arc, atomic::AtomicUsize}; + +use aquatic_common::access_list::AccessList; +use glommio::{channels::channel_mesh::MeshBuilder, prelude::*}; use crate::config::Config; mod network; +const SHARED_CHANNEL_SIZE: usize = 1024; + pub fn run( config: Config, ) -> anyhow::Result<()> { + let access_list = if config.access_list.mode.is_on() { + AccessList::create_from_path(&config.access_list.path).expect("Load access list") + } else { + AccessList::default() + }; + + let num_peers = config.socket_workers + config.request_workers; + + let request_mesh_builder = MeshBuilder::partial(num_peers, SHARED_CHANNEL_SIZE); + let response_mesh_builder = MeshBuilder::partial(num_peers, SHARED_CHANNEL_SIZE); + + let num_bound_sockets = Arc::new(AtomicUsize::new(0)); + + let mut executors = Vec::new(); + + for i in 0..(config.socket_workers) { + let config = config.clone(); + let request_mesh_builder = request_mesh_builder.clone(); + let response_mesh_builder = response_mesh_builder.clone(); + let num_bound_sockets = num_bound_sockets.clone(); + let access_list = access_list.clone(); + + let mut builder = LocalExecutorBuilder::default(); + + // if config.core_affinity.set_affinities { + // builder = builder.pin_to_cpu(config.core_affinity.offset + 1 + i); + // } + + let executor = builder.spawn(|| async move { + network::run_socket_worker( + config, + request_mesh_builder, + response_mesh_builder, + num_bound_sockets, + // access_list, + ) + .await + }); + + executors.push(executor); + } + + for executor in executors { + executor + .expect("failed to spawn local executor") + .join() + .unwrap(); + } Ok(()) } \ No newline at end of file diff --git a/aquatic_http/src/lib/glommio/network.rs b/aquatic_http/src/lib/glommio/network.rs index 7f1fcdf..87cb2ec 100644 --- a/aquatic_http/src/lib/glommio/network.rs +++ b/aquatic_http/src/lib/glommio/network.rs @@ -2,6 +2,7 @@ use std::cell::RefCell; use std::io::{BufReader, Cursor, Read}; use std::rc::Rc; use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; use aquatic_http_protocol::request::{Request, RequestParseError}; use aquatic_http_protocol::response::Response; @@ -38,11 +39,13 @@ pub async fn run_socket_worker( config: Config, request_mesh_builder: MeshBuilder<(ConnectionId, Request), Partial>, response_mesh_builder: MeshBuilder<(ConnectionId, Response), Partial>, + num_bound_sockets: Arc, ) { let tls_config = Arc::new(create_tls_config(&config)); let config = Rc::new(config); let listener = TcpListener::bind(config.network.address).expect("bind socket"); + num_bound_sockets.fetch_add(1, Ordering::SeqCst); let (_, mut response_receivers) = response_mesh_builder.join(Role::Consumer).await.unwrap(); From 7fd2d4c42ee3262759697b379fc91757856803ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20Frosteg=C3=A5rd?= Date: Tue, 26 Oct 2021 19:38:37 +0200 Subject: [PATCH 07/59] aquatic_http glommio: modify tls config and file parsing --- aquatic_http/src/lib/config.rs | 6 +++- aquatic_http/src/lib/glommio/mod.rs | 37 ++++++++++++++++++++++++- aquatic_http/src/lib/glommio/network.rs | 34 +---------------------- 3 files changed, 42 insertions(+), 35 deletions(-) diff --git a/aquatic_http/src/lib/config.rs b/aquatic_http/src/lib/config.rs index fe75932..08ccfad 100644 --- a/aquatic_http/src/lib/config.rs +++ b/aquatic_http/src/lib/config.rs @@ -1,4 +1,4 @@ -use std::net::SocketAddr; +use std::{net::SocketAddr, path::PathBuf}; use aquatic_common::access_list::AccessListConfig; use serde::{Deserialize, Serialize}; @@ -37,6 +37,8 @@ pub struct TlsConfig { pub use_tls: bool, pub tls_pkcs12_path: String, pub tls_pkcs12_password: String, + pub tls_certificate_path: PathBuf, + pub tls_private_key_path: PathBuf, } #[derive(Clone, Debug, Serialize, Deserialize)] @@ -182,6 +184,8 @@ impl Default for TlsConfig { use_tls: false, tls_pkcs12_path: "".into(), tls_pkcs12_password: "".into(), + tls_certificate_path: "".into(), + tls_private_key_path: "".into(), } } } diff --git a/aquatic_http/src/lib/glommio/mod.rs b/aquatic_http/src/lib/glommio/mod.rs index 3bb89fb..e5b577e 100644 --- a/aquatic_http/src/lib/glommio/mod.rs +++ b/aquatic_http/src/lib/glommio/mod.rs @@ -1,4 +1,4 @@ -use std::sync::{Arc, atomic::AtomicUsize}; +use std::{fs::File, io::BufReader, sync::{Arc, atomic::AtomicUsize}}; use aquatic_common::access_list::AccessList; use glommio::{channels::channel_mesh::MeshBuilder, prelude::*}; @@ -25,10 +25,13 @@ pub fn run( let num_bound_sockets = Arc::new(AtomicUsize::new(0)); + let tls_config = Arc::new(create_tls_config(&config).unwrap()); + let mut executors = Vec::new(); for i in 0..(config.socket_workers) { let config = config.clone(); + let tls_config = tls_config.clone(); let request_mesh_builder = request_mesh_builder.clone(); let response_mesh_builder = response_mesh_builder.clone(); let num_bound_sockets = num_bound_sockets.clone(); @@ -43,6 +46,7 @@ pub fn run( let executor = builder.spawn(|| async move { network::run_socket_worker( config, + tls_config, request_mesh_builder, response_mesh_builder, num_bound_sockets, @@ -62,4 +66,35 @@ pub fn run( } Ok(()) +} + +fn create_tls_config( + config: &Config, +) -> anyhow::Result { + let certs = { + let f = File::open(&config.network.tls.tls_certificate_path)?; + let mut f = BufReader::new(f); + + rustls_pemfile::certs(&mut f)? + .into_iter() + .map(|bytes| rustls::Certificate(bytes)) + .collect() + }; + + let private_key = { + let f = File::open(&config.network.tls.tls_private_key_path)?; + let mut f = BufReader::new(f); + + rustls_pemfile::pkcs8_private_keys(&mut f)? + .first() + .map(|bytes| rustls::PrivateKey(bytes.clone())) + .ok_or(anyhow::anyhow!("No private keys in file"))? + }; + + let tls_config = rustls::ServerConfig::builder() + .with_safe_defaults() + .with_no_client_auth() + .with_single_cert(certs, private_key)?; + + Ok(tls_config) } \ No newline at end of file diff --git a/aquatic_http/src/lib/glommio/network.rs b/aquatic_http/src/lib/glommio/network.rs index 87cb2ec..904b32e 100644 --- a/aquatic_http/src/lib/glommio/network.rs +++ b/aquatic_http/src/lib/glommio/network.rs @@ -37,11 +37,11 @@ struct Connection { pub async fn run_socket_worker( config: Config, + tls_config: Arc, request_mesh_builder: MeshBuilder<(ConnectionId, Request), Partial>, response_mesh_builder: MeshBuilder<(ConnectionId, Response), Partial>, num_bound_sockets: Arc, ) { - let tls_config = Arc::new(create_tls_config(&config)); let config = Rc::new(config); let listener = TcpListener::bind(config.network.address).expect("bind socket"); @@ -174,35 +174,3 @@ impl Connection { Request::from_bytes(&request_bytes[..]).map_err(|err| anyhow::anyhow!("{:?}", err)) } } - -fn create_tls_config( - config: &Config, -) -> rustls::ServerConfig { - let mut certs = Vec::new(); - let mut private_key = None; - - use std::iter; - use rustls_pemfile::{Item, read_one}; - - let pemfile = Vec::new(); - let mut reader = BufReader::new(&pemfile[..]); - - for item in iter::from_fn(|| read_one(&mut reader).transpose()) { - match item.unwrap() { - Item::X509Certificate(cert) => { - certs.push(rustls::Certificate(cert)); - }, - Item::RSAKey(key) | Item::PKCS8Key(key) => { - if private_key.is_none(){ - private_key = Some(rustls::PrivateKey(key)); - } - } - } - } - - rustls::ServerConfig::builder() - .with_safe_defaults() - .with_no_client_auth() - .with_single_cert(certs, private_key.expect("no private key")) - .expect("bad certificate/key") -} \ No newline at end of file From 79bbf957c0bed501aabf17782c405dfa20ca1d34 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20Frosteg=C3=A5rd?= Date: Tue, 26 Oct 2021 19:39:05 +0200 Subject: [PATCH 08/59] add scripts/gen-tls.sh for generating snakeoil tls cert & key --- scripts/gen-tls.sh | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100755 scripts/gen-tls.sh diff --git a/scripts/gen-tls.sh b/scripts/gen-tls.sh new file mode 100755 index 0000000..5fd4512 --- /dev/null +++ b/scripts/gen-tls.sh @@ -0,0 +1,16 @@ +#/bin/bash + +set -e + +mkdir -p tmp/tls + +cd tmp/tls + +openssl ecparam -genkey -name prime256v1 -out key.pem +openssl req -new -sha256 -key key.pem -out csr.csr -subj "/C=GB/ST=Test/L=Test/O=Test/OU=Test/CN=example.com" +openssl req -x509 -sha256 -nodes -days 365 -key key.pem -in csr.csr -out cert.crt + +sudo cp cert.crt /usr/local/share/ca-certificates/snakeoil.crt +sudo update-ca-certificates + +openssl pkcs8 -in key.pem -topk8 -nocrypt -out key.pk8 From 96593c97fcbe87f64151bf4883234e982b75882d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20Frosteg=C3=A5rd?= Date: Tue, 26 Oct 2021 21:12:34 +0200 Subject: [PATCH 09/59] WIP: aquatic_http glommio work --- aquatic_http/src/lib/glommio/network.rs | 112 +++++++++++++++++------- 1 file changed, 81 insertions(+), 31 deletions(-) diff --git a/aquatic_http/src/lib/glommio/network.rs b/aquatic_http/src/lib/glommio/network.rs index 904b32e..2737234 100644 --- a/aquatic_http/src/lib/glommio/network.rs +++ b/aquatic_http/src/lib/glommio/network.rs @@ -1,5 +1,5 @@ use std::cell::RefCell; -use std::io::{BufReader, Cursor, Read}; +use std::io::{BufReader, Cursor, ErrorKind, Read}; use std::rc::Rc; use std::sync::Arc; use std::sync::atomic::{AtomicUsize, Ordering}; @@ -27,12 +27,13 @@ struct ConnectionReference { } struct Connection { - request_senders: Rc>, - response_receiver: LocalReceiver, + // request_senders: Rc>, + // response_receiver: LocalReceiver, tls: ServerConnection, stream: TcpStream, index: ConnectionId, expects_request: bool, + request_buffer: Vec, } pub async fn run_socket_worker( @@ -47,16 +48,16 @@ pub async fn run_socket_worker( let listener = TcpListener::bind(config.network.address).expect("bind socket"); num_bound_sockets.fetch_add(1, Ordering::SeqCst); - let (_, mut response_receivers) = response_mesh_builder.join(Role::Consumer).await.unwrap(); + // let (_, mut response_receivers) = response_mesh_builder.join(Role::Consumer).await.unwrap(); - let (request_senders, _) = request_mesh_builder.join(Role::Producer).await.unwrap(); - let request_senders = Rc::new(request_senders); + // let (request_senders, _) = request_mesh_builder.join(Role::Producer).await.unwrap(); + // let request_senders = Rc::new(request_senders); let connection_slab = Rc::new(RefCell::new(Slab::new())); - for (_, response_receiver) in response_receivers.streams() { - spawn_local(receive_responses(response_receiver, connection_slab.clone())).detach(); - } + // for (_, response_receiver) in response_receivers.streams() { + // spawn_local(receive_responses(response_receiver, connection_slab.clone())).detach(); + // } let mut incoming = listener.incoming(); @@ -69,12 +70,13 @@ pub async fn run_socket_worker( let entry = slab.vacant_entry(); let conn = Connection { - request_senders: request_senders.clone(), - response_receiver, + // request_senders: request_senders.clone(), + // response_receiver, tls: ServerConnection::new(tls_config.clone()).unwrap(), stream, index: ConnectionId(entry.key()), expects_request: true, + request_buffer: Vec::new(), }; async fn handle_stream(mut conn: Connection) { @@ -113,37 +115,91 @@ async fn receive_responses( impl Connection { async fn handle_stream(&mut self) -> anyhow::Result<()> { + ::log::info!("incoming stream"); loop { self.write_tls().await?; - self.read_tls().await; + self.read_tls().await?; + /* if !self.tls.is_handshaking() { if self.expects_request { let request = self.extract_request()?; - self.request_senders.try_send_to(0, (self.index, request)); + ::log::info!("request received: {:?}", request); + + // self.request_senders.try_send_to(0, (self.index, request)); self.expects_request = false; - } else if let Some(response) = self.response_receiver.recv().await { + }/* + else if let Some(response) = self.response_receiver.recv().await { response.write(&mut self.tls.writer())?; self.expects_request = true; - } + } */ } + */ } } async fn read_tls(&mut self) -> anyhow::Result<()> { - while self.tls.wants_read() { - let mut buf = Vec::new(); + loop { + ::log::info!("read_tls (wants read)"); - let _ciphertext_bytes_read = self.stream.read_to_end(&mut buf).await?; + let mut buf = [0u8; 1024]; - let mut cursor = Cursor::new(&buf[..]); + let bytes_read = self.stream.read(&mut buf).await?; - let _plaintext_bytes_read = self.tls.read_tls(&mut cursor)?; + if bytes_read == 0 { + // Peer has closed connection. Remove it. + return Err(anyhow::anyhow!("peer has closed connection")); + } - let _io_state = self.tls.process_new_packets()?; + let _ = self.tls.read_tls(&mut &buf[..bytes_read]).unwrap(); + + let io_state = self.tls.process_new_packets()?; + + let mut added_plaintext = false; + + while io_state.plaintext_bytes_to_read() != 0 { + match self.tls.reader().read(&mut buf) { + Ok(0) => { + break; + } + Ok(amt) => { + self.request_buffer.extend_from_slice(&buf[..amt]); + + added_plaintext = true; + }, + Err(err) if err.kind() == ErrorKind::WouldBlock => { + break; + } + Err(err) => { + ::log::info!("tls.reader().read error: {:?}", err); + + break; + } + } + } + + if added_plaintext { + match Request::from_bytes(&self.request_buffer[..]) { + Ok(request) => { + self.expects_request = false; + + ::log::info!("received request: {:?}", request); + } + Err(RequestParseError::NeedMoreData) => { + ::log::info!("need more request data. current data: {:?}", std::str::from_utf8(&self.request_buffer)); + } + Err(RequestParseError::Invalid(err)) => { + return Err(anyhow::anyhow!("request parse error: {:?}", err)); + } + } + } + + if self.tls.wants_write() { + break + } } Ok(()) @@ -154,23 +210,17 @@ impl Connection { return Ok(()); } + ::log::info!("write_tls (wants write)"); + let mut buf = Vec::new(); let mut buf = Cursor::new(&mut buf); while self.tls.wants_write() { - self.tls.write_tls(&mut buf)?; + self.tls.write_tls(&mut buf).unwrap(); } - self.stream.write_all(&buf.into_inner()).await?; + self.stream.write_all(&buf.into_inner()).await.unwrap(); Ok(()) } - - fn extract_request(&mut self) -> anyhow::Result { - let mut request_bytes = Vec::new(); - - self.tls.reader().read_to_end(&mut request_bytes)?; - - Request::from_bytes(&request_bytes[..]).map_err(|err| anyhow::anyhow!("{:?}", err)) - } } From eebfa69c70a5d5ee12b1c7e9ac44b7663bb57921 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20Frosteg=C3=A5rd?= Date: Tue, 26 Oct 2021 21:54:41 +0200 Subject: [PATCH 10/59] aquatic_http: work on glommio network implementation --- aquatic_http/src/lib/common/mod.rs | 31 ++++ aquatic_http/src/lib/glommio/network.rs | 139 ++++++++++++------ aquatic_http/src/lib/lib.rs | 1 + .../src/lib/mio/network/connection.rs | 35 +---- 4 files changed, 124 insertions(+), 82 deletions(-) create mode 100644 aquatic_http/src/lib/common/mod.rs diff --git a/aquatic_http/src/lib/common/mod.rs b/aquatic_http/src/lib/common/mod.rs new file mode 100644 index 0000000..8002a13 --- /dev/null +++ b/aquatic_http/src/lib/common/mod.rs @@ -0,0 +1,31 @@ +pub fn num_digits_in_usize(mut number: usize) -> usize { + let mut num_digits = 1usize; + + while number >= 10 { + num_digits += 1; + + number /= 10; + } + + num_digits +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_num_digits_in_usize() { + let f = num_digits_in_usize; + + assert_eq!(f(0), 1); + assert_eq!(f(1), 1); + assert_eq!(f(9), 1); + assert_eq!(f(10), 2); + assert_eq!(f(11), 2); + assert_eq!(f(99), 2); + assert_eq!(f(100), 3); + assert_eq!(f(101), 3); + assert_eq!(f(1000), 4); + } +} diff --git a/aquatic_http/src/lib/glommio/network.rs b/aquatic_http/src/lib/glommio/network.rs index 2737234..589a907 100644 --- a/aquatic_http/src/lib/glommio/network.rs +++ b/aquatic_http/src/lib/glommio/network.rs @@ -1,11 +1,11 @@ use std::cell::RefCell; -use std::io::{BufReader, Cursor, ErrorKind, Read}; +use std::io::{BufReader, Cursor, ErrorKind, Read, Write}; use std::rc::Rc; use std::sync::Arc; use std::sync::atomic::{AtomicUsize, Ordering}; use aquatic_http_protocol::request::{Request, RequestParseError}; -use aquatic_http_protocol::response::Response; +use aquatic_http_protocol::response::{FailureResponse, Response}; use futures_lite::{AsyncReadExt, AsyncWriteExt, StreamExt}; use glommio::channels::channel_mesh::{MeshBuilder, Partial, Receivers, Role, Senders}; use glommio::channels::shared_channel::{ConnectedReceiver, ConnectedSender, SharedSender}; @@ -16,8 +16,11 @@ use glommio::task::JoinHandle; use rustls::{IoState, ServerConnection}; use slab::Slab; +use crate::common::num_digits_in_usize; use crate::config::Config; +const BUFFER_SIZE: usize = 1024; + #[derive(Clone, Copy, Debug)] pub struct ConnectionId(pub usize); @@ -28,12 +31,13 @@ struct ConnectionReference { struct Connection { // request_senders: Rc>, - // response_receiver: LocalReceiver, + response_receiver: LocalReceiver, tls: ServerConnection, stream: TcpStream, index: ConnectionId, - expects_request: bool, request_buffer: Vec, + wait_for_response: bool, + close_after_writing: bool, } pub async fn run_socket_worker( @@ -71,17 +75,18 @@ pub async fn run_socket_worker( let conn = Connection { // request_senders: request_senders.clone(), - // response_receiver, + response_receiver, tls: ServerConnection::new(tls_config.clone()).unwrap(), stream, index: ConnectionId(entry.key()), - expects_request: true, request_buffer: Vec::new(), + wait_for_response: false, + close_after_writing: false, }; async fn handle_stream(mut conn: Connection) { if let Err(err) = conn.handle_stream().await { - ::log::error!("conn.handle_stream() error: {:?}", err); + ::log::info!("conn.handle_stream() error: {:?}", err); } } @@ -115,43 +120,45 @@ async fn receive_responses( impl Connection { async fn handle_stream(&mut self) -> anyhow::Result<()> { - ::log::info!("incoming stream"); loop { - self.write_tls().await?; self.read_tls().await?; - /* - if !self.tls.is_handshaking() { - if self.expects_request { - let request = self.extract_request()?; + if self.wait_for_response { + if let Some(response) = self.response_receiver.recv().await { + self.queue_response(&response)?; - ::log::info!("request received: {:?}", request); + self.wait_for_response = false; - // self.request_senders.try_send_to(0, (self.index, request)); - self.expects_request = false; - - }/* - else if let Some(response) = self.response_receiver.recv().await { - response.write(&mut self.tls.writer())?; - - self.expects_request = true; - } */ + // TODO: trigger close here if keepalive is false + } + } + + self.write_tls().await?; + + if self.close_after_writing { + let _ = self.stream.shutdown(std::net::Shutdown::Both).await; + + break; } - */ } + + Ok(()) } async fn read_tls(&mut self) -> anyhow::Result<()> { loop { - ::log::info!("read_tls (wants read)"); + ::log::debug!("read_tls"); - let mut buf = [0u8; 1024]; + let mut buf = [0u8; BUFFER_SIZE]; let bytes_read = self.stream.read(&mut buf).await?; if bytes_read == 0 { - // Peer has closed connection. Remove it. - return Err(anyhow::anyhow!("peer has closed connection")); + ::log::debug!("peer has closed connection"); + + self.close_after_writing = true; + + break; } let _ = self.tls.read_tls(&mut &buf[..bytes_read]).unwrap(); @@ -160,23 +167,26 @@ impl Connection { let mut added_plaintext = false; - while io_state.plaintext_bytes_to_read() != 0 { - match self.tls.reader().read(&mut buf) { - Ok(0) => { - break; - } - Ok(amt) => { - self.request_buffer.extend_from_slice(&buf[..amt]); + if io_state.plaintext_bytes_to_read() != 0 { + loop { + match self.tls.reader().read(&mut buf) { + Ok(0) => { + break; + } + Ok(amt) => { + self.request_buffer.extend_from_slice(&buf[..amt]); - added_plaintext = true; - }, - Err(err) if err.kind() == ErrorKind::WouldBlock => { - break; - } - Err(err) => { - ::log::info!("tls.reader().read error: {:?}", err); + added_plaintext = true; + }, + Err(err) if err.kind() == ErrorKind::WouldBlock => { + break; + } + Err(err) => { + // Should never happen + ::log::error!("tls.reader().read error: {:?}", err); - break; + break; + } } } } @@ -184,15 +194,24 @@ impl Connection { if added_plaintext { match Request::from_bytes(&self.request_buffer[..]) { Ok(request) => { - self.expects_request = false; + self.wait_for_response = true; - ::log::info!("received request: {:?}", request); + ::log::trace!("received request: {:?}", request); } Err(RequestParseError::NeedMoreData) => { - ::log::info!("need more request data. current data: {:?}", std::str::from_utf8(&self.request_buffer)); + ::log::debug!("need more request data. current data: {:?}", std::str::from_utf8(&self.request_buffer)); } Err(RequestParseError::Invalid(err)) => { - return Err(anyhow::anyhow!("request parse error: {:?}", err)); + ::log::debug!("invalid request: {:?}", err); + + let response = Response::Failure(FailureResponse { + failure_reason: "Invalid request".into(), + }); + + self.queue_response(&response)?; + self.close_after_writing = true; + + break; } } } @@ -210,7 +229,7 @@ impl Connection { return Ok(()); } - ::log::info!("write_tls (wants write)"); + ::log::debug!("write_tls (wants write)"); let mut buf = Vec::new(); let mut buf = Cursor::new(&mut buf); @@ -219,7 +238,29 @@ impl Connection { self.tls.write_tls(&mut buf).unwrap(); } - self.stream.write_all(&buf.into_inner()).await.unwrap(); + self.stream.write_all(&buf.into_inner()).await?; + self.stream.flush().await?; + + Ok(()) + } + + fn queue_response(&mut self, response: &Response) -> anyhow::Result<()> { + let mut body = Vec::new(); + + response.write(&mut body).unwrap(); + + let content_len = body.len() + 2; // 2 is for newlines at end + let content_len_num_digits = num_digits_in_usize(content_len); + + let mut response_bytes = Vec::with_capacity(39 + content_len_num_digits + body.len()); + + response_bytes.extend_from_slice(b"HTTP/1.1 200 OK\r\nContent-Length: "); + ::itoa::write(&mut response_bytes, content_len)?; + response_bytes.extend_from_slice(b"\r\n\r\n"); + response_bytes.append(&mut body); + response_bytes.extend_from_slice(b"\r\n"); + + self.tls.writer().write(&response_bytes[..])?; Ok(()) } diff --git a/aquatic_http/src/lib/lib.rs b/aquatic_http/src/lib/lib.rs index ee6f8e8..7f1456f 100644 --- a/aquatic_http/src/lib/lib.rs +++ b/aquatic_http/src/lib/lib.rs @@ -1,6 +1,7 @@ use cfg_if::cfg_if; pub mod config; +pub mod common; #[cfg(feature = "with-mio")] pub mod mio; diff --git a/aquatic_http/src/lib/mio/network/connection.rs b/aquatic_http/src/lib/mio/network/connection.rs index 57fee71..7ac3aa6 100644 --- a/aquatic_http/src/lib/mio/network/connection.rs +++ b/aquatic_http/src/lib/mio/network/connection.rs @@ -10,6 +10,7 @@ use native_tls::{MidHandshakeTlsStream, TlsAcceptor}; use aquatic_http_protocol::request::{Request, RequestParseError}; +use crate::common::num_digits_in_usize; use crate::mio::common::*; use super::stream::Stream; @@ -85,7 +86,7 @@ impl EstablishedConnection { pub fn send_response(&mut self, body: &[u8]) -> ::std::io::Result<()> { let content_len = body.len() + 2; // 2 is for newlines at end - let content_len_num_digits = Self::num_digits_in_usize(content_len); + let content_len_num_digits = num_digits_in_usize(content_len); let mut response = Vec::with_capacity(39 + content_len_num_digits + body.len()); @@ -110,18 +111,6 @@ impl EstablishedConnection { Ok(()) } - fn num_digits_in_usize(mut number: usize) -> usize { - let mut num_digits = 1usize; - - while number >= 10 { - num_digits += 1; - - number /= 10; - } - - num_digits - } - #[inline] pub fn clear_buffer(&mut self) { self.bytes_read = 0; @@ -269,23 +258,3 @@ impl Connection { } pub type ConnectionMap = HashMap; - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_num_digits_in_usize() { - let f = EstablishedConnection::num_digits_in_usize; - - assert_eq!(f(0), 1); - assert_eq!(f(1), 1); - assert_eq!(f(9), 1); - assert_eq!(f(10), 2); - assert_eq!(f(11), 2); - assert_eq!(f(99), 2); - assert_eq!(f(100), 3); - assert_eq!(f(101), 3); - assert_eq!(f(1000), 4); - } -} From 636a434ca6cfd087e6ebca83ae50bbd776ed4e15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20Frosteg=C3=A5rd?= Date: Tue, 26 Oct 2021 21:59:40 +0200 Subject: [PATCH 11/59] aquatic_http: glommio: unless keep_alive set, close after send --- aquatic_http/src/lib/glommio/network.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/aquatic_http/src/lib/glommio/network.rs b/aquatic_http/src/lib/glommio/network.rs index 589a907..bbea89f 100644 --- a/aquatic_http/src/lib/glommio/network.rs +++ b/aquatic_http/src/lib/glommio/network.rs @@ -30,6 +30,7 @@ struct ConnectionReference { } struct Connection { + config: Rc, // request_senders: Rc>, response_receiver: LocalReceiver, tls: ServerConnection, @@ -74,6 +75,7 @@ pub async fn run_socket_worker( let entry = slab.vacant_entry(); let conn = Connection { + config: config.clone(), // request_senders: request_senders.clone(), response_receiver, tls: ServerConnection::new(tls_config.clone()).unwrap(), @@ -129,7 +131,9 @@ impl Connection { self.wait_for_response = false; - // TODO: trigger close here if keepalive is false + if !self.config.network.keep_alive { + self.close_after_writing = true; + } } } From 8a66b5ce6963ce86dc38ce619425d2a479eb4753 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20Frosteg=C3=A5rd?= Date: Tue, 26 Oct 2021 22:21:38 +0200 Subject: [PATCH 12/59] aquatic_udp: glommio: return Request in read_tls to reduce state --- aquatic_http/src/lib/glommio/network.rs | 29 +++++++++++++++---------- 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/aquatic_http/src/lib/glommio/network.rs b/aquatic_http/src/lib/glommio/network.rs index bbea89f..6b21c84 100644 --- a/aquatic_http/src/lib/glommio/network.rs +++ b/aquatic_http/src/lib/glommio/network.rs @@ -1,5 +1,5 @@ use std::cell::RefCell; -use std::io::{BufReader, Cursor, ErrorKind, Read, Write}; +use std::io::{Cursor, ErrorKind, Read, Write}; use std::rc::Rc; use std::sync::Arc; use std::sync::atomic::{AtomicUsize, Ordering}; @@ -13,7 +13,7 @@ use glommio::prelude::*; use glommio::net::{TcpListener, TcpStream}; use glommio::channels::local_channel::{new_bounded, LocalReceiver, LocalSender}; use glommio::task::JoinHandle; -use rustls::{IoState, ServerConnection}; +use rustls::{ServerConnection}; use slab::Slab; use crate::common::num_digits_in_usize; @@ -37,7 +37,6 @@ struct Connection { stream: TcpStream, index: ConnectionId, request_buffer: Vec, - wait_for_response: bool, close_after_writing: bool, } @@ -82,7 +81,6 @@ pub async fn run_socket_worker( stream, index: ConnectionId(entry.key()), request_buffer: Vec::new(), - wait_for_response: false, close_after_writing: false, }; @@ -123,13 +121,20 @@ async fn receive_responses( impl Connection { async fn handle_stream(&mut self) -> anyhow::Result<()> { loop { - self.read_tls().await?; + let opt_request = self.read_tls().await?; - if self.wait_for_response { + if let Some(request) = opt_request { + let peer_addr = self.stream + .peer_addr() + .map_err(|err| anyhow::anyhow!("Couldn't get peer addr: {:?}", err))?; + + // TODO: send request to channel + + // Wait for response to arrive, then send it if let Some(response) = self.response_receiver.recv().await { - self.queue_response(&response)?; + // TODO: compare IP addresses? - self.wait_for_response = false; + self.queue_response(&response)?; if !self.config.network.keep_alive { self.close_after_writing = true; @@ -149,7 +154,7 @@ impl Connection { Ok(()) } - async fn read_tls(&mut self) -> anyhow::Result<()> { + async fn read_tls(&mut self) -> anyhow::Result> { loop { ::log::debug!("read_tls"); @@ -198,9 +203,9 @@ impl Connection { if added_plaintext { match Request::from_bytes(&self.request_buffer[..]) { Ok(request) => { - self.wait_for_response = true; + ::log::debug!("received request: {:?}", request); - ::log::trace!("received request: {:?}", request); + return Ok(Some(request)); } Err(RequestParseError::NeedMoreData) => { ::log::debug!("need more request data. current data: {:?}", std::str::from_utf8(&self.request_buffer)); @@ -225,7 +230,7 @@ impl Connection { } } - Ok(()) + Ok(None) } async fn write_tls(&mut self) -> anyhow::Result<()> { From ce8d1ba0d6a6770778c093aabf5d8d5c27cffe2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20Frosteg=C3=A5rd?= Date: Tue, 26 Oct 2021 23:33:08 +0200 Subject: [PATCH 13/59] aquatic_http: move much logic formerly in mio module into common mod --- aquatic_http/src/lib/common/handlers.rs | 217 +++++++++++++++++++++++ aquatic_http/src/lib/common/mod.rs | 160 +++++++++++++++++ aquatic_http/src/lib/glommio/handlers.rs | 0 aquatic_http/src/lib/glommio/mod.rs | 1 + aquatic_http/src/lib/mio/common.rs | 153 +--------------- aquatic_http/src/lib/mio/handler.rs | 195 +------------------- aquatic_http/src/lib/mio/network/mod.rs | 1 + 7 files changed, 387 insertions(+), 340 deletions(-) create mode 100644 aquatic_http/src/lib/common/handlers.rs create mode 100644 aquatic_http/src/lib/glommio/handlers.rs diff --git a/aquatic_http/src/lib/common/handlers.rs b/aquatic_http/src/lib/common/handlers.rs new file mode 100644 index 0000000..7f10290 --- /dev/null +++ b/aquatic_http/src/lib/common/handlers.rs @@ -0,0 +1,217 @@ +use std::collections::BTreeMap; +use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; + +use either::Either; +use rand::{Rng}; + +use aquatic_common::extract_response_peers; +use aquatic_http_protocol::request::*; +use aquatic_http_protocol::response::*; + +use crate::config::Config; +use super::*; + +pub fn handle_announce_request( + config: &Config, + rng: &mut impl Rng, + torrent_maps: &mut TorrentMaps, + valid_until: ValidUntil, + meta: ConnectionMeta, + request: AnnounceRequest, +) -> AnnounceResponse { + let peer_ip = convert_ipv4_mapped_ipv6(meta.peer_addr.ip()); + + ::log::debug!("peer ip: {:?}", peer_ip); + + match peer_ip { + IpAddr::V4(peer_ip_address) => { + let torrent_data: &mut TorrentData = + torrent_maps.ipv4.entry(request.info_hash).or_default(); + + let peer_connection_meta = PeerConnectionMeta { + worker_index: meta.worker_index, + poll_token: meta.poll_token, + peer_ip_address, + }; + + let (seeders, leechers, response_peers) = upsert_peer_and_get_response_peers( + config, + rng, + peer_connection_meta, + torrent_data, + request, + valid_until, + ); + + let response = AnnounceResponse { + complete: seeders, + incomplete: leechers, + announce_interval: config.protocol.peer_announce_interval, + peers: ResponsePeerListV4(response_peers), + peers6: ResponsePeerListV6(vec![]), + }; + + response + } + IpAddr::V6(peer_ip_address) => { + let torrent_data: &mut TorrentData = + torrent_maps.ipv6.entry(request.info_hash).or_default(); + + let peer_connection_meta = PeerConnectionMeta { + worker_index: meta.worker_index, + poll_token: meta.poll_token, + peer_ip_address, + }; + + let (seeders, leechers, response_peers) = upsert_peer_and_get_response_peers( + config, + rng, + peer_connection_meta, + torrent_data, + request, + valid_until, + ); + + let response = AnnounceResponse { + complete: seeders, + incomplete: leechers, + announce_interval: config.protocol.peer_announce_interval, + peers: ResponsePeerListV4(vec![]), + peers6: ResponsePeerListV6(response_peers), + }; + + response + } + } +} + +/// Insert/update peer. Return num_seeders, num_leechers and response peers +pub fn upsert_peer_and_get_response_peers( + config: &Config, + rng: &mut impl Rng, + request_sender_meta: PeerConnectionMeta, + torrent_data: &mut TorrentData, + request: AnnounceRequest, + valid_until: ValidUntil, +) -> (usize, usize, Vec>) { + // Insert/update/remove peer who sent this request + + let peer_status = + PeerStatus::from_event_and_bytes_left(request.event, Some(request.bytes_left)); + + let peer = Peer { + connection_meta: request_sender_meta, + port: request.port, + status: peer_status, + valid_until, + }; + + ::log::debug!("peer: {:?}", peer); + + let ip_or_key = request + .key + .map(Either::Right) + .unwrap_or_else(|| Either::Left(request_sender_meta.peer_ip_address)); + + let peer_map_key = PeerMapKey { + peer_id: request.peer_id, + ip_or_key, + }; + + ::log::debug!("peer map key: {:?}", peer_map_key); + + let opt_removed_peer = match peer_status { + PeerStatus::Leeching => { + torrent_data.num_leechers += 1; + + torrent_data.peers.insert(peer_map_key.clone(), peer) + } + PeerStatus::Seeding => { + torrent_data.num_seeders += 1; + + torrent_data.peers.insert(peer_map_key.clone(), peer) + } + PeerStatus::Stopped => torrent_data.peers.remove(&peer_map_key), + }; + + ::log::debug!("opt_removed_peer: {:?}", opt_removed_peer); + + match opt_removed_peer.map(|peer| peer.status) { + Some(PeerStatus::Leeching) => { + torrent_data.num_leechers -= 1; + } + Some(PeerStatus::Seeding) => { + torrent_data.num_seeders -= 1; + } + _ => {} + } + + ::log::debug!("peer request numwant: {:?}", request.numwant); + + let max_num_peers_to_take = match request.numwant { + Some(0) | None => config.protocol.max_peers, + Some(numwant) => numwant.min(config.protocol.max_peers), + }; + + let response_peers: Vec> = extract_response_peers( + rng, + &torrent_data.peers, + max_num_peers_to_take, + peer_map_key, + Peer::to_response_peer, + ); + + ( + torrent_data.num_seeders, + torrent_data.num_leechers, + response_peers, + ) +} + +pub fn handle_scrape_request( + config: &Config, + torrent_maps: &mut TorrentMaps, + (meta, request): (ConnectionMeta, ScrapeRequest), +) -> ScrapeResponse { + let num_to_take = request + .info_hashes + .len() + .min(config.protocol.max_scrape_torrents); + + let mut response = ScrapeResponse { + files: BTreeMap::new(), + }; + + let peer_ip = convert_ipv4_mapped_ipv6(meta.peer_addr.ip()); + + // If request.info_hashes is empty, don't return scrape for all + // torrents, even though reference server does it. It is too expensive. + if peer_ip.is_ipv4() { + for info_hash in request.info_hashes.into_iter().take(num_to_take) { + if let Some(torrent_data) = torrent_maps.ipv4.get(&info_hash) { + let stats = ScrapeStatistics { + complete: torrent_data.num_seeders, + downloaded: 0, // No implementation planned + incomplete: torrent_data.num_leechers, + }; + + response.files.insert(info_hash, stats); + } + } + } else { + for info_hash in request.info_hashes.into_iter().take(num_to_take) { + if let Some(torrent_data) = torrent_maps.ipv6.get(&info_hash) { + let stats = ScrapeStatistics { + complete: torrent_data.num_seeders, + downloaded: 0, // No implementation planned + incomplete: torrent_data.num_leechers, + }; + + response.files.insert(info_hash, stats); + } + } + }; + + response +} + diff --git a/aquatic_http/src/lib/common/mod.rs b/aquatic_http/src/lib/common/mod.rs index 8002a13..845db1c 100644 --- a/aquatic_http/src/lib/common/mod.rs +++ b/aquatic_http/src/lib/common/mod.rs @@ -1,3 +1,163 @@ +use std::net::{Ipv4Addr, Ipv6Addr, SocketAddr}; +use std::sync::Arc; +use std::time::Instant; + +use aquatic_common::access_list::{AccessList}; +use either::Either; +use hashbrown::HashMap; +use indexmap::IndexMap; +use mio::Token; +use smartstring::{LazyCompact, SmartString}; + +pub use aquatic_common::{convert_ipv4_mapped_ipv6, ValidUntil}; + +use aquatic_http_protocol::common::*; +use aquatic_http_protocol::response::{ResponsePeer}; + +use crate::config::Config; + +pub mod handlers; + +pub trait Ip: ::std::fmt::Debug + Copy + Eq + ::std::hash::Hash {} + +impl Ip for Ipv4Addr {} +impl Ip for Ipv6Addr {} + +#[derive(Clone, Copy, Debug)] +pub struct ConnectionMeta { + /// Index of socket worker responsible for this connection. Required for + /// sending back response through correct channel to correct worker. + pub worker_index: usize, + pub peer_addr: SocketAddr, + pub poll_token: Token, +} + +#[derive(Clone, Copy, Debug)] +pub struct PeerConnectionMeta { + pub worker_index: usize, + pub poll_token: Token, + pub peer_ip_address: I, +} + +#[derive(PartialEq, Eq, Clone, Copy, Debug)] +pub enum PeerStatus { + Seeding, + Leeching, + Stopped, +} + +impl PeerStatus { + /// Determine peer status from announce event and number of bytes left. + /// + /// Likely, the last branch will be taken most of the time. + #[inline] + pub fn from_event_and_bytes_left(event: AnnounceEvent, opt_bytes_left: Option) -> Self { + if let AnnounceEvent::Stopped = event { + Self::Stopped + } else if let Some(0) = opt_bytes_left { + Self::Seeding + } else { + Self::Leeching + } + } +} + +#[derive(Debug, Clone, Copy)] +pub struct Peer { + pub connection_meta: PeerConnectionMeta, + pub port: u16, + pub status: PeerStatus, + pub valid_until: ValidUntil, +} + +impl Peer { + pub fn to_response_peer(&self) -> ResponsePeer { + ResponsePeer { + ip_address: self.connection_meta.peer_ip_address, + port: self.port, + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct PeerMapKey { + pub peer_id: PeerId, + pub ip_or_key: Either>, +} + +pub type PeerMap = IndexMap, Peer>; + +pub struct TorrentData { + pub peers: PeerMap, + pub num_seeders: usize, + pub num_leechers: usize, +} + +impl Default for TorrentData { + #[inline] + fn default() -> Self { + Self { + peers: IndexMap::new(), + num_seeders: 0, + num_leechers: 0, + } + } +} + +pub type TorrentMap = HashMap>; + +#[derive(Default)] +pub struct TorrentMaps { + pub ipv4: TorrentMap, + pub ipv6: TorrentMap, +} + +impl TorrentMaps { + pub fn clean(&mut self, config: &Config, access_list: &Arc) { + Self::clean_torrent_map(config, access_list, &mut self.ipv4); + Self::clean_torrent_map(config, access_list, &mut self.ipv6); + } + + fn clean_torrent_map( + config: &Config, + access_list: &Arc, + torrent_map: &mut TorrentMap, + ) { + let now = Instant::now(); + + torrent_map.retain(|info_hash, torrent_data| { + if !access_list.allows(config.access_list.mode, &info_hash.0) { + return false; + } + + let num_seeders = &mut torrent_data.num_seeders; + let num_leechers = &mut torrent_data.num_leechers; + + torrent_data.peers.retain(|_, peer| { + let keep = peer.valid_until.0 >= now; + + if !keep { + match peer.status { + PeerStatus::Seeding => { + *num_seeders -= 1; + } + PeerStatus::Leeching => { + *num_leechers -= 1; + } + _ => (), + }; + } + + keep + }); + + !torrent_data.peers.is_empty() + }); + + torrent_map.shrink_to_fit(); + } +} + pub fn num_digits_in_usize(mut number: usize) -> usize { let mut num_digits = 1usize; diff --git a/aquatic_http/src/lib/glommio/handlers.rs b/aquatic_http/src/lib/glommio/handlers.rs new file mode 100644 index 0000000..e69de29 diff --git a/aquatic_http/src/lib/glommio/mod.rs b/aquatic_http/src/lib/glommio/mod.rs index e5b577e..a88a8d4 100644 --- a/aquatic_http/src/lib/glommio/mod.rs +++ b/aquatic_http/src/lib/glommio/mod.rs @@ -5,6 +5,7 @@ use glommio::{channels::channel_mesh::MeshBuilder, prelude::*}; use crate::config::Config; +mod handlers; mod network; const SHARED_CHANNEL_SIZE: usize = 1024; diff --git a/aquatic_http/src/lib/mio/common.rs b/aquatic_http/src/lib/mio/common.rs index e4a9832..5c68ca6 100644 --- a/aquatic_http/src/lib/mio/common.rs +++ b/aquatic_http/src/lib/mio/common.rs @@ -1,168 +1,21 @@ -use std::net::{Ipv4Addr, Ipv6Addr, SocketAddr}; use std::sync::Arc; -use std::time::Instant; -use aquatic_common::access_list::{AccessList, AccessListArcSwap}; +use aquatic_common::access_list::{AccessListArcSwap}; use crossbeam_channel::{Receiver, Sender}; -use either::Either; -use hashbrown::HashMap; -use indexmap::IndexMap; use log::error; use mio::Token; use parking_lot::Mutex; -use smartstring::{LazyCompact, SmartString}; pub use aquatic_common::{convert_ipv4_mapped_ipv6, ValidUntil}; -use aquatic_http_protocol::common::*; use aquatic_http_protocol::request::Request; -use aquatic_http_protocol::response::{Response, ResponsePeer}; +use aquatic_http_protocol::response::{Response}; -use crate::config::Config; +use crate::common::*; pub const LISTENER_TOKEN: Token = Token(0); pub const CHANNEL_TOKEN: Token = Token(1); -pub trait Ip: ::std::fmt::Debug + Copy + Eq + ::std::hash::Hash {} - -impl Ip for Ipv4Addr {} -impl Ip for Ipv6Addr {} - -#[derive(Clone, Copy, Debug)] -pub struct ConnectionMeta { - /// Index of socket worker responsible for this connection. Required for - /// sending back response through correct channel to correct worker. - pub worker_index: usize, - pub peer_addr: SocketAddr, - pub poll_token: Token, -} - -#[derive(Clone, Copy, Debug)] -pub struct PeerConnectionMeta { - pub worker_index: usize, - pub poll_token: Token, - pub peer_ip_address: I, -} - -#[derive(PartialEq, Eq, Clone, Copy, Debug)] -pub enum PeerStatus { - Seeding, - Leeching, - Stopped, -} - -impl PeerStatus { - /// Determine peer status from announce event and number of bytes left. - /// - /// Likely, the last branch will be taken most of the time. - #[inline] - pub fn from_event_and_bytes_left(event: AnnounceEvent, opt_bytes_left: Option) -> Self { - if let AnnounceEvent::Stopped = event { - Self::Stopped - } else if let Some(0) = opt_bytes_left { - Self::Seeding - } else { - Self::Leeching - } - } -} - -#[derive(Debug, Clone, Copy)] -pub struct Peer { - pub connection_meta: PeerConnectionMeta, - pub port: u16, - pub status: PeerStatus, - pub valid_until: ValidUntil, -} - -impl Peer { - pub fn to_response_peer(&self) -> ResponsePeer { - ResponsePeer { - ip_address: self.connection_meta.peer_ip_address, - port: self.port, - } - } -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct PeerMapKey { - pub peer_id: PeerId, - pub ip_or_key: Either>, -} - -pub type PeerMap = IndexMap, Peer>; - -pub struct TorrentData { - pub peers: PeerMap, - pub num_seeders: usize, - pub num_leechers: usize, -} - -impl Default for TorrentData { - #[inline] - fn default() -> Self { - Self { - peers: IndexMap::new(), - num_seeders: 0, - num_leechers: 0, - } - } -} - -pub type TorrentMap = HashMap>; - -#[derive(Default)] -pub struct TorrentMaps { - pub ipv4: TorrentMap, - pub ipv6: TorrentMap, -} - -impl TorrentMaps { - pub fn clean(&mut self, config: &Config, access_list: &Arc) { - Self::clean_torrent_map(config, access_list, &mut self.ipv4); - Self::clean_torrent_map(config, access_list, &mut self.ipv6); - } - - fn clean_torrent_map( - config: &Config, - access_list: &Arc, - torrent_map: &mut TorrentMap, - ) { - let now = Instant::now(); - - torrent_map.retain(|info_hash, torrent_data| { - if !access_list.allows(config.access_list.mode, &info_hash.0) { - return false; - } - - let num_seeders = &mut torrent_data.num_seeders; - let num_leechers = &mut torrent_data.num_leechers; - - torrent_data.peers.retain(|_, peer| { - let keep = peer.valid_until.0 >= now; - - if !keep { - match peer.status { - PeerStatus::Seeding => { - *num_seeders -= 1; - } - PeerStatus::Leeching => { - *num_leechers -= 1; - } - _ => (), - }; - } - - keep - }); - - !torrent_data.peers.is_empty() - }); - - torrent_map.shrink_to_fit(); - } -} - #[derive(Clone)] pub struct State { pub access_list: Arc, diff --git a/aquatic_http/src/lib/mio/handler.rs b/aquatic_http/src/lib/mio/handler.rs index c54df07..510bac7 100644 --- a/aquatic_http/src/lib/mio/handler.rs +++ b/aquatic_http/src/lib/mio/handler.rs @@ -1,18 +1,16 @@ -use std::collections::BTreeMap; -use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; use std::sync::Arc; use std::time::Duration; use std::vec::Drain; -use either::Either; use mio::Waker; use parking_lot::MutexGuard; use rand::{rngs::SmallRng, Rng, SeedableRng}; -use aquatic_common::extract_response_peers; use aquatic_http_protocol::request::*; use aquatic_http_protocol::response::*; +use crate::common::handlers::{handle_announce_request, handle_scrape_request}; +use crate::common::*; use crate::config::Config; use super::common::*; @@ -105,159 +103,13 @@ pub fn handle_announce_requests( let valid_until = ValidUntil::new(config.cleaning.max_peer_age); for (meta, request) in requests { - let peer_ip = convert_ipv4_mapped_ipv6(meta.peer_addr.ip()); + let response = handle_announce_request(config, rng, torrent_maps, valid_until, meta, request); - ::log::debug!("peer ip: {:?}", peer_ip); - - let response = match peer_ip { - IpAddr::V4(peer_ip_address) => { - let torrent_data: &mut TorrentData = - torrent_maps.ipv4.entry(request.info_hash).or_default(); - - let peer_connection_meta = PeerConnectionMeta { - worker_index: meta.worker_index, - poll_token: meta.poll_token, - peer_ip_address, - }; - - let (seeders, leechers, response_peers) = upsert_peer_and_get_response_peers( - config, - rng, - peer_connection_meta, - torrent_data, - request, - valid_until, - ); - - let response = AnnounceResponse { - complete: seeders, - incomplete: leechers, - announce_interval: config.protocol.peer_announce_interval, - peers: ResponsePeerListV4(response_peers), - peers6: ResponsePeerListV6(vec![]), - }; - - Response::Announce(response) - } - IpAddr::V6(peer_ip_address) => { - let torrent_data: &mut TorrentData = - torrent_maps.ipv6.entry(request.info_hash).or_default(); - - let peer_connection_meta = PeerConnectionMeta { - worker_index: meta.worker_index, - poll_token: meta.poll_token, - peer_ip_address, - }; - - let (seeders, leechers, response_peers) = upsert_peer_and_get_response_peers( - config, - rng, - peer_connection_meta, - torrent_data, - request, - valid_until, - ); - - let response = AnnounceResponse { - complete: seeders, - incomplete: leechers, - announce_interval: config.protocol.peer_announce_interval, - peers: ResponsePeerListV4(vec![]), - peers6: ResponsePeerListV6(response_peers), - }; - - Response::Announce(response) - } - }; - - response_channel_sender.send(meta, response); + response_channel_sender.send(meta, Response::Announce(response)); wake_socket_workers[meta.worker_index] = true; } } -/// Insert/update peer. Return num_seeders, num_leechers and response peers -fn upsert_peer_and_get_response_peers( - config: &Config, - rng: &mut impl Rng, - request_sender_meta: PeerConnectionMeta, - torrent_data: &mut TorrentData, - request: AnnounceRequest, - valid_until: ValidUntil, -) -> (usize, usize, Vec>) { - // Insert/update/remove peer who sent this request - - let peer_status = - PeerStatus::from_event_and_bytes_left(request.event, Some(request.bytes_left)); - - let peer = Peer { - connection_meta: request_sender_meta, - port: request.port, - status: peer_status, - valid_until, - }; - - ::log::debug!("peer: {:?}", peer); - - let ip_or_key = request - .key - .map(Either::Right) - .unwrap_or_else(|| Either::Left(request_sender_meta.peer_ip_address)); - - let peer_map_key = PeerMapKey { - peer_id: request.peer_id, - ip_or_key, - }; - - ::log::debug!("peer map key: {:?}", peer_map_key); - - let opt_removed_peer = match peer_status { - PeerStatus::Leeching => { - torrent_data.num_leechers += 1; - - torrent_data.peers.insert(peer_map_key.clone(), peer) - } - PeerStatus::Seeding => { - torrent_data.num_seeders += 1; - - torrent_data.peers.insert(peer_map_key.clone(), peer) - } - PeerStatus::Stopped => torrent_data.peers.remove(&peer_map_key), - }; - - ::log::debug!("opt_removed_peer: {:?}", opt_removed_peer); - - match opt_removed_peer.map(|peer| peer.status) { - Some(PeerStatus::Leeching) => { - torrent_data.num_leechers -= 1; - } - Some(PeerStatus::Seeding) => { - torrent_data.num_seeders -= 1; - } - _ => {} - } - - ::log::debug!("peer request numwant: {:?}", request.numwant); - - let max_num_peers_to_take = match request.numwant { - Some(0) | None => config.protocol.max_peers, - Some(numwant) => numwant.min(config.protocol.max_peers), - }; - - let response_peers: Vec> = extract_response_peers( - rng, - &torrent_data.peers, - max_num_peers_to_take, - peer_map_key, - Peer::to_response_peer, - ); - - ( - torrent_data.num_seeders, - torrent_data.num_leechers, - response_peers, - ) -} - pub fn handle_scrape_requests( config: &Config, torrent_maps: &mut TorrentMaps, @@ -266,44 +118,7 @@ pub fn handle_scrape_requests( requests: Drain<(ConnectionMeta, ScrapeRequest)>, ) { for (meta, request) in requests { - let num_to_take = request - .info_hashes - .len() - .min(config.protocol.max_scrape_torrents); - - let mut response = ScrapeResponse { - files: BTreeMap::new(), - }; - - let peer_ip = convert_ipv4_mapped_ipv6(meta.peer_addr.ip()); - - // If request.info_hashes is empty, don't return scrape for all - // torrents, even though reference server does it. It is too expensive. - if peer_ip.is_ipv4() { - for info_hash in request.info_hashes.into_iter().take(num_to_take) { - if let Some(torrent_data) = torrent_maps.ipv4.get(&info_hash) { - let stats = ScrapeStatistics { - complete: torrent_data.num_seeders, - downloaded: 0, // No implementation planned - incomplete: torrent_data.num_leechers, - }; - - response.files.insert(info_hash, stats); - } - } - } else { - for info_hash in request.info_hashes.into_iter().take(num_to_take) { - if let Some(torrent_data) = torrent_maps.ipv6.get(&info_hash) { - let stats = ScrapeStatistics { - complete: torrent_data.num_seeders, - downloaded: 0, // No implementation planned - incomplete: torrent_data.num_leechers, - }; - - response.files.insert(info_hash, stats); - } - } - }; + let response = handle_scrape_request(config, torrent_maps, (meta, request)); response_channel_sender.send(meta, Response::Scrape(response)); wake_socket_workers[meta.worker_index] = true; diff --git a/aquatic_http/src/lib/mio/network/mod.rs b/aquatic_http/src/lib/mio/network/mod.rs index ed0a552..52e8b0d 100644 --- a/aquatic_http/src/lib/mio/network/mod.rs +++ b/aquatic_http/src/lib/mio/network/mod.rs @@ -15,6 +15,7 @@ use aquatic_http_protocol::response::*; use crate::mio::common::*; use crate::config::Config; +use crate::common::*; pub mod connection; pub mod stream; From ea2366c8086b96e6484d9af541d5967030b5d2a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20Frosteg=C3=A5rd?= Date: Tue, 26 Oct 2021 23:38:15 +0200 Subject: [PATCH 14/59] aquatic_http: simplify mio request handler --- aquatic_http/src/lib/mio/handler.rs | 72 +++++++++-------------------- 1 file changed, 22 insertions(+), 50 deletions(-) diff --git a/aquatic_http/src/lib/mio/handler.rs b/aquatic_http/src/lib/mio/handler.rs index 510bac7..0425a7b 100644 --- a/aquatic_http/src/lib/mio/handler.rs +++ b/aquatic_http/src/lib/mio/handler.rs @@ -1,10 +1,9 @@ use std::sync::Arc; use std::time::Duration; -use std::vec::Drain; use mio::Waker; use parking_lot::MutexGuard; -use rand::{rngs::SmallRng, Rng, SeedableRng}; +use rand::{rngs::SmallRng, SeedableRng}; use aquatic_http_protocol::request::*; use aquatic_http_protocol::response::*; @@ -63,22 +62,28 @@ pub fn run_request_worker( let mut torrent_map_guard = opt_torrent_map_guard.unwrap_or_else(|| state.torrent_maps.lock()); - handle_announce_requests( - &config, - &mut rng, - &mut torrent_map_guard, - &response_channel_sender, - &mut wake_socket_workers, - announce_requests.drain(..), - ); + let valid_until = ValidUntil::new(config.cleaning.max_peer_age); - handle_scrape_requests( - &config, - &mut torrent_map_guard, - &response_channel_sender, - &mut wake_socket_workers, - scrape_requests.drain(..), - ); + for (meta, request) in announce_requests.drain(..) { + let response = handle_announce_request( + &config, + &mut rng, + &mut torrent_map_guard, + valid_until, + meta, + request + ); + + response_channel_sender.send(meta, Response::Announce(response)); + wake_socket_workers[meta.worker_index] = true; + } + + for (meta, request) in scrape_requests.drain(..) { + let response = handle_scrape_request(&config, &mut torrent_map_guard, (meta, request)); + + response_channel_sender.send(meta, Response::Scrape(response)); + wake_socket_workers[meta.worker_index] = true; + } for (worker_index, wake) in wake_socket_workers.iter_mut().enumerate() { if *wake { @@ -91,36 +96,3 @@ pub fn run_request_worker( } } } - -pub fn handle_announce_requests( - config: &Config, - rng: &mut impl Rng, - torrent_maps: &mut TorrentMaps, - response_channel_sender: &ResponseChannelSender, - wake_socket_workers: &mut Vec, - requests: Drain<(ConnectionMeta, AnnounceRequest)>, -) { - let valid_until = ValidUntil::new(config.cleaning.max_peer_age); - - for (meta, request) in requests { - let response = handle_announce_request(config, rng, torrent_maps, valid_until, meta, request); - - response_channel_sender.send(meta, Response::Announce(response)); - wake_socket_workers[meta.worker_index] = true; - } -} - -pub fn handle_scrape_requests( - config: &Config, - torrent_maps: &mut TorrentMaps, - response_channel_sender: &ResponseChannelSender, - wake_socket_workers: &mut Vec, - requests: Drain<(ConnectionMeta, ScrapeRequest)>, -) { - for (meta, request) in requests { - let response = handle_scrape_request(config, torrent_maps, (meta, request)); - - response_channel_sender.send(meta, Response::Scrape(response)); - wake_socket_workers[meta.worker_index] = true; - } -} From 4fc1509a7950015e0d2ed6071345bda300d67208 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20Frosteg=C3=A5rd?= Date: Tue, 26 Oct 2021 23:40:11 +0200 Subject: [PATCH 15/59] aquatic_http: improve mio and common request handling code --- aquatic_http/src/lib/common/handlers.rs | 3 ++- aquatic_http/src/lib/mio/handler.rs | 17 +++++++++++------ 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/aquatic_http/src/lib/common/handlers.rs b/aquatic_http/src/lib/common/handlers.rs index 7f10290..fa7fe3d 100644 --- a/aquatic_http/src/lib/common/handlers.rs +++ b/aquatic_http/src/lib/common/handlers.rs @@ -171,7 +171,8 @@ pub fn upsert_peer_and_get_response_peers( pub fn handle_scrape_request( config: &Config, torrent_maps: &mut TorrentMaps, - (meta, request): (ConnectionMeta, ScrapeRequest), + meta: ConnectionMeta, + request: ScrapeRequest, ) -> ScrapeResponse { let num_to_take = request .info_hashes diff --git a/aquatic_http/src/lib/mio/handler.rs b/aquatic_http/src/lib/mio/handler.rs index 0425a7b..c42acb1 100644 --- a/aquatic_http/src/lib/mio/handler.rs +++ b/aquatic_http/src/lib/mio/handler.rs @@ -30,7 +30,7 @@ pub fn run_request_worker( let timeout = Duration::from_micros(config.handlers.channel_recv_timeout_microseconds); loop { - let mut opt_torrent_map_guard: Option> = None; + let mut opt_torrent_maps: Option> = None; // If torrent state mutex is locked, just keep collecting requests // and process them later. This can happen with either multiple @@ -51,7 +51,7 @@ pub fn run_request_worker( } None => { if let Some(torrent_guard) = state.torrent_maps.try_lock() { - opt_torrent_map_guard = Some(torrent_guard); + opt_torrent_maps = Some(torrent_guard); break; } @@ -59,8 +59,8 @@ pub fn run_request_worker( } } - let mut torrent_map_guard = - opt_torrent_map_guard.unwrap_or_else(|| state.torrent_maps.lock()); + let mut torrent_maps = + opt_torrent_maps.unwrap_or_else(|| state.torrent_maps.lock()); let valid_until = ValidUntil::new(config.cleaning.max_peer_age); @@ -68,7 +68,7 @@ pub fn run_request_worker( let response = handle_announce_request( &config, &mut rng, - &mut torrent_map_guard, + &mut torrent_maps, valid_until, meta, request @@ -79,7 +79,12 @@ pub fn run_request_worker( } for (meta, request) in scrape_requests.drain(..) { - let response = handle_scrape_request(&config, &mut torrent_map_guard, (meta, request)); + let response = handle_scrape_request( + &config, + &mut torrent_maps, + meta, + request + ); response_channel_sender.send(meta, Response::Scrape(response)); wake_socket_workers[meta.worker_index] = true; From 8f0dabc70624ae30eb1fb7835042d0b7fc66e664 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20Frosteg=C3=A5rd?= Date: Wed, 27 Oct 2021 00:38:53 +0200 Subject: [PATCH 16/59] aquatic_http: work on glommio request handlers --- aquatic_http/src/lib/common/mod.rs | 12 +- aquatic_http/src/lib/glommio/common.rs | 56 +++++++++ aquatic_http/src/lib/glommio/handlers.rs | 151 +++++++++++++++++++++++ aquatic_http/src/lib/glommio/mod.rs | 29 +++++ aquatic_http/src/lib/glommio/network.rs | 91 ++++++++++---- aquatic_http/src/lib/mio/network/mod.rs | 12 +- 6 files changed, 311 insertions(+), 40 deletions(-) create mode 100644 aquatic_http/src/lib/glommio/common.rs diff --git a/aquatic_http/src/lib/common/mod.rs b/aquatic_http/src/lib/common/mod.rs index 845db1c..9ff51b1 100644 --- a/aquatic_http/src/lib/common/mod.rs +++ b/aquatic_http/src/lib/common/mod.rs @@ -1,12 +1,10 @@ use std::net::{Ipv4Addr, Ipv6Addr, SocketAddr}; -use std::sync::Arc; use std::time::Instant; use aquatic_common::access_list::{AccessList}; use either::Either; use hashbrown::HashMap; use indexmap::IndexMap; -use mio::Token; use smartstring::{LazyCompact, SmartString}; pub use aquatic_common::{convert_ipv4_mapped_ipv6, ValidUntil}; @@ -27,15 +25,15 @@ impl Ip for Ipv6Addr {} pub struct ConnectionMeta { /// Index of socket worker responsible for this connection. Required for /// sending back response through correct channel to correct worker. - pub worker_index: usize, + pub worker_index: usize, // Or response consumer id in glommio pub peer_addr: SocketAddr, - pub poll_token: Token, + pub poll_token: usize, // Or connection id in glommio } #[derive(Clone, Copy, Debug)] pub struct PeerConnectionMeta { pub worker_index: usize, - pub poll_token: Token, + pub poll_token: usize, pub peer_ip_address: I, } @@ -113,14 +111,14 @@ pub struct TorrentMaps { } impl TorrentMaps { - pub fn clean(&mut self, config: &Config, access_list: &Arc) { + pub fn clean(&mut self, config: &Config, access_list: &AccessList) { Self::clean_torrent_map(config, access_list, &mut self.ipv4); Self::clean_torrent_map(config, access_list, &mut self.ipv6); } fn clean_torrent_map( config: &Config, - access_list: &Arc, + access_list: &AccessList, torrent_map: &mut TorrentMap, ) { let now = Instant::now(); diff --git a/aquatic_http/src/lib/glommio/common.rs b/aquatic_http/src/lib/glommio/common.rs new file mode 100644 index 0000000..1752deb --- /dev/null +++ b/aquatic_http/src/lib/glommio/common.rs @@ -0,0 +1,56 @@ +use std::net::SocketAddr; + +use aquatic_http_protocol::{request::{AnnounceRequest, ScrapeRequest}, response::{AnnounceResponse, ScrapeResponse}}; + +#[derive(Copy, Clone, Debug)] +pub struct ConsumerId(pub usize); + +#[derive(Clone, Copy, Debug)] +pub struct ConnectionId(pub usize); + +#[derive(Debug)] +pub enum ChannelRequest { + Announce { + request: AnnounceRequest, + peer_addr: SocketAddr, + connection_id: ConnectionId, + response_consumer_id: ConsumerId, + }, + Scrape { + request: ScrapeRequest, + peer_addr: SocketAddr, + original_indices: Vec, + connection_id: ConnectionId, + response_consumer_id: ConsumerId, + } +} + +#[derive(Debug)] +pub enum ChannelResponse { + Announce { + response: AnnounceResponse, + peer_addr: SocketAddr, + connection_id: ConnectionId, + }, + Scrape { + response: ScrapeResponse, + peer_addr: SocketAddr, + original_indices: Vec, + connection_id: ConnectionId, + } +} + +impl ChannelResponse { + pub fn get_connection_id(&self) -> ConnectionId { + match self { + Self::Announce { connection_id, .. } => *connection_id, + Self::Scrape { connection_id, .. } => *connection_id, + } + } + pub fn get_peer_addr(&self) -> SocketAddr { + match self { + Self::Announce { peer_addr, .. } => *peer_addr, + Self::Scrape { peer_addr, .. } => *peer_addr, + } + } +} \ No newline at end of file diff --git a/aquatic_http/src/lib/glommio/handlers.rs b/aquatic_http/src/lib/glommio/handlers.rs index e69de29..6457318 100644 --- a/aquatic_http/src/lib/glommio/handlers.rs +++ b/aquatic_http/src/lib/glommio/handlers.rs @@ -0,0 +1,151 @@ +use std::cell::RefCell; +use std::rc::Rc; +use std::time::Duration; + +use aquatic_common::access_list::AccessList; +use futures_lite::{Stream, StreamExt}; +use glommio::channels::channel_mesh::{MeshBuilder, Partial, Role, Senders}; +use glommio::timer::TimerActionRepeat; +use glommio::{enclose, prelude::*}; +use rand::prelude::SmallRng; +use rand::SeedableRng; + +use crate::common::handlers::handle_announce_request; +use crate::common::handlers::*; +use crate::common::*; +use crate::config::Config; + +use super::common::*; + +pub async fn run_request_worker( + config: Config, + request_mesh_builder: MeshBuilder, + response_mesh_builder: MeshBuilder, + access_list: AccessList, +) { + let (_, mut request_receivers) = request_mesh_builder.join(Role::Consumer).await.unwrap(); + let (response_senders, _) = response_mesh_builder.join(Role::Producer).await.unwrap(); + + let response_senders = Rc::new(response_senders); + + let torrents = Rc::new(RefCell::new(TorrentMaps::default())); + let access_list = Rc::new(RefCell::new(access_list)); + + // Periodically clean torrents and update access list + TimerActionRepeat::repeat(enclose!((config, torrents, access_list) move || { + enclose!((config, torrents, access_list) move || async move { + // update_access_list(config.clone(), access_list.clone()).await; + + torrents.borrow_mut().clean(&config, &*access_list.borrow()); + + Some(Duration::from_secs(config.cleaning.interval)) + })() + })); + + let mut handles = Vec::new(); + + for (_, receiver) in request_receivers.streams() { + let handle = spawn_local(handle_request_stream( + config.clone(), + torrents.clone(), + response_senders.clone(), + receiver, + )) + .detach(); + + handles.push(handle); + } + + for handle in handles { + handle.await; + } +} + +async fn handle_request_stream( + config: Config, + torrents: Rc>, + response_senders: Rc>, + mut stream: S, +) where + S: Stream + ::std::marker::Unpin, +{ + let mut rng = SmallRng::from_entropy(); + + let max_peer_age = config.cleaning.max_peer_age; + let peer_valid_until = Rc::new(RefCell::new(ValidUntil::new(max_peer_age))); + + TimerActionRepeat::repeat(enclose!((peer_valid_until) move || { + enclose!((peer_valid_until) move || async move { + *peer_valid_until.borrow_mut() = ValidUntil::new(max_peer_age); + + Some(Duration::from_secs(1)) + })() + })); + + while let Some(channel_request) = stream.next().await { + let (response, consumer_id) = match channel_request { + ChannelRequest::Announce { + request, + peer_addr, + response_consumer_id, + connection_id + } => { + let meta = ConnectionMeta { + worker_index: response_consumer_id.0, + poll_token: connection_id.0, + peer_addr, + }; + + let response = handle_announce_request( + &config, + &mut rng, + &mut torrents.borrow_mut(), + peer_valid_until.borrow().to_owned(), + meta, + request, + ); + + let response = ChannelResponse::Announce { + response, + peer_addr, + connection_id, + }; + + (response, response_consumer_id) + } + ChannelRequest::Scrape { + request, + peer_addr, + response_consumer_id, + connection_id, + original_indices, + } => { + let meta = ConnectionMeta { + worker_index: response_consumer_id.0, + poll_token: connection_id.0, + peer_addr, + }; + + let response = handle_scrape_request(&config, &mut torrents.borrow_mut(), meta, request); + + let response = ChannelResponse::Scrape { + response, + peer_addr, + connection_id, + original_indices, + }; + + (response, response_consumer_id) + } + }; + + ::log::debug!("preparing to send response to channel: {:?}", response); + + if let Err(err) = response_senders.try_send_to(consumer_id.0, response) { + ::log::warn!("response_sender.try_send: {:?}", err); + } + + yield_if_needed().await; + } +} + diff --git a/aquatic_http/src/lib/glommio/mod.rs b/aquatic_http/src/lib/glommio/mod.rs index a88a8d4..531a300 100644 --- a/aquatic_http/src/lib/glommio/mod.rs +++ b/aquatic_http/src/lib/glommio/mod.rs @@ -5,6 +5,7 @@ use glommio::{channels::channel_mesh::MeshBuilder, prelude::*}; use crate::config::Config; +mod common; mod handlers; mod network; @@ -59,6 +60,34 @@ pub fn run( executors.push(executor); } + for i in 0..(config.request_workers) { + let config = config.clone(); + let request_mesh_builder = request_mesh_builder.clone(); + let response_mesh_builder = response_mesh_builder.clone(); + let access_list = access_list.clone(); + + let mut builder = LocalExecutorBuilder::default(); + + // if config.core_affinity.set_affinities { + // builder = + // builder.pin_to_cpu(config.core_affinity.offset + 1 + config.socket_workers + i); + // } + + let executor = builder.spawn(|| async move { + handlers::run_request_worker( + config, + request_mesh_builder, + response_mesh_builder, + access_list, + ) + .await + }); + + executors.push(executor); + } + + // drop_privileges_after_socket_binding(&config, num_bound_sockets).unwrap(); + for executor in executors { executor .expect("failed to spawn local executor") diff --git a/aquatic_http/src/lib/glommio/network.rs b/aquatic_http/src/lib/glommio/network.rs index 6b21c84..29f1e75 100644 --- a/aquatic_http/src/lib/glommio/network.rs +++ b/aquatic_http/src/lib/glommio/network.rs @@ -4,7 +4,8 @@ use std::rc::Rc; use std::sync::Arc; use std::sync::atomic::{AtomicUsize, Ordering}; -use aquatic_http_protocol::request::{Request, RequestParseError}; +use aquatic_http_protocol::common::InfoHash; +use aquatic_http_protocol::request::{AnnounceRequest, Request, RequestParseError}; use aquatic_http_protocol::response::{FailureResponse, Response}; use futures_lite::{AsyncReadExt, AsyncWriteExt, StreamExt}; use glommio::channels::channel_mesh::{MeshBuilder, Partial, Receivers, Role, Senders}; @@ -19,23 +20,24 @@ use slab::Slab; use crate::common::num_digits_in_usize; use crate::config::Config; +use super::common::*; + const BUFFER_SIZE: usize = 1024; -#[derive(Clone, Copy, Debug)] -pub struct ConnectionId(pub usize); struct ConnectionReference { - response_sender: LocalSender, + response_sender: LocalSender, handle: JoinHandle<()>, } struct Connection { config: Rc, - // request_senders: Rc>, - response_receiver: LocalReceiver, + request_senders: Rc>, + response_receiver: LocalReceiver, + response_consumer_id: ConsumerId, tls: ServerConnection, stream: TcpStream, - index: ConnectionId, + connection_id: ConnectionId, request_buffer: Vec, close_after_writing: bool, } @@ -43,8 +45,8 @@ struct Connection { pub async fn run_socket_worker( config: Config, tls_config: Arc, - request_mesh_builder: MeshBuilder<(ConnectionId, Request), Partial>, - response_mesh_builder: MeshBuilder<(ConnectionId, Response), Partial>, + request_mesh_builder: MeshBuilder, + response_mesh_builder: MeshBuilder, num_bound_sockets: Arc, ) { let config = Rc::new(config); @@ -52,16 +54,18 @@ pub async fn run_socket_worker( let listener = TcpListener::bind(config.network.address).expect("bind socket"); num_bound_sockets.fetch_add(1, Ordering::SeqCst); - // let (_, mut response_receivers) = response_mesh_builder.join(Role::Consumer).await.unwrap(); + let (_, mut response_receivers) = response_mesh_builder.join(Role::Consumer).await.unwrap(); - // let (request_senders, _) = request_mesh_builder.join(Role::Producer).await.unwrap(); - // let request_senders = Rc::new(request_senders); + let response_consumer_id = ConsumerId(response_receivers.consumer_id().unwrap()); + + let (request_senders, _) = request_mesh_builder.join(Role::Producer).await.unwrap(); + let request_senders = Rc::new(request_senders); let connection_slab = Rc::new(RefCell::new(Slab::new())); - // for (_, response_receiver) in response_receivers.streams() { - // spawn_local(receive_responses(response_receiver, connection_slab.clone())).detach(); - // } + for (_, response_receiver) in response_receivers.streams() { + spawn_local(receive_responses(response_receiver, connection_slab.clone())).detach(); + } let mut incoming = listener.incoming(); @@ -75,11 +79,12 @@ pub async fn run_socket_worker( let conn = Connection { config: config.clone(), - // request_senders: request_senders.clone(), + request_senders: request_senders.clone(), response_receiver, + response_consumer_id, tls: ServerConnection::new(tls_config.clone()).unwrap(), stream, - index: ConnectionId(entry.key()), + connection_id: ConnectionId(entry.key()), request_buffer: Vec::new(), close_after_writing: false, }; @@ -108,12 +113,12 @@ pub async fn run_socket_worker( } async fn receive_responses( - mut response_receiver: ConnectedReceiver<(ConnectionId, Response)>, + mut response_receiver: ConnectedReceiver, connection_references: Rc>>, ) { - while let Some((connection_id, response)) = response_receiver.next().await { - if let Some(reference) = connection_references.borrow().get(connection_id.0) { - reference.response_sender.try_send(response); + while let Some(channel_response) = response_receiver.next().await { + if let Some(reference) = connection_references.borrow().get(channel_response.get_connection_id().0) { + reference.response_sender.try_send(channel_response); } } } @@ -128,16 +133,44 @@ impl Connection { .peer_addr() .map_err(|err| anyhow::anyhow!("Couldn't get peer addr: {:?}", err))?; - // TODO: send request to channel + match request { + Request::Announce(request@AnnounceRequest { info_hash, .. }) => { + let request = ChannelRequest::Announce { + request, + connection_id: self.connection_id, + response_consumer_id: self.response_consumer_id, + peer_addr, + }; + + let consumer_index = calculate_request_consumer_index(&self.config, info_hash); + self.request_senders.try_send_to(consumer_index, request); + }, + Request::Scrape(request) => { + // TODO + }, + } // Wait for response to arrive, then send it - if let Some(response) = self.response_receiver.recv().await { - // TODO: compare IP addresses? + if let Some(channel_response) = self.response_receiver.recv().await { + if channel_response.get_peer_addr() != peer_addr { + return Err(anyhow::anyhow!("peer addressess didn't match")); + } + + let opt_response = match channel_response { + ChannelResponse::Announce { response, .. } => { + Some(Response::Announce(response)) + } + ChannelResponse::Scrape { response, original_indices, .. } => { + None // TODO: accumulate scrape requests + } + }; - self.queue_response(&response)?; + if let Some(response) = opt_response { + self.queue_response(&response)?; - if !self.config.network.keep_alive { - self.close_after_writing = true; + if !self.config.network.keep_alive { + self.close_after_writing = true; + } } } } @@ -274,3 +307,7 @@ impl Connection { Ok(()) } } + +fn calculate_request_consumer_index(config: &Config, info_hash: InfoHash) -> usize { + (info_hash.0[0] as usize) % config.request_workers +} \ No newline at end of file diff --git a/aquatic_http/src/lib/mio/network/mod.rs b/aquatic_http/src/lib/mio/network/mod.rs index 52e8b0d..7887f95 100644 --- a/aquatic_http/src/lib/mio/network/mod.rs +++ b/aquatic_http/src/lib/mio/network/mod.rs @@ -217,7 +217,7 @@ pub fn handle_connection_read_event( { let meta = ConnectionMeta { worker_index: socket_worker_index, - poll_token, + poll_token: poll_token.0, peer_addr: established.peer_addr, }; let response = FailureResponse::new("Info hash not allowed"); @@ -231,7 +231,7 @@ pub fn handle_connection_read_event( Ok(request) => { let meta = ConnectionMeta { worker_index: socket_worker_index, - poll_token, + poll_token: poll_token.0, peer_addr: established.peer_addr, }; @@ -254,7 +254,7 @@ pub fn handle_connection_read_event( let meta = ConnectionMeta { worker_index: socket_worker_index, - poll_token, + poll_token: poll_token.0, peer_addr: established.peer_addr, }; @@ -322,7 +322,7 @@ pub fn send_responses( for (meta, response) in local_responses.chain(channel_responses_drain) { if let Some(established) = connections - .get_mut(&meta.poll_token) + .get_mut(&Token(meta.poll_token)) .and_then(Connection::get_established) { if established.peer_addr != meta.peer_addr { @@ -344,7 +344,7 @@ pub fn send_responses( ); if !config.network.keep_alive { - remove_connection(poll, connections, &meta.poll_token); + remove_connection(poll, connections, &Token(meta.poll_token)); } } Err(err) if err.kind() == ErrorKind::WouldBlock => { @@ -353,7 +353,7 @@ pub fn send_responses( Err(err) => { info!("error sending response: {}", err); - remove_connection(poll, connections, &meta.poll_token); + remove_connection(poll, connections, &Token(meta.poll_token)); } } } From 17412868b9db1fc80e513be1b4270060f20941cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20Frosteg=C3=A5rd?= Date: Wed, 27 Oct 2021 00:47:46 +0200 Subject: [PATCH 17/59] Run cargo fmt --- aquatic_http/src/lib/common/handlers.rs | 5 +- aquatic_http/src/lib/common/mod.rs | 4 +- aquatic_http/src/lib/glommio/common.rs | 11 +++-- aquatic_http/src/lib/glommio/handlers.rs | 6 +-- aquatic_http/src/lib/glommio/mod.rs | 20 ++++---- aquatic_http/src/lib/glommio/network.rs | 60 +++++++++++++++--------- aquatic_http/src/lib/lib.rs | 6 +-- aquatic_http/src/lib/mio/common.rs | 4 +- aquatic_http/src/lib/mio/handler.rs | 14 ++---- aquatic_http/src/lib/mio/network/mod.rs | 4 +- aquatic_http/src/lib/mio/tasks.rs | 2 +- 11 files changed, 73 insertions(+), 63 deletions(-) diff --git a/aquatic_http/src/lib/common/handlers.rs b/aquatic_http/src/lib/common/handlers.rs index fa7fe3d..01f7bc0 100644 --- a/aquatic_http/src/lib/common/handlers.rs +++ b/aquatic_http/src/lib/common/handlers.rs @@ -2,14 +2,14 @@ use std::collections::BTreeMap; use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; use either::Either; -use rand::{Rng}; +use rand::Rng; use aquatic_common::extract_response_peers; use aquatic_http_protocol::request::*; use aquatic_http_protocol::response::*; -use crate::config::Config; use super::*; +use crate::config::Config; pub fn handle_announce_request( config: &Config, @@ -215,4 +215,3 @@ pub fn handle_scrape_request( response } - diff --git a/aquatic_http/src/lib/common/mod.rs b/aquatic_http/src/lib/common/mod.rs index 9ff51b1..b9c26b7 100644 --- a/aquatic_http/src/lib/common/mod.rs +++ b/aquatic_http/src/lib/common/mod.rs @@ -1,7 +1,7 @@ use std::net::{Ipv4Addr, Ipv6Addr, SocketAddr}; use std::time::Instant; -use aquatic_common::access_list::{AccessList}; +use aquatic_common::access_list::AccessList; use either::Either; use hashbrown::HashMap; use indexmap::IndexMap; @@ -10,7 +10,7 @@ use smartstring::{LazyCompact, SmartString}; pub use aquatic_common::{convert_ipv4_mapped_ipv6, ValidUntil}; use aquatic_http_protocol::common::*; -use aquatic_http_protocol::response::{ResponsePeer}; +use aquatic_http_protocol::response::ResponsePeer; use crate::config::Config; diff --git a/aquatic_http/src/lib/glommio/common.rs b/aquatic_http/src/lib/glommio/common.rs index 1752deb..0ca0bed 100644 --- a/aquatic_http/src/lib/glommio/common.rs +++ b/aquatic_http/src/lib/glommio/common.rs @@ -1,6 +1,9 @@ use std::net::SocketAddr; -use aquatic_http_protocol::{request::{AnnounceRequest, ScrapeRequest}, response::{AnnounceResponse, ScrapeResponse}}; +use aquatic_http_protocol::{ + request::{AnnounceRequest, ScrapeRequest}, + response::{AnnounceResponse, ScrapeResponse}, +}; #[derive(Copy, Clone, Debug)] pub struct ConsumerId(pub usize); @@ -22,7 +25,7 @@ pub enum ChannelRequest { original_indices: Vec, connection_id: ConnectionId, response_consumer_id: ConsumerId, - } + }, } #[derive(Debug)] @@ -37,7 +40,7 @@ pub enum ChannelResponse { peer_addr: SocketAddr, original_indices: Vec, connection_id: ConnectionId, - } + }, } impl ChannelResponse { @@ -53,4 +56,4 @@ impl ChannelResponse { Self::Scrape { peer_addr, .. } => *peer_addr, } } -} \ No newline at end of file +} diff --git a/aquatic_http/src/lib/glommio/handlers.rs b/aquatic_http/src/lib/glommio/handlers.rs index 6457318..e7abd65 100644 --- a/aquatic_http/src/lib/glommio/handlers.rs +++ b/aquatic_http/src/lib/glommio/handlers.rs @@ -88,7 +88,7 @@ async fn handle_request_stream( request, peer_addr, response_consumer_id, - connection_id + connection_id, } => { let meta = ConnectionMeta { worker_index: response_consumer_id.0, @@ -126,7 +126,8 @@ async fn handle_request_stream( peer_addr, }; - let response = handle_scrape_request(&config, &mut torrents.borrow_mut(), meta, request); + let response = + handle_scrape_request(&config, &mut torrents.borrow_mut(), meta, request); let response = ChannelResponse::Scrape { response, @@ -148,4 +149,3 @@ async fn handle_request_stream( yield_if_needed().await; } } - diff --git a/aquatic_http/src/lib/glommio/mod.rs b/aquatic_http/src/lib/glommio/mod.rs index 531a300..acb3e41 100644 --- a/aquatic_http/src/lib/glommio/mod.rs +++ b/aquatic_http/src/lib/glommio/mod.rs @@ -1,4 +1,8 @@ -use std::{fs::File, io::BufReader, sync::{Arc, atomic::AtomicUsize}}; +use std::{ + fs::File, + io::BufReader, + sync::{atomic::AtomicUsize, Arc}, +}; use aquatic_common::access_list::AccessList; use glommio::{channels::channel_mesh::MeshBuilder, prelude::*}; @@ -11,9 +15,7 @@ mod network; const SHARED_CHANNEL_SIZE: usize = 1024; -pub fn run( - config: Config, -) -> anyhow::Result<()> { +pub fn run(config: Config) -> anyhow::Result<()> { let access_list = if config.access_list.mode.is_on() { AccessList::create_from_path(&config.access_list.path).expect("Load access list") } else { @@ -94,13 +96,11 @@ pub fn run( .join() .unwrap(); } - + Ok(()) } -fn create_tls_config( - config: &Config, -) -> anyhow::Result { +fn create_tls_config(config: &Config) -> anyhow::Result { let certs = { let f = File::open(&config.network.tls.tls_certificate_path)?; let mut f = BufReader::new(f); @@ -125,6 +125,6 @@ fn create_tls_config( .with_safe_defaults() .with_no_client_auth() .with_single_cert(certs, private_key)?; - + Ok(tls_config) -} \ No newline at end of file +} diff --git a/aquatic_http/src/lib/glommio/network.rs b/aquatic_http/src/lib/glommio/network.rs index 29f1e75..b0097d6 100644 --- a/aquatic_http/src/lib/glommio/network.rs +++ b/aquatic_http/src/lib/glommio/network.rs @@ -1,20 +1,20 @@ use std::cell::RefCell; use std::io::{Cursor, ErrorKind, Read, Write}; use std::rc::Rc; -use std::sync::Arc; use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Arc; use aquatic_http_protocol::common::InfoHash; use aquatic_http_protocol::request::{AnnounceRequest, Request, RequestParseError}; use aquatic_http_protocol::response::{FailureResponse, Response}; use futures_lite::{AsyncReadExt, AsyncWriteExt, StreamExt}; use glommio::channels::channel_mesh::{MeshBuilder, Partial, Receivers, Role, Senders}; -use glommio::channels::shared_channel::{ConnectedReceiver, ConnectedSender, SharedSender}; -use glommio::prelude::*; -use glommio::net::{TcpListener, TcpStream}; use glommio::channels::local_channel::{new_bounded, LocalReceiver, LocalSender}; +use glommio::channels::shared_channel::{ConnectedReceiver, ConnectedSender, SharedSender}; +use glommio::net::{TcpListener, TcpStream}; +use glommio::prelude::*; use glommio::task::JoinHandle; -use rustls::{ServerConnection}; +use rustls::ServerConnection; use slab::Slab; use crate::common::num_digits_in_usize; @@ -24,7 +24,6 @@ use super::common::*; const BUFFER_SIZE: usize = 1024; - struct ConnectionReference { response_sender: LocalSender, handle: JoinHandle<()>, @@ -64,7 +63,11 @@ pub async fn run_socket_worker( let connection_slab = Rc::new(RefCell::new(Slab::new())); for (_, response_receiver) in response_receivers.streams() { - spawn_local(receive_responses(response_receiver, connection_slab.clone())).detach(); + spawn_local(receive_responses( + response_receiver, + connection_slab.clone(), + )) + .detach(); } let mut incoming = listener.incoming(); @@ -103,12 +106,11 @@ pub async fn run_socket_worker( }; entry.insert(connection_reference); - }, + } Err(err) => { ::log::error!("accept connection: {:?}", err); } } - } } @@ -117,7 +119,10 @@ async fn receive_responses( connection_references: Rc>>, ) { while let Some(channel_response) = response_receiver.next().await { - if let Some(reference) = connection_references.borrow().get(channel_response.get_connection_id().0) { + if let Some(reference) = connection_references + .borrow() + .get(channel_response.get_connection_id().0) + { reference.response_sender.try_send(channel_response); } } @@ -129,12 +134,13 @@ impl Connection { let opt_request = self.read_tls().await?; if let Some(request) = opt_request { - let peer_addr = self.stream + let peer_addr = self + .stream .peer_addr() .map_err(|err| anyhow::anyhow!("Couldn't get peer addr: {:?}", err))?; - + match request { - Request::Announce(request@AnnounceRequest { info_hash, .. }) => { + Request::Announce(request @ AnnounceRequest { info_hash, .. }) => { let request = ChannelRequest::Announce { request, connection_id: self.connection_id, @@ -142,12 +148,13 @@ impl Connection { peer_addr, }; - let consumer_index = calculate_request_consumer_index(&self.config, info_hash); + let consumer_index = + calculate_request_consumer_index(&self.config, info_hash); self.request_senders.try_send_to(consumer_index, request); - }, + } Request::Scrape(request) => { // TODO - }, + } } // Wait for response to arrive, then send it @@ -155,12 +162,16 @@ impl Connection { if channel_response.get_peer_addr() != peer_addr { return Err(anyhow::anyhow!("peer addressess didn't match")); } - + let opt_response = match channel_response { - ChannelResponse::Announce { response, .. } => { + ChannelResponse::Announce { response, .. } => { Some(Response::Announce(response)) } - ChannelResponse::Scrape { response, original_indices, .. } => { + ChannelResponse::Scrape { + response, + original_indices, + .. + } => { None // TODO: accumulate scrape requests } }; @@ -219,7 +230,7 @@ impl Connection { self.request_buffer.extend_from_slice(&buf[..amt]); added_plaintext = true; - }, + } Err(err) if err.kind() == ErrorKind::WouldBlock => { break; } @@ -241,7 +252,10 @@ impl Connection { return Ok(Some(request)); } Err(RequestParseError::NeedMoreData) => { - ::log::debug!("need more request data. current data: {:?}", std::str::from_utf8(&self.request_buffer)); + ::log::debug!( + "need more request data. current data: {:?}", + std::str::from_utf8(&self.request_buffer) + ); } Err(RequestParseError::Invalid(err)) => { ::log::debug!("invalid request: {:?}", err); @@ -259,7 +273,7 @@ impl Connection { } if self.tls.wants_write() { - break + break; } } @@ -310,4 +324,4 @@ impl Connection { fn calculate_request_consumer_index(config: &Config, info_hash: InfoHash) -> usize { (info_hash.0[0] as usize) % config.request_workers -} \ No newline at end of file +} diff --git a/aquatic_http/src/lib/lib.rs b/aquatic_http/src/lib/lib.rs index 7f1456f..10cb683 100644 --- a/aquatic_http/src/lib/lib.rs +++ b/aquatic_http/src/lib/lib.rs @@ -1,12 +1,12 @@ use cfg_if::cfg_if; -pub mod config; pub mod common; +pub mod config; -#[cfg(feature = "with-mio")] -pub mod mio; #[cfg(all(feature = "with-glommio", target_os = "linux"))] pub mod glommio; +#[cfg(feature = "with-mio")] +pub mod mio; pub const APP_NAME: &str = "aquatic_http: HTTP/TLS BitTorrent tracker"; diff --git a/aquatic_http/src/lib/mio/common.rs b/aquatic_http/src/lib/mio/common.rs index 5c68ca6..ef224d6 100644 --- a/aquatic_http/src/lib/mio/common.rs +++ b/aquatic_http/src/lib/mio/common.rs @@ -1,6 +1,6 @@ use std::sync::Arc; -use aquatic_common::access_list::{AccessListArcSwap}; +use aquatic_common::access_list::AccessListArcSwap; use crossbeam_channel::{Receiver, Sender}; use log::error; use mio::Token; @@ -9,7 +9,7 @@ use parking_lot::Mutex; pub use aquatic_common::{convert_ipv4_mapped_ipv6, ValidUntil}; use aquatic_http_protocol::request::Request; -use aquatic_http_protocol::response::{Response}; +use aquatic_http_protocol::response::Response; use crate::common::*; diff --git a/aquatic_http/src/lib/mio/handler.rs b/aquatic_http/src/lib/mio/handler.rs index c42acb1..fcdc4e9 100644 --- a/aquatic_http/src/lib/mio/handler.rs +++ b/aquatic_http/src/lib/mio/handler.rs @@ -8,10 +8,10 @@ use rand::{rngs::SmallRng, SeedableRng}; use aquatic_http_protocol::request::*; use aquatic_http_protocol::response::*; +use super::common::*; use crate::common::handlers::{handle_announce_request, handle_scrape_request}; use crate::common::*; use crate::config::Config; -use super::common::*; pub fn run_request_worker( config: Config, @@ -59,8 +59,7 @@ pub fn run_request_worker( } } - let mut torrent_maps = - opt_torrent_maps.unwrap_or_else(|| state.torrent_maps.lock()); + let mut torrent_maps = opt_torrent_maps.unwrap_or_else(|| state.torrent_maps.lock()); let valid_until = ValidUntil::new(config.cleaning.max_peer_age); @@ -71,7 +70,7 @@ pub fn run_request_worker( &mut torrent_maps, valid_until, meta, - request + request, ); response_channel_sender.send(meta, Response::Announce(response)); @@ -79,12 +78,7 @@ pub fn run_request_worker( } for (meta, request) in scrape_requests.drain(..) { - let response = handle_scrape_request( - &config, - &mut torrent_maps, - meta, - request - ); + let response = handle_scrape_request(&config, &mut torrent_maps, meta, request); response_channel_sender.send(meta, Response::Scrape(response)); wake_socket_workers[meta.worker_index] = true; diff --git a/aquatic_http/src/lib/mio/network/mod.rs b/aquatic_http/src/lib/mio/network/mod.rs index 7887f95..fa0c9bd 100644 --- a/aquatic_http/src/lib/mio/network/mod.rs +++ b/aquatic_http/src/lib/mio/network/mod.rs @@ -13,9 +13,9 @@ use native_tls::TlsAcceptor; use aquatic_http_protocol::response::*; -use crate::mio::common::*; -use crate::config::Config; use crate::common::*; +use crate::config::Config; +use crate::mio::common::*; pub mod connection; pub mod stream; diff --git a/aquatic_http/src/lib/mio/tasks.rs b/aquatic_http/src/lib/mio/tasks.rs index 67c9158..0106e4e 100644 --- a/aquatic_http/src/lib/mio/tasks.rs +++ b/aquatic_http/src/lib/mio/tasks.rs @@ -2,8 +2,8 @@ use histogram::Histogram; use aquatic_common::access_list::{AccessListMode, AccessListQuery}; -use crate::config::Config; use super::common::*; +use crate::config::Config; pub fn update_access_list(config: &Config, state: &State) { match config.access_list.mode { From d03bf48433276726b2250b871b3a6751168a1ae5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20Frosteg=C3=A5rd?= Date: Wed, 27 Oct 2021 01:03:31 +0200 Subject: [PATCH 18/59] aquatic_http glommio: various fixes to network code --- aquatic_http/src/lib/glommio/network.rs | 127 ++++++++++++++---------- 1 file changed, 73 insertions(+), 54 deletions(-) diff --git a/aquatic_http/src/lib/glommio/network.rs b/aquatic_http/src/lib/glommio/network.rs index b0097d6..552731e 100644 --- a/aquatic_http/src/lib/glommio/network.rs +++ b/aquatic_http/src/lib/glommio/network.rs @@ -1,5 +1,6 @@ use std::cell::RefCell; use std::io::{Cursor, ErrorKind, Read, Write}; +use std::net::SocketAddr; use std::rc::Rc; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Arc; @@ -8,9 +9,9 @@ use aquatic_http_protocol::common::InfoHash; use aquatic_http_protocol::request::{AnnounceRequest, Request, RequestParseError}; use aquatic_http_protocol::response::{FailureResponse, Response}; use futures_lite::{AsyncReadExt, AsyncWriteExt, StreamExt}; -use glommio::channels::channel_mesh::{MeshBuilder, Partial, Receivers, Role, Senders}; +use glommio::channels::channel_mesh::{MeshBuilder, Partial, Role, Senders}; use glommio::channels::local_channel::{new_bounded, LocalReceiver, LocalSender}; -use glommio::channels::shared_channel::{ConnectedReceiver, ConnectedSender, SharedSender}; +use glommio::channels::shared_channel::ConnectedReceiver; use glommio::net::{TcpListener, TcpStream}; use glommio::prelude::*; use glommio::task::JoinHandle; @@ -75,7 +76,7 @@ pub async fn run_socket_worker( while let Some(stream) = incoming.next().await { match stream { Ok(stream) => { - let (response_sender, response_receiver) = new_bounded(1); + let (response_sender, response_receiver) = new_bounded(config.request_workers); let mut slab = connection_slab.borrow_mut(); let entry = slab.vacant_entry(); @@ -123,7 +124,9 @@ async fn receive_responses( .borrow() .get(channel_response.get_connection_id().0) { - reference.response_sender.try_send(channel_response); + if let Err(err) = reference.response_sender.try_send(channel_response) { + ::log::error!("Couldn't send response to local receiver: {:?}", err); + } } } } @@ -134,56 +137,8 @@ impl Connection { let opt_request = self.read_tls().await?; if let Some(request) = opt_request { - let peer_addr = self - .stream - .peer_addr() - .map_err(|err| anyhow::anyhow!("Couldn't get peer addr: {:?}", err))?; - - match request { - Request::Announce(request @ AnnounceRequest { info_hash, .. }) => { - let request = ChannelRequest::Announce { - request, - connection_id: self.connection_id, - response_consumer_id: self.response_consumer_id, - peer_addr, - }; - - let consumer_index = - calculate_request_consumer_index(&self.config, info_hash); - self.request_senders.try_send_to(consumer_index, request); - } - Request::Scrape(request) => { - // TODO - } - } - - // Wait for response to arrive, then send it - if let Some(channel_response) = self.response_receiver.recv().await { - if channel_response.get_peer_addr() != peer_addr { - return Err(anyhow::anyhow!("peer addressess didn't match")); - } - - let opt_response = match channel_response { - ChannelResponse::Announce { response, .. } => { - Some(Response::Announce(response)) - } - ChannelResponse::Scrape { - response, - original_indices, - .. - } => { - None // TODO: accumulate scrape requests - } - }; - - if let Some(response) = opt_response { - self.queue_response(&response)?; - - if !self.config.network.keep_alive { - self.close_after_writing = true; - } - } - } + self.handle_request(request)?; + self.wait_for_and_send_response().await?; } self.write_tls().await?; @@ -300,6 +255,63 @@ impl Connection { Ok(()) } + /// Send on request to proper request worker/workers + fn handle_request(&mut self, request: Request) -> anyhow::Result<()> { + let peer_addr = self.get_peer_addr()?; + + match request { + Request::Announce(request @ AnnounceRequest { info_hash, .. }) => { + let request = ChannelRequest::Announce { + request, + connection_id: self.connection_id, + response_consumer_id: self.response_consumer_id, + peer_addr, + }; + + let consumer_index = + calculate_request_consumer_index(&self.config, info_hash); + self.request_senders.try_send_to(consumer_index, request); + } + Request::Scrape(request) => { + // TODO + } + } + + Ok(()) + } + + // Wait for response to arrive, then queue it for sending to peer + async fn wait_for_and_send_response(&mut self) -> anyhow::Result<()> { + if let Some(channel_response) = self.response_receiver.recv().await { + if channel_response.get_peer_addr() != self.get_peer_addr()? { + return Err(anyhow::anyhow!("peer addressess didn't match")); + } + + let opt_response = match channel_response { + ChannelResponse::Announce { response, .. } => { + Some(Response::Announce(response)) + } + ChannelResponse::Scrape { + response, + original_indices, + .. + } => { + None // TODO: accumulate scrape requests + } + }; + + if let Some(response) = opt_response { + self.queue_response(&response)?; + + if !self.config.network.keep_alive { + self.close_after_writing = true; + } + } + } + + Ok(()) + } + fn queue_response(&mut self, response: &Response) -> anyhow::Result<()> { let mut body = Vec::new(); @@ -320,6 +332,13 @@ impl Connection { Ok(()) } + + fn get_peer_addr(&self) -> anyhow::Result { + self + .stream + .peer_addr() + .map_err(|err| anyhow::anyhow!("Couldn't get peer addr: {:?}", err)) + } } fn calculate_request_consumer_index(config: &Config, info_hash: InfoHash) -> usize { From bbf781ddd6aa7d2b70a9954dbc4c7d28bcd8da03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20Frosteg=C3=A5rd?= Date: Wed, 27 Oct 2021 01:44:17 +0200 Subject: [PATCH 19/59] Update TODO --- TODO.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/TODO.md b/TODO.md index d7ceb85..e6b9423 100644 --- a/TODO.md +++ b/TODO.md @@ -1,6 +1,15 @@ # TODO +* aquatic_http glommio: + * cpu pinning - also rename config + * access lists + * scrape requests + * clean out connections regularly + * timeout inside of task for "it took to long to receive request, send response"? + * remove finished tasks from slab + * aquatic_udp glommio + * Check that it is the response consumer id responses are sent back to * Add to file transfer CI * consider adding ConnectedScrapeRequest::Scrape(PendingScrapeRequest) containing TransactionId and BTreeMap, and same for From 3e912bb379584b2435b0b7f77dcd08d2b8d5ef27 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20Frosteg=C3=A5rd?= Date: Wed, 27 Oct 2021 01:52:49 +0200 Subject: [PATCH 20/59] aquatic_http: glommio: fix channel join deadlock in network.rs --- aquatic_http/src/lib/glommio/network.rs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/aquatic_http/src/lib/glommio/network.rs b/aquatic_http/src/lib/glommio/network.rs index 552731e..9d8a264 100644 --- a/aquatic_http/src/lib/glommio/network.rs +++ b/aquatic_http/src/lib/glommio/network.rs @@ -54,13 +54,12 @@ pub async fn run_socket_worker( let listener = TcpListener::bind(config.network.address).expect("bind socket"); num_bound_sockets.fetch_add(1, Ordering::SeqCst); - let (_, mut response_receivers) = response_mesh_builder.join(Role::Consumer).await.unwrap(); - - let response_consumer_id = ConsumerId(response_receivers.consumer_id().unwrap()); - let (request_senders, _) = request_mesh_builder.join(Role::Producer).await.unwrap(); let request_senders = Rc::new(request_senders); + let (_, mut response_receivers) = response_mesh_builder.join(Role::Consumer).await.unwrap(); + let response_consumer_id = ConsumerId(response_receivers.consumer_id().unwrap()); + let connection_slab = Rc::new(RefCell::new(Slab::new())); for (_, response_receiver) in response_receivers.streams() { From 96931196243b810779bce72e92e0e2b4fccddce3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20Frosteg=C3=A5rd?= Date: Wed, 27 Oct 2021 02:02:16 +0200 Subject: [PATCH 21/59] Update TODO --- TODO.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/TODO.md b/TODO.md index e6b9423..498ced1 100644 --- a/TODO.md +++ b/TODO.md @@ -3,10 +3,16 @@ * aquatic_http glommio: * cpu pinning - also rename config * access lists + * privdrop * scrape requests * clean out connections regularly * timeout inside of task for "it took to long to receive request, send response"? * remove finished tasks from slab + * Don't return read request immediately. Set it as self.read_request + and continue looping to wait for any new input. Then check after + read_tls is finished. This might prevent issues when using plain HTTP + where only part of request is read, but that part is valid, and reading + is stopped, which might lead to various issues. * aquatic_udp glommio * Check that it is the response consumer id responses are sent back to From b7d61cecd94a61ef5b9cdbef7b0e6d4095c4b3ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20Frosteg=C3=A5rd?= Date: Wed, 27 Oct 2021 12:13:54 +0200 Subject: [PATCH 22/59] aquatic_http: glommio: implement scrape requests --- aquatic_http/src/lib/glommio/common.rs | 7 +- aquatic_http/src/lib/glommio/handlers.rs | 2 - aquatic_http/src/lib/glommio/network.rs | 113 +++++++++++++++++------ 3 files changed, 86 insertions(+), 36 deletions(-) diff --git a/aquatic_http/src/lib/glommio/common.rs b/aquatic_http/src/lib/glommio/common.rs index 0ca0bed..bfc8053 100644 --- a/aquatic_http/src/lib/glommio/common.rs +++ b/aquatic_http/src/lib/glommio/common.rs @@ -1,9 +1,6 @@ use std::net::SocketAddr; -use aquatic_http_protocol::{ - request::{AnnounceRequest, ScrapeRequest}, - response::{AnnounceResponse, ScrapeResponse}, -}; +use aquatic_http_protocol::{common::InfoHash, request::{AnnounceRequest, ScrapeRequest}, response::{AnnounceResponse, ScrapeResponse, ScrapeStatistics}}; #[derive(Copy, Clone, Debug)] pub struct ConsumerId(pub usize); @@ -22,7 +19,6 @@ pub enum ChannelRequest { Scrape { request: ScrapeRequest, peer_addr: SocketAddr, - original_indices: Vec, connection_id: ConnectionId, response_consumer_id: ConsumerId, }, @@ -38,7 +34,6 @@ pub enum ChannelResponse { Scrape { response: ScrapeResponse, peer_addr: SocketAddr, - original_indices: Vec, connection_id: ConnectionId, }, } diff --git a/aquatic_http/src/lib/glommio/handlers.rs b/aquatic_http/src/lib/glommio/handlers.rs index e7abd65..13ca6b2 100644 --- a/aquatic_http/src/lib/glommio/handlers.rs +++ b/aquatic_http/src/lib/glommio/handlers.rs @@ -118,7 +118,6 @@ async fn handle_request_stream( peer_addr, response_consumer_id, connection_id, - original_indices, } => { let meta = ConnectionMeta { worker_index: response_consumer_id.0, @@ -133,7 +132,6 @@ async fn handle_request_stream( response, peer_addr, connection_id, - original_indices, }; (response, response_consumer_id) diff --git a/aquatic_http/src/lib/glommio/network.rs b/aquatic_http/src/lib/glommio/network.rs index 9d8a264..7f46e84 100644 --- a/aquatic_http/src/lib/glommio/network.rs +++ b/aquatic_http/src/lib/glommio/network.rs @@ -1,4 +1,5 @@ use std::cell::RefCell; +use std::collections::BTreeMap; use std::io::{Cursor, ErrorKind, Read, Write}; use std::net::SocketAddr; use std::rc::Rc; @@ -6,8 +7,8 @@ use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Arc; use aquatic_http_protocol::common::InfoHash; -use aquatic_http_protocol::request::{AnnounceRequest, Request, RequestParseError}; -use aquatic_http_protocol::response::{FailureResponse, Response}; +use aquatic_http_protocol::request::{AnnounceRequest, Request, RequestParseError, ScrapeRequest}; +use aquatic_http_protocol::response::{FailureResponse, Response, ScrapeResponse, ScrapeStatistics}; use futures_lite::{AsyncReadExt, AsyncWriteExt, StreamExt}; use glommio::channels::channel_mesh::{MeshBuilder, Partial, Role, Senders}; use glommio::channels::local_channel::{new_bounded, LocalReceiver, LocalSender}; @@ -25,6 +26,11 @@ use super::common::*; const BUFFER_SIZE: usize = 1024; +struct PendingScrapeResponse { + pending_worker_responses: usize, + stats: BTreeMap, +} + struct ConnectionReference { response_sender: LocalSender, handle: JoinHandle<()>, @@ -40,6 +46,7 @@ struct Connection { connection_id: ConnectionId, request_buffer: Vec, close_after_writing: bool, + pending_scrape_response: Option, } pub async fn run_socket_worker( @@ -90,6 +97,7 @@ pub async fn run_socket_worker( connection_id: ConnectionId(entry.key()), request_buffer: Vec::new(), close_after_writing: false, + pending_scrape_response: None, }; async fn handle_stream(mut conn: Connection) { @@ -269,43 +277,92 @@ impl Connection { let consumer_index = calculate_request_consumer_index(&self.config, info_hash); - self.request_senders.try_send_to(consumer_index, request); + + if let Err(err) = self.request_senders.try_send_to( + consumer_index, + request, + ) { + ::log::warn!("request_sender.try_send failed: {:?}", err); + } } - Request::Scrape(request) => { - // TODO + Request::Scrape(ScrapeRequest { info_hashes }) => { + let mut info_hashes_by_worker: BTreeMap> = BTreeMap::new(); + + for info_hash in info_hashes.into_iter() { + let info_hashes = info_hashes_by_worker + .entry(calculate_request_consumer_index(&self.config, info_hash)) + .or_default(); + + info_hashes.push(info_hash); + } + + self.pending_scrape_response = Some(PendingScrapeResponse { + pending_worker_responses: info_hashes_by_worker.len(), + stats: Default::default(), + }); + + for (consumer_index, info_hashes) in info_hashes_by_worker { + let request = ChannelRequest::Scrape { + request: ScrapeRequest { info_hashes }, + peer_addr, + response_consumer_id: self.response_consumer_id, + connection_id: self.connection_id, + }; + + if let Err(err) = self.request_senders.try_send_to( + consumer_index, + request, + ) { + ::log::warn!("request_sender.try_send failed: {:?}", err); + } + } } } Ok(()) } - // Wait for response to arrive, then queue it for sending to peer + // Wait for response/responses to arrive, then queue response for sending to peer async fn wait_for_and_send_response(&mut self) -> anyhow::Result<()> { - if let Some(channel_response) = self.response_receiver.recv().await { - if channel_response.get_peer_addr() != self.get_peer_addr()? { - return Err(anyhow::anyhow!("peer addressess didn't match")); + let response = loop { + if let Some(channel_response) = self.response_receiver.recv().await { + if channel_response.get_peer_addr() != self.get_peer_addr()? { + return Err(anyhow::anyhow!("peer addressess didn't match")); + } + + match channel_response { + ChannelResponse::Announce { response, .. } => { + break Response::Announce(response); + } + ChannelResponse::Scrape { response, .. } => { + if let Some(mut pending) = self.pending_scrape_response.take() { + pending.stats.extend(response.files); + pending.pending_worker_responses -= 1; + + if pending.pending_worker_responses == 0 { + let response = Response::Scrape(ScrapeResponse { + files: pending.stats, + }); + + break response; + } else { + self.pending_scrape_response = Some(pending); + } + } else { + return Err(anyhow::anyhow!("received channel scrape response without pending scrape response")); + } + } + }; + } else { + // TODO: this is a serious error condition and should maybe be handled differently + return Err(anyhow::anyhow!("response receiver can't receive - sender is closed")); } + }; - let opt_response = match channel_response { - ChannelResponse::Announce { response, .. } => { - Some(Response::Announce(response)) - } - ChannelResponse::Scrape { - response, - original_indices, - .. - } => { - None // TODO: accumulate scrape requests - } - }; + self.queue_response(&response)?; - if let Some(response) = opt_response { - self.queue_response(&response)?; - - if !self.config.network.keep_alive { - self.close_after_writing = true; - } - } + if !self.config.network.keep_alive { + self.close_after_writing = true; } Ok(()) From c02f8f228eef441e6531c695eb90775a8fe958a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20Frosteg=C3=A5rd?= Date: Wed, 27 Oct 2021 12:14:14 +0200 Subject: [PATCH 23/59] aquatic_http_protocol: fail scrape parse with no info hashes --- aquatic_http_protocol/src/request.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/aquatic_http_protocol/src/request.rs b/aquatic_http_protocol/src/request.rs index 95f0fdd..9b8e2ee 100644 --- a/aquatic_http_protocol/src/request.rs +++ b/aquatic_http_protocol/src/request.rs @@ -238,6 +238,10 @@ impl Request { Ok(Request::Announce(request)) } else { + if info_hashes.is_empty() { + return Err(anyhow::anyhow!("No info hashes sent")); + } + let request = ScrapeRequest { info_hashes }; Ok(Request::Scrape(request)) From e7305114ad03352e07ef1d9f1d726f5b76cb6937 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20Frosteg=C3=A5rd?= Date: Wed, 27 Oct 2021 12:22:01 +0200 Subject: [PATCH 24/59] aquatic_http: glommio: panic if request receiver channel is closed --- aquatic_http/src/lib/glommio/network.rs | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/aquatic_http/src/lib/glommio/network.rs b/aquatic_http/src/lib/glommio/network.rs index 7f46e84..816852a 100644 --- a/aquatic_http/src/lib/glommio/network.rs +++ b/aquatic_http/src/lib/glommio/network.rs @@ -144,7 +144,7 @@ impl Connection { let opt_request = self.read_tls().await?; if let Some(request) = opt_request { - self.handle_request(request)?; + self.handle_request(request).await?; self.wait_for_and_send_response().await?; } @@ -263,7 +263,7 @@ impl Connection { } /// Send on request to proper request worker/workers - fn handle_request(&mut self, request: Request) -> anyhow::Result<()> { + async fn handle_request(&mut self, request: Request) -> anyhow::Result<()> { let peer_addr = self.get_peer_addr()?; match request { @@ -278,12 +278,8 @@ impl Connection { let consumer_index = calculate_request_consumer_index(&self.config, info_hash); - if let Err(err) = self.request_senders.try_send_to( - consumer_index, - request, - ) { - ::log::warn!("request_sender.try_send failed: {:?}", err); - } + // Only fails when receiver is closed + self.request_senders.send_to(consumer_index, request).await.unwrap(); } Request::Scrape(ScrapeRequest { info_hashes }) => { let mut info_hashes_by_worker: BTreeMap> = BTreeMap::new(); @@ -309,12 +305,8 @@ impl Connection { connection_id: self.connection_id, }; - if let Err(err) = self.request_senders.try_send_to( - consumer_index, - request, - ) { - ::log::warn!("request_sender.try_send failed: {:?}", err); - } + // Only fails when receiver is closed + self.request_senders.send_to(consumer_index, request).await.unwrap(); } } } From 92fc427665c6b0729ee6e49181a6f33cbc97cf57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20Frosteg=C3=A5rd?= Date: Wed, 27 Oct 2021 12:23:56 +0200 Subject: [PATCH 25/59] Update TODO --- TODO.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/TODO.md b/TODO.md index 498ced1..c4cf0fc 100644 --- a/TODO.md +++ b/TODO.md @@ -4,10 +4,14 @@ * cpu pinning - also rename config * access lists * privdrop - * scrape requests * clean out connections regularly * timeout inside of task for "it took to long to receive request, send response"? * remove finished tasks from slab + * test with load tester with multiple workers + * consider better error type for request parsing, so that better error + messages can be sent back (e.g., "full scrapes are not supported") + * Scrape: should stats with only zeroes be sent back for non-registered info hashes? + Relevant for mio implementation too. * Don't return read request immediately. Set it as self.read_request and continue looping to wait for any new input. Then check after read_tls is finished. This might prevent issues when using plain HTTP @@ -15,7 +19,6 @@ is stopped, which might lead to various issues. * aquatic_udp glommio - * Check that it is the response consumer id responses are sent back to * Add to file transfer CI * consider adding ConnectedScrapeRequest::Scrape(PendingScrapeRequest) containing TransactionId and BTreeMap, and same for From c9233726ab14d58016fba57f4cb700a0296c5d49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20Frosteg=C3=A5rd?= Date: Wed, 27 Oct 2021 12:24:27 +0200 Subject: [PATCH 26/59] Run cargo fmt --- aquatic_http/src/lib/glommio/common.rs | 5 ++++- aquatic_http/src/lib/glommio/network.rs | 30 ++++++++++++++++--------- 2 files changed, 24 insertions(+), 11 deletions(-) diff --git a/aquatic_http/src/lib/glommio/common.rs b/aquatic_http/src/lib/glommio/common.rs index bfc8053..11b2036 100644 --- a/aquatic_http/src/lib/glommio/common.rs +++ b/aquatic_http/src/lib/glommio/common.rs @@ -1,6 +1,9 @@ use std::net::SocketAddr; -use aquatic_http_protocol::{common::InfoHash, request::{AnnounceRequest, ScrapeRequest}, response::{AnnounceResponse, ScrapeResponse, ScrapeStatistics}}; +use aquatic_http_protocol::{ + request::{AnnounceRequest, ScrapeRequest}, + response::{AnnounceResponse, ScrapeResponse}, +}; #[derive(Copy, Clone, Debug)] pub struct ConsumerId(pub usize); diff --git a/aquatic_http/src/lib/glommio/network.rs b/aquatic_http/src/lib/glommio/network.rs index 816852a..2788bc0 100644 --- a/aquatic_http/src/lib/glommio/network.rs +++ b/aquatic_http/src/lib/glommio/network.rs @@ -8,7 +8,9 @@ use std::sync::Arc; use aquatic_http_protocol::common::InfoHash; use aquatic_http_protocol::request::{AnnounceRequest, Request, RequestParseError, ScrapeRequest}; -use aquatic_http_protocol::response::{FailureResponse, Response, ScrapeResponse, ScrapeStatistics}; +use aquatic_http_protocol::response::{ + FailureResponse, Response, ScrapeResponse, ScrapeStatistics, +}; use futures_lite::{AsyncReadExt, AsyncWriteExt, StreamExt}; use glommio::channels::channel_mesh::{MeshBuilder, Partial, Role, Senders}; use glommio::channels::local_channel::{new_bounded, LocalReceiver, LocalSender}; @@ -275,11 +277,13 @@ impl Connection { peer_addr, }; - let consumer_index = - calculate_request_consumer_index(&self.config, info_hash); + let consumer_index = calculate_request_consumer_index(&self.config, info_hash); // Only fails when receiver is closed - self.request_senders.send_to(consumer_index, request).await.unwrap(); + self.request_senders + .send_to(consumer_index, request) + .await + .unwrap(); } Request::Scrape(ScrapeRequest { info_hashes }) => { let mut info_hashes_by_worker: BTreeMap> = BTreeMap::new(); @@ -306,7 +310,10 @@ impl Connection { }; // Only fails when receiver is closed - self.request_senders.send_to(consumer_index, request).await.unwrap(); + self.request_senders + .send_to(consumer_index, request) + .await + .unwrap(); } } } @@ -326,7 +333,7 @@ impl Connection { ChannelResponse::Announce { response, .. } => { break Response::Announce(response); } - ChannelResponse::Scrape { response, .. } => { + ChannelResponse::Scrape { response, .. } => { if let Some(mut pending) = self.pending_scrape_response.take() { pending.stats.extend(response.files); pending.pending_worker_responses -= 1; @@ -341,13 +348,17 @@ impl Connection { self.pending_scrape_response = Some(pending); } } else { - return Err(anyhow::anyhow!("received channel scrape response without pending scrape response")); + return Err(anyhow::anyhow!( + "received channel scrape response without pending scrape response" + )); } } }; } else { // TODO: this is a serious error condition and should maybe be handled differently - return Err(anyhow::anyhow!("response receiver can't receive - sender is closed")); + return Err(anyhow::anyhow!( + "response receiver can't receive - sender is closed" + )); } }; @@ -382,8 +393,7 @@ impl Connection { } fn get_peer_addr(&self) -> anyhow::Result { - self - .stream + self.stream .peer_addr() .map_err(|err| anyhow::anyhow!("Couldn't get peer addr: {:?}", err)) } From 9bea7c08980930a160489404ba19b2d01ca6d210 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20Frosteg=C3=A5rd?= Date: Wed, 27 Oct 2021 12:30:44 +0200 Subject: [PATCH 27/59] aquatic_http: glommio: deallocate request buffer after successful parse --- aquatic_http/src/lib/glommio/network.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/aquatic_http/src/lib/glommio/network.rs b/aquatic_http/src/lib/glommio/network.rs index 2788bc0..117e3d3 100644 --- a/aquatic_http/src/lib/glommio/network.rs +++ b/aquatic_http/src/lib/glommio/network.rs @@ -213,6 +213,8 @@ impl Connection { Ok(request) => { ::log::debug!("received request: {:?}", request); + self.request_buffer = Vec::new(); + return Ok(Some(request)); } Err(RequestParseError::NeedMoreData) => { From 00e77e8655aae9fd205ab23bb3b98cf29627bc6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20Frosteg=C3=A5rd?= Date: Wed, 27 Oct 2021 12:35:03 +0200 Subject: [PATCH 28/59] Update TODO --- TODO.md | 1 + 1 file changed, 1 insertion(+) diff --git a/TODO.md b/TODO.md index c4cf0fc..e363f42 100644 --- a/TODO.md +++ b/TODO.md @@ -8,6 +8,7 @@ * timeout inside of task for "it took to long to receive request, send response"? * remove finished tasks from slab * test with load tester with multiple workers + * get rid of / improve ConnectionMeta stuff in handler * consider better error type for request parsing, so that better error messages can be sent back (e.g., "full scrapes are not supported") * Scrape: should stats with only zeroes be sent back for non-registered info hashes? From 1a4e5750a330b9959596512b5f02576b92e1f3bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20Frosteg=C3=A5rd?= Date: Wed, 27 Oct 2021 13:19:02 +0200 Subject: [PATCH 29/59] aquatic_http: glommio: periodically remove closed connections --- TODO.md | 2 +- aquatic_http/src/lib/glommio/network.rs | 36 +++++++++++++++++++++---- 2 files changed, 32 insertions(+), 6 deletions(-) diff --git a/TODO.md b/TODO.md index e363f42..6ceb42a 100644 --- a/TODO.md +++ b/TODO.md @@ -6,7 +6,7 @@ * privdrop * clean out connections regularly * timeout inside of task for "it took to long to receive request, send response"? - * remove finished tasks from slab + * handle panicked/cancelled tasks * test with load tester with multiple workers * get rid of / improve ConnectionMeta stuff in handler * consider better error type for request parsing, so that better error diff --git a/aquatic_http/src/lib/glommio/network.rs b/aquatic_http/src/lib/glommio/network.rs index 117e3d3..9ca5984 100644 --- a/aquatic_http/src/lib/glommio/network.rs +++ b/aquatic_http/src/lib/glommio/network.rs @@ -5,6 +5,7 @@ use std::net::SocketAddr; use std::rc::Rc; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Arc; +use std::time::Duration; use aquatic_http_protocol::common::InfoHash; use aquatic_http_protocol::request::{AnnounceRequest, Request, RequestParseError, ScrapeRequest}; @@ -16,8 +17,9 @@ use glommio::channels::channel_mesh::{MeshBuilder, Partial, Role, Senders}; use glommio::channels::local_channel::{new_bounded, LocalReceiver, LocalSender}; use glommio::channels::shared_channel::ConnectedReceiver; use glommio::net::{TcpListener, TcpStream}; -use glommio::prelude::*; +use glommio::{enclose, prelude::*}; use glommio::task::JoinHandle; +use glommio::timer::TimerActionRepeat; use rustls::ServerConnection; use slab::Slab; @@ -70,6 +72,27 @@ pub async fn run_socket_worker( let response_consumer_id = ConsumerId(response_receivers.consumer_id().unwrap()); let connection_slab = Rc::new(RefCell::new(Slab::new())); + let connections_to_remove = Rc::new(RefCell::new(Vec::new())); + + // Periodically remove closed connections + TimerActionRepeat::repeat(enclose!((config, connection_slab, connections_to_remove) move || { + enclose!((config, connection_slab, connections_to_remove) move || async move { + let connections_to_remove = connections_to_remove.replace(Vec::new()); + + for connection_id in connections_to_remove { + if let Some(_) = connection_slab.borrow_mut().try_remove(connection_id) { + ::log::debug!("removed connection with id {}", connection_id); + } else { + ::log::error!( + "couldn't remove connection with id {}, it is not in connection slab", + connection_id + ); + } + } + + Some(Duration::from_secs(config.cleaning.interval)) + })() + })); for (_, response_receiver) in response_receivers.streams() { spawn_local(receive_responses( @@ -88,8 +111,9 @@ pub async fn run_socket_worker( let mut slab = connection_slab.borrow_mut(); let entry = slab.vacant_entry(); + let key = entry.key(); - let conn = Connection { + let mut conn = Connection { config: config.clone(), request_senders: request_senders.clone(), response_receiver, @@ -102,13 +126,15 @@ pub async fn run_socket_worker( pending_scrape_response: None, }; - async fn handle_stream(mut conn: Connection) { + let connections_to_remove = connections_to_remove.clone(); + + let handle = spawn_local(async move { if let Err(err) = conn.handle_stream().await { ::log::info!("conn.handle_stream() error: {:?}", err); } - } - let handle = spawn_local(handle_stream(conn)).detach(); + connections_to_remove.borrow_mut().push(key); + }).detach(); let connection_reference = ConnectionReference { response_sender, From 6404be8ef8ac0c88012e800183d945179b0fe8ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20Frosteg=C3=A5rd?= Date: Wed, 27 Oct 2021 13:55:17 +0200 Subject: [PATCH 30/59] aquatic_http: glommio: add access list, refactor networking slightly --- aquatic_http/src/lib/glommio/mod.rs | 2 +- aquatic_http/src/lib/glommio/network.rs | 112 +++++++++++++++--------- 2 files changed, 74 insertions(+), 40 deletions(-) diff --git a/aquatic_http/src/lib/glommio/mod.rs b/aquatic_http/src/lib/glommio/mod.rs index acb3e41..c93fbc5 100644 --- a/aquatic_http/src/lib/glommio/mod.rs +++ b/aquatic_http/src/lib/glommio/mod.rs @@ -54,7 +54,7 @@ pub fn run(config: Config) -> anyhow::Result<()> { request_mesh_builder, response_mesh_builder, num_bound_sockets, - // access_list, + access_list, ) .await }); diff --git a/aquatic_http/src/lib/glommio/network.rs b/aquatic_http/src/lib/glommio/network.rs index 9ca5984..a7a2b37 100644 --- a/aquatic_http/src/lib/glommio/network.rs +++ b/aquatic_http/src/lib/glommio/network.rs @@ -7,11 +7,13 @@ use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Arc; use std::time::Duration; +use aquatic_common::access_list::AccessList; use aquatic_http_protocol::common::InfoHash; use aquatic_http_protocol::request::{AnnounceRequest, Request, RequestParseError, ScrapeRequest}; use aquatic_http_protocol::response::{ FailureResponse, Response, ScrapeResponse, ScrapeStatistics, }; +use either::Either; use futures_lite::{AsyncReadExt, AsyncWriteExt, StreamExt}; use glommio::channels::channel_mesh::{MeshBuilder, Partial, Role, Senders}; use glommio::channels::local_channel::{new_bounded, LocalReceiver, LocalSender}; @@ -42,6 +44,7 @@ struct ConnectionReference { struct Connection { config: Rc, + access_list: Rc>, request_senders: Rc>, response_receiver: LocalReceiver, response_consumer_id: ConsumerId, @@ -50,7 +53,6 @@ struct Connection { connection_id: ConnectionId, request_buffer: Vec, close_after_writing: bool, - pending_scrape_response: Option, } pub async fn run_socket_worker( @@ -59,8 +61,10 @@ pub async fn run_socket_worker( request_mesh_builder: MeshBuilder, response_mesh_builder: MeshBuilder, num_bound_sockets: Arc, + access_list: AccessList, ) { let config = Rc::new(config); + let access_list = Rc::new(RefCell::new(access_list)); let listener = TcpListener::bind(config.network.address).expect("bind socket"); num_bound_sockets.fetch_add(1, Ordering::SeqCst); @@ -115,6 +119,7 @@ pub async fn run_socket_worker( let mut conn = Connection { config: config.clone(), + access_list: access_list.clone(), request_senders: request_senders.clone(), response_receiver, response_consumer_id, @@ -123,7 +128,6 @@ pub async fn run_socket_worker( connection_id: ConnectionId(entry.key()), request_buffer: Vec::new(), close_after_writing: false, - pending_scrape_response: None, }; let connections_to_remove = connections_to_remove.clone(); @@ -172,8 +176,23 @@ impl Connection { let opt_request = self.read_tls().await?; if let Some(request) = opt_request { - self.handle_request(request).await?; - self.wait_for_and_send_response().await?; + let response = match self.handle_request(request).await? { + Some(Either::Left(response)) => { + response + } + Some(Either::Right(pending_scrape_response)) => { + self.wait_for_response(Some(pending_scrape_response)).await? + }, + None => { + self.wait_for_response(None).await? + } + }; + + self.queue_response(&response)?; + + if !self.config.network.keep_alive { + self.close_after_writing = true; + } } self.write_tls().await?; @@ -292,26 +311,43 @@ impl Connection { Ok(()) } - /// Send on request to proper request worker/workers - async fn handle_request(&mut self, request: Request) -> anyhow::Result<()> { + /// Take a request and: + /// - Return error response if request is not allowed + /// - If it is an announce requests, pass it on to request workers and return None + /// - If it is a scrape requests, split it up and pass on parts to + /// relevant request workers, and return PendingScrapeResponse struct. + async fn handle_request( + &self, + request: Request + ) -> anyhow::Result>> { let peer_addr = self.get_peer_addr()?; match request { Request::Announce(request @ AnnounceRequest { info_hash, .. }) => { - let request = ChannelRequest::Announce { - request, - connection_id: self.connection_id, - response_consumer_id: self.response_consumer_id, - peer_addr, - }; + if self.access_list.borrow().allows(self.config.access_list.mode, &info_hash.0) { + let request = ChannelRequest::Announce { + request, + connection_id: self.connection_id, + response_consumer_id: self.response_consumer_id, + peer_addr, + }; - let consumer_index = calculate_request_consumer_index(&self.config, info_hash); + let consumer_index = calculate_request_consumer_index(&self.config, info_hash); - // Only fails when receiver is closed - self.request_senders - .send_to(consumer_index, request) - .await - .unwrap(); + // Only fails when receiver is closed + self.request_senders + .send_to(consumer_index, request) + .await + .unwrap(); + + Ok(None) + } else { + let response = Response::Failure(FailureResponse { + failure_reason: "Info hash not allowed".into(), + }); + + Ok(Some(Either::Left(response))) + } } Request::Scrape(ScrapeRequest { info_hashes }) => { let mut info_hashes_by_worker: BTreeMap> = BTreeMap::new(); @@ -324,10 +360,7 @@ impl Connection { info_hashes.push(info_hash); } - self.pending_scrape_response = Some(PendingScrapeResponse { - pending_worker_responses: info_hashes_by_worker.len(), - stats: Default::default(), - }); + let pending_worker_responses = info_hashes_by_worker.len(); for (consumer_index, info_hashes) in info_hashes_by_worker { let request = ChannelRequest::Scrape { @@ -343,15 +376,24 @@ impl Connection { .await .unwrap(); } + + let pending_scrape_response = PendingScrapeResponse { + pending_worker_responses, + stats: Default::default(), + }; + + Ok(Some(Either::Right(pending_scrape_response))) } } - - Ok(()) } - // Wait for response/responses to arrive, then queue response for sending to peer - async fn wait_for_and_send_response(&mut self) -> anyhow::Result<()> { - let response = loop { + /// Wait for announce response or partial scrape responses to arrive, + /// return full response + async fn wait_for_response( + &self, + mut opt_pending_scrape_response: Option + ) -> anyhow::Result { + loop { if let Some(channel_response) = self.response_receiver.recv().await { if channel_response.get_peer_addr() != self.get_peer_addr()? { return Err(anyhow::anyhow!("peer addressess didn't match")); @@ -359,10 +401,10 @@ impl Connection { match channel_response { ChannelResponse::Announce { response, .. } => { - break Response::Announce(response); + break Ok(Response::Announce(response)); } ChannelResponse::Scrape { response, .. } => { - if let Some(mut pending) = self.pending_scrape_response.take() { + if let Some(mut pending) = opt_pending_scrape_response.take() { pending.stats.extend(response.files); pending.pending_worker_responses -= 1; @@ -371,9 +413,9 @@ impl Connection { files: pending.stats, }); - break response; + break Ok(response); } else { - self.pending_scrape_response = Some(pending); + opt_pending_scrape_response = Some(pending); } } else { return Err(anyhow::anyhow!( @@ -388,15 +430,7 @@ impl Connection { "response receiver can't receive - sender is closed" )); } - }; - - self.queue_response(&response)?; - - if !self.config.network.keep_alive { - self.close_after_writing = true; } - - Ok(()) } fn queue_response(&mut self, response: &Response) -> anyhow::Result<()> { From dd968821e4f0272902c0a6f73cfa3d1ce2712b80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20Frosteg=C3=A5rd?= Date: Wed, 27 Oct 2021 14:07:49 +0200 Subject: [PATCH 31/59] aquatic_udp: glommio: refactor networking for less state in Connection --- aquatic_http/src/lib/glommio/network.rs | 65 +++++++++++++------------ 1 file changed, 33 insertions(+), 32 deletions(-) diff --git a/aquatic_http/src/lib/glommio/network.rs b/aquatic_http/src/lib/glommio/network.rs index a7a2b37..041dbb2 100644 --- a/aquatic_http/src/lib/glommio/network.rs +++ b/aquatic_http/src/lib/glommio/network.rs @@ -52,7 +52,6 @@ struct Connection { stream: TcpStream, connection_id: ConnectionId, request_buffer: Vec, - close_after_writing: bool, } pub async fn run_socket_worker( @@ -127,7 +126,6 @@ pub async fn run_socket_worker( stream, connection_id: ConnectionId(entry.key()), request_buffer: Vec::new(), - close_after_writing: false, }; let connections_to_remove = connections_to_remove.clone(); @@ -172,32 +170,42 @@ async fn receive_responses( impl Connection { async fn handle_stream(&mut self) -> anyhow::Result<()> { + let mut close_after_writing = false; + loop { - let opt_request = self.read_tls().await?; + match self.read_tls().await? { + Some(Either::Left(request)) => { + let response = match self.handle_request(request).await? { + Some(Either::Left(response)) => { + response + } + Some(Either::Right(pending_scrape_response)) => { + self.wait_for_response(Some(pending_scrape_response)).await? + }, + None => { + self.wait_for_response(None).await? + } + }; - if let Some(request) = opt_request { - let response = match self.handle_request(request).await? { - Some(Either::Left(response)) => { - response + self.queue_response(&response)?; + + if !self.config.network.keep_alive { + close_after_writing = true; } - Some(Either::Right(pending_scrape_response)) => { - self.wait_for_response(Some(pending_scrape_response)).await? - }, - None => { - self.wait_for_response(None).await? - } - }; + } + Some(Either::Right(response)) => { + self.queue_response(&Response::Failure(response))?; - self.queue_response(&response)?; - - if !self.config.network.keep_alive { - self.close_after_writing = true; + close_after_writing = true; + } + None => { + // Still handshaking } } self.write_tls().await?; - if self.close_after_writing { + if close_after_writing { let _ = self.stream.shutdown(std::net::Shutdown::Both).await; break; @@ -207,7 +215,7 @@ impl Connection { Ok(()) } - async fn read_tls(&mut self) -> anyhow::Result> { + async fn read_tls(&mut self) -> anyhow::Result>> { loop { ::log::debug!("read_tls"); @@ -216,11 +224,7 @@ impl Connection { let bytes_read = self.stream.read(&mut buf).await?; if bytes_read == 0 { - ::log::debug!("peer has closed connection"); - - self.close_after_writing = true; - - break; + return Err(anyhow::anyhow!("Peer has closed connection")); } let _ = self.tls.read_tls(&mut &buf[..bytes_read]).unwrap(); @@ -260,7 +264,7 @@ impl Connection { self.request_buffer = Vec::new(); - return Ok(Some(request)); + return Ok(Some(Either::Left(request))); } Err(RequestParseError::NeedMoreData) => { ::log::debug!( @@ -271,14 +275,11 @@ impl Connection { Err(RequestParseError::Invalid(err)) => { ::log::debug!("invalid request: {:?}", err); - let response = Response::Failure(FailureResponse { + let response = FailureResponse { failure_reason: "Invalid request".into(), - }); + }; - self.queue_response(&response)?; - self.close_after_writing = true; - - break; + return Ok(Some(Either::Right(response))); } } } From 8747f8de4e8d4e1b2547f7d259a88df496aa9141 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20Frosteg=C3=A5rd?= Date: Wed, 27 Oct 2021 14:28:15 +0200 Subject: [PATCH 32/59] aquatic_udp: glommio: limit request size, use array as buffer --- aquatic_http/src/lib/glommio/network.rs | 29 ++++++++++++++++++------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/aquatic_http/src/lib/glommio/network.rs b/aquatic_http/src/lib/glommio/network.rs index 041dbb2..31b613b 100644 --- a/aquatic_http/src/lib/glommio/network.rs +++ b/aquatic_http/src/lib/glommio/network.rs @@ -30,7 +30,8 @@ use crate::config::Config; use super::common::*; -const BUFFER_SIZE: usize = 1024; +const INTERMEDIATE_BUFFER_SIZE: usize = 1024; +const MAX_REQUEST_SIZE: usize = 2048; struct PendingScrapeResponse { pending_worker_responses: usize, @@ -51,7 +52,8 @@ struct Connection { tls: ServerConnection, stream: TcpStream, connection_id: ConnectionId, - request_buffer: Vec, + request_buffer: [u8; MAX_REQUEST_SIZE], + request_buffer_position: usize, } pub async fn run_socket_worker( @@ -125,7 +127,8 @@ pub async fn run_socket_worker( tls: ServerConnection::new(tls_config.clone()).unwrap(), stream, connection_id: ConnectionId(entry.key()), - request_buffer: Vec::new(), + request_buffer: [0u8; MAX_REQUEST_SIZE], + request_buffer_position: 0, }; let connections_to_remove = connections_to_remove.clone(); @@ -219,7 +222,7 @@ impl Connection { loop { ::log::debug!("read_tls"); - let mut buf = [0u8; BUFFER_SIZE]; + let mut buf = [0u8; INTERMEDIATE_BUFFER_SIZE]; let bytes_read = self.stream.read(&mut buf).await?; @@ -240,9 +243,19 @@ impl Connection { break; } Ok(amt) => { - self.request_buffer.extend_from_slice(&buf[..amt]); + let end = self.request_buffer_position + amt; - added_plaintext = true; + if end > self.request_buffer.len() { + return Err(anyhow::anyhow!("request too large")); + } else { + let request_buffer_slice = &mut self.request_buffer[self.request_buffer_position..end]; + + request_buffer_slice.copy_from_slice(&buf[..amt]); + + self.request_buffer_position = end; + + added_plaintext = true; + } } Err(err) if err.kind() == ErrorKind::WouldBlock => { break; @@ -258,11 +271,11 @@ impl Connection { } if added_plaintext { - match Request::from_bytes(&self.request_buffer[..]) { + match Request::from_bytes(&self.request_buffer[..self.request_buffer_position]) { Ok(request) => { ::log::debug!("received request: {:?}", request); - self.request_buffer = Vec::new(); + self.request_buffer_position = 0; return Ok(Some(Either::Left(request))); } From d659117ae51caa399a6e44d5024305db41f87fef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20Frosteg=C3=A5rd?= Date: Wed, 27 Oct 2021 20:19:02 +0200 Subject: [PATCH 33/59] Move CoreAffinityConfig to aquatic_common, use in glommio http --- Cargo.lock | 1 + aquatic_common/src/cpu_pinning.rs | 17 +++++++++++++++++ aquatic_common/src/lib.rs | 1 + aquatic_http/Cargo.toml | 1 + aquatic_http/src/lib/config.rs | 3 +++ aquatic_http/src/lib/glommio/mod.rs | 20 +++++++++++++------- aquatic_udp/src/lib/config.rs | 17 +---------------- 7 files changed, 37 insertions(+), 23 deletions(-) create mode 100644 aquatic_common/src/cpu_pinning.rs diff --git a/Cargo.lock b/Cargo.lock index d4959b1..b8de54a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -92,6 +92,7 @@ dependencies = [ "aquatic_common", "aquatic_http_protocol", "cfg-if", + "core_affinity", "crossbeam-channel", "either", "futures-lite", diff --git a/aquatic_common/src/cpu_pinning.rs b/aquatic_common/src/cpu_pinning.rs new file mode 100644 index 0000000..e76f652 --- /dev/null +++ b/aquatic_common/src/cpu_pinning.rs @@ -0,0 +1,17 @@ +use serde::{Serialize, Deserialize}; + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(default)] +pub struct CoreAffinityConfig { + pub set_affinities: bool, + pub offset: usize, +} + +impl Default for CoreAffinityConfig { + fn default() -> Self { + Self { + set_affinities: false, + offset: 0, + } + } +} diff --git a/aquatic_common/src/lib.rs b/aquatic_common/src/lib.rs index 3e90bbc..4ad9e8e 100644 --- a/aquatic_common/src/lib.rs +++ b/aquatic_common/src/lib.rs @@ -5,6 +5,7 @@ use indexmap::IndexMap; use rand::Rng; pub mod access_list; +pub mod cpu_pinning; /// Peer or connection valid until this instant /// diff --git a/aquatic_http/Cargo.toml b/aquatic_http/Cargo.toml index 97f7bab..49cac1c 100644 --- a/aquatic_http/Cargo.toml +++ b/aquatic_http/Cargo.toml @@ -26,6 +26,7 @@ aquatic_cli_helpers = "0.1.0" aquatic_common = "0.1.0" aquatic_http_protocol = "0.1.0" cfg-if = "1" +core_affinity = "0.5" either = "1" hashbrown = "0.11.2" indexmap = "1" diff --git a/aquatic_http/src/lib/config.rs b/aquatic_http/src/lib/config.rs index 08ccfad..e725dbf 100644 --- a/aquatic_http/src/lib/config.rs +++ b/aquatic_http/src/lib/config.rs @@ -1,6 +1,7 @@ use std::{net::SocketAddr, path::PathBuf}; use aquatic_common::access_list::AccessListConfig; +use aquatic_common::cpu_pinning::CoreAffinityConfig; use serde::{Deserialize, Serialize}; use aquatic_cli_helpers::LogLevel; @@ -23,6 +24,7 @@ pub struct Config { pub statistics: StatisticsConfig, pub privileges: PrivilegeConfig, pub access_list: AccessListConfig, + pub core_affinity: CoreAffinityConfig, } impl aquatic_cli_helpers::Config for Config { @@ -116,6 +118,7 @@ impl Default for Config { statistics: StatisticsConfig::default(), privileges: PrivilegeConfig::default(), access_list: AccessListConfig::default(), + core_affinity: CoreAffinityConfig::default(), } } } diff --git a/aquatic_http/src/lib/glommio/mod.rs b/aquatic_http/src/lib/glommio/mod.rs index c93fbc5..f53ed78 100644 --- a/aquatic_http/src/lib/glommio/mod.rs +++ b/aquatic_http/src/lib/glommio/mod.rs @@ -16,6 +16,12 @@ mod network; const SHARED_CHANNEL_SIZE: usize = 1024; pub fn run(config: Config) -> anyhow::Result<()> { + if config.core_affinity.set_affinities { + core_affinity::set_for_current(core_affinity::CoreId { + id: config.core_affinity.offset, + }); + } + let access_list = if config.access_list.mode.is_on() { AccessList::create_from_path(&config.access_list.path).expect("Load access list") } else { @@ -43,9 +49,9 @@ pub fn run(config: Config) -> anyhow::Result<()> { let mut builder = LocalExecutorBuilder::default(); - // if config.core_affinity.set_affinities { - // builder = builder.pin_to_cpu(config.core_affinity.offset + 1 + i); - // } + if config.core_affinity.set_affinities { + builder = builder.pin_to_cpu(config.core_affinity.offset + 1 + i); + } let executor = builder.spawn(|| async move { network::run_socket_worker( @@ -70,10 +76,10 @@ pub fn run(config: Config) -> anyhow::Result<()> { let mut builder = LocalExecutorBuilder::default(); - // if config.core_affinity.set_affinities { - // builder = - // builder.pin_to_cpu(config.core_affinity.offset + 1 + config.socket_workers + i); - // } + if config.core_affinity.set_affinities { + builder = + builder.pin_to_cpu(config.core_affinity.offset + 1 + config.socket_workers + i); + } let executor = builder.spawn(|| async move { handlers::run_request_worker( diff --git a/aquatic_udp/src/lib/config.rs b/aquatic_udp/src/lib/config.rs index d05b338..f897a96 100644 --- a/aquatic_udp/src/lib/config.rs +++ b/aquatic_udp/src/lib/config.rs @@ -1,6 +1,7 @@ use std::net::SocketAddr; use aquatic_common::access_list::AccessListConfig; +use aquatic_common::cpu_pinning::CoreAffinityConfig; use serde::{Deserialize, Serialize}; use aquatic_cli_helpers::LogLevel; @@ -109,13 +110,6 @@ pub struct PrivilegeConfig { pub user: String, } -#[derive(Clone, Debug, Serialize, Deserialize)] -#[serde(default)] -pub struct CoreAffinityConfig { - pub set_affinities: bool, - pub offset: usize, -} - impl Default for Config { fn default() -> Self { Self { @@ -193,12 +187,3 @@ impl Default for PrivilegeConfig { } } } - -impl Default for CoreAffinityConfig { - fn default() -> Self { - Self { - set_affinities: false, - offset: 0, - } - } -} From 35b8a1820d4332f22c8eebae3d6ff41ebcaf51bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20Frosteg=C3=A5rd?= Date: Wed, 27 Oct 2021 20:21:11 +0200 Subject: [PATCH 34/59] Rename CoreAffinityConfig to CpuPinningConfig, rename fields --- aquatic_common/src/cpu_pinning.rs | 8 ++++---- aquatic_http/src/lib/config.rs | 6 +++--- aquatic_http/src/lib/glommio/mod.rs | 12 ++++++------ aquatic_udp/src/lib/config.rs | 6 +++--- aquatic_udp/src/lib/glommio/mod.rs | 12 ++++++------ aquatic_udp/src/lib/mio/mod.rs | 16 ++++++++-------- 6 files changed, 30 insertions(+), 30 deletions(-) diff --git a/aquatic_common/src/cpu_pinning.rs b/aquatic_common/src/cpu_pinning.rs index e76f652..386e52b 100644 --- a/aquatic_common/src/cpu_pinning.rs +++ b/aquatic_common/src/cpu_pinning.rs @@ -2,15 +2,15 @@ use serde::{Serialize, Deserialize}; #[derive(Clone, Debug, Serialize, Deserialize)] #[serde(default)] -pub struct CoreAffinityConfig { - pub set_affinities: bool, +pub struct CpuPinningConfig { + pub active: bool, pub offset: usize, } -impl Default for CoreAffinityConfig { +impl Default for CpuPinningConfig { fn default() -> Self { Self { - set_affinities: false, + active: false, offset: 0, } } diff --git a/aquatic_http/src/lib/config.rs b/aquatic_http/src/lib/config.rs index e725dbf..0be33ed 100644 --- a/aquatic_http/src/lib/config.rs +++ b/aquatic_http/src/lib/config.rs @@ -1,7 +1,7 @@ use std::{net::SocketAddr, path::PathBuf}; use aquatic_common::access_list::AccessListConfig; -use aquatic_common::cpu_pinning::CoreAffinityConfig; +use aquatic_common::cpu_pinning::CpuPinningConfig; use serde::{Deserialize, Serialize}; use aquatic_cli_helpers::LogLevel; @@ -24,7 +24,7 @@ pub struct Config { pub statistics: StatisticsConfig, pub privileges: PrivilegeConfig, pub access_list: AccessListConfig, - pub core_affinity: CoreAffinityConfig, + pub cpu_pinning: CpuPinningConfig, } impl aquatic_cli_helpers::Config for Config { @@ -118,7 +118,7 @@ impl Default for Config { statistics: StatisticsConfig::default(), privileges: PrivilegeConfig::default(), access_list: AccessListConfig::default(), - core_affinity: CoreAffinityConfig::default(), + cpu_pinning: CpuPinningConfig::default(), } } } diff --git a/aquatic_http/src/lib/glommio/mod.rs b/aquatic_http/src/lib/glommio/mod.rs index f53ed78..5976602 100644 --- a/aquatic_http/src/lib/glommio/mod.rs +++ b/aquatic_http/src/lib/glommio/mod.rs @@ -16,9 +16,9 @@ mod network; const SHARED_CHANNEL_SIZE: usize = 1024; pub fn run(config: Config) -> anyhow::Result<()> { - if config.core_affinity.set_affinities { + if config.cpu_pinning.active { core_affinity::set_for_current(core_affinity::CoreId { - id: config.core_affinity.offset, + id: config.cpu_pinning.offset, }); } @@ -49,8 +49,8 @@ pub fn run(config: Config) -> anyhow::Result<()> { let mut builder = LocalExecutorBuilder::default(); - if config.core_affinity.set_affinities { - builder = builder.pin_to_cpu(config.core_affinity.offset + 1 + i); + if config.cpu_pinning.active { + builder = builder.pin_to_cpu(config.cpu_pinning.offset + 1 + i); } let executor = builder.spawn(|| async move { @@ -76,9 +76,9 @@ pub fn run(config: Config) -> anyhow::Result<()> { let mut builder = LocalExecutorBuilder::default(); - if config.core_affinity.set_affinities { + if config.cpu_pinning.active { builder = - builder.pin_to_cpu(config.core_affinity.offset + 1 + config.socket_workers + i); + builder.pin_to_cpu(config.cpu_pinning.offset + 1 + config.socket_workers + i); } let executor = builder.spawn(|| async move { diff --git a/aquatic_udp/src/lib/config.rs b/aquatic_udp/src/lib/config.rs index f897a96..ccf1b37 100644 --- a/aquatic_udp/src/lib/config.rs +++ b/aquatic_udp/src/lib/config.rs @@ -1,7 +1,7 @@ use std::net::SocketAddr; use aquatic_common::access_list::AccessListConfig; -use aquatic_common::cpu_pinning::CoreAffinityConfig; +use aquatic_common::cpu_pinning::CpuPinningConfig; use serde::{Deserialize, Serialize}; use aquatic_cli_helpers::LogLevel; @@ -26,7 +26,7 @@ pub struct Config { pub cleaning: CleaningConfig, pub privileges: PrivilegeConfig, pub access_list: AccessListConfig, - pub core_affinity: CoreAffinityConfig, + pub cpu_pinning: CpuPinningConfig, } impl aquatic_cli_helpers::Config for Config { @@ -125,7 +125,7 @@ impl Default for Config { cleaning: CleaningConfig::default(), privileges: PrivilegeConfig::default(), access_list: AccessListConfig::default(), - core_affinity: CoreAffinityConfig::default(), + cpu_pinning: CpuPinningConfig::default(), } } } diff --git a/aquatic_udp/src/lib/glommio/mod.rs b/aquatic_udp/src/lib/glommio/mod.rs index d95121f..407bb18 100644 --- a/aquatic_udp/src/lib/glommio/mod.rs +++ b/aquatic_udp/src/lib/glommio/mod.rs @@ -14,9 +14,9 @@ pub mod network; pub const SHARED_CHANNEL_SIZE: usize = 4096; pub fn run(config: Config) -> anyhow::Result<()> { - if config.core_affinity.set_affinities { + if config.cpu_pinning.active { core_affinity::set_for_current(core_affinity::CoreId { - id: config.core_affinity.offset, + id: config.cpu_pinning.offset, }); } @@ -44,8 +44,8 @@ pub fn run(config: Config) -> anyhow::Result<()> { let mut builder = LocalExecutorBuilder::default(); - if config.core_affinity.set_affinities { - builder = builder.pin_to_cpu(config.core_affinity.offset + 1 + i); + if config.cpu_pinning.active { + builder = builder.pin_to_cpu(config.cpu_pinning.offset + 1 + i); } let executor = builder.spawn(|| async move { @@ -70,9 +70,9 @@ pub fn run(config: Config) -> anyhow::Result<()> { let mut builder = LocalExecutorBuilder::default(); - if config.core_affinity.set_affinities { + if config.cpu_pinning.active { builder = - builder.pin_to_cpu(config.core_affinity.offset + 1 + config.socket_workers + i); + builder.pin_to_cpu(config.cpu_pinning.offset + 1 + config.socket_workers + i); } let executor = builder.spawn(|| async move { diff --git a/aquatic_udp/src/lib/mio/mod.rs b/aquatic_udp/src/lib/mio/mod.rs index 5c5f649..0a74702 100644 --- a/aquatic_udp/src/lib/mio/mod.rs +++ b/aquatic_udp/src/lib/mio/mod.rs @@ -21,9 +21,9 @@ use crate::drop_privileges_after_socket_binding; use common::State; pub fn run(config: Config) -> ::anyhow::Result<()> { - if config.core_affinity.set_affinities { + if config.cpu_pinning.active { core_affinity::set_for_current(core_affinity::CoreId { - id: config.core_affinity.offset, + id: config.cpu_pinning.offset, }); } @@ -66,9 +66,9 @@ pub fn start_workers( Builder::new() .name(format!("request-{:02}", i + 1)) .spawn(move || { - if config.core_affinity.set_affinities { + if config.cpu_pinning.active { core_affinity::set_for_current(core_affinity::CoreId { - id: config.core_affinity.offset + 1 + i, + id: config.cpu_pinning.offset + 1 + i, }); } @@ -87,9 +87,9 @@ pub fn start_workers( Builder::new() .name(format!("socket-{:02}", i + 1)) .spawn(move || { - if config.core_affinity.set_affinities { + if config.cpu_pinning.active { core_affinity::set_for_current(core_affinity::CoreId { - id: config.core_affinity.offset + 1 + config.request_workers + i, + id: config.cpu_pinning.offset + 1 + config.request_workers + i, }); } @@ -112,9 +112,9 @@ pub fn start_workers( Builder::new() .name("statistics-collector".to_string()) .spawn(move || { - if config.core_affinity.set_affinities { + if config.cpu_pinning.active { core_affinity::set_for_current(core_affinity::CoreId { - id: config.core_affinity.offset, + id: config.cpu_pinning.offset, }); } From af5816e2abf803a92b377bad2a906f3045b6c031 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20Frosteg=C3=A5rd?= Date: Wed, 27 Oct 2021 20:33:54 +0200 Subject: [PATCH 35/59] aquatic_http: glommio: periodically update access lists --- aquatic_http/src/lib/glommio/common.rs | 51 ++++++++++++++++++++++++ aquatic_http/src/lib/glommio/handlers.rs | 2 +- aquatic_http/src/lib/glommio/network.rs | 9 +++++ 3 files changed, 61 insertions(+), 1 deletion(-) diff --git a/aquatic_http/src/lib/glommio/common.rs b/aquatic_http/src/lib/glommio/common.rs index 11b2036..7495165 100644 --- a/aquatic_http/src/lib/glommio/common.rs +++ b/aquatic_http/src/lib/glommio/common.rs @@ -1,10 +1,20 @@ +use std::borrow::Borrow; +use std::cell::RefCell; +use std::rc::Rc; use std::net::SocketAddr; +use aquatic_common::access_list::AccessList; +use futures_lite::AsyncBufReadExt; +use glommio::io::{BufferedFile, StreamReaderBuilder}; +use glommio::prelude::*; + use aquatic_http_protocol::{ request::{AnnounceRequest, ScrapeRequest}, response::{AnnounceResponse, ScrapeResponse}, }; +use crate::config::Config; + #[derive(Copy, Clone, Debug)] pub struct ConsumerId(pub usize); @@ -55,3 +65,44 @@ impl ChannelResponse { } } } + +pub async fn update_access_list>(config: C, access_list: Rc>) { + if config.borrow().access_list.mode.is_on() { + match BufferedFile::open(&config.borrow().access_list.path).await { + Ok(file) => { + let mut reader = StreamReaderBuilder::new(file).build(); + let mut new_access_list = AccessList::default(); + + loop { + let mut buf = String::with_capacity(42); + + match reader.read_line(&mut buf).await { + Ok(_) => { + if let Err(err) = new_access_list.insert_from_line(&buf) { + ::log::error!( + "Couln't parse access list line '{}': {:?}", + buf, + err + ); + } + } + Err(err) => { + ::log::error!("Couln't read access list line {:?}", err); + + break; + } + } + + yield_if_needed().await; + } + + *access_list.borrow_mut() = new_access_list; + } + Err(err) => { + ::log::error!("Couldn't open access list file: {:?}", err) + } + }; + } +} + + diff --git a/aquatic_http/src/lib/glommio/handlers.rs b/aquatic_http/src/lib/glommio/handlers.rs index 13ca6b2..21a4daf 100644 --- a/aquatic_http/src/lib/glommio/handlers.rs +++ b/aquatic_http/src/lib/glommio/handlers.rs @@ -34,7 +34,7 @@ pub async fn run_request_worker( // Periodically clean torrents and update access list TimerActionRepeat::repeat(enclose!((config, torrents, access_list) move || { enclose!((config, torrents, access_list) move || async move { - // update_access_list(config.clone(), access_list.clone()).await; + update_access_list(&config, access_list.clone()).await; torrents.borrow_mut().clean(&config, &*access_list.borrow()); diff --git a/aquatic_http/src/lib/glommio/network.rs b/aquatic_http/src/lib/glommio/network.rs index 31b613b..a756704 100644 --- a/aquatic_http/src/lib/glommio/network.rs +++ b/aquatic_http/src/lib/glommio/network.rs @@ -79,6 +79,15 @@ pub async fn run_socket_worker( let connection_slab = Rc::new(RefCell::new(Slab::new())); let connections_to_remove = Rc::new(RefCell::new(Vec::new())); + // Periodically update access list + TimerActionRepeat::repeat(enclose!((config, access_list) move || { + enclose!((config, access_list) move || async move { + update_access_list(config.clone(), access_list.clone()).await; + + Some(Duration::from_secs(config.cleaning.interval)) + })() + })); + // Periodically remove closed connections TimerActionRepeat::repeat(enclose!((config, connection_slab, connections_to_remove) move || { enclose!((config, connection_slab, connections_to_remove) move || async move { From ead7650d41a57679f210de88314c130fa4b76ffd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20Frosteg=C3=A5rd?= Date: Wed, 27 Oct 2021 20:37:32 +0200 Subject: [PATCH 36/59] aquatic_udp: glommio: fix access list update bug --- aquatic_udp/src/lib/glommio/common.rs | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/aquatic_udp/src/lib/glommio/common.rs b/aquatic_udp/src/lib/glommio/common.rs index d1d398c..393038d 100644 --- a/aquatic_udp/src/lib/glommio/common.rs +++ b/aquatic_udp/src/lib/glommio/common.rs @@ -1,3 +1,4 @@ +use std::borrow::Borrow; use std::cell::RefCell; use std::rc::Rc; @@ -8,18 +9,19 @@ use glommio::prelude::*; use crate::common::*; use crate::config::Config; -pub async fn update_access_list(config: Config, access_list: Rc>) { - if config.access_list.mode.is_on() { - match BufferedFile::open(config.access_list.path).await { +pub async fn update_access_list>(config: C, access_list: Rc>) { + if config.borrow().access_list.mode.is_on() { + match BufferedFile::open(&config.borrow().access_list.path).await { Ok(file) => { let mut reader = StreamReaderBuilder::new(file).build(); + let mut new_access_list = AccessList::default(); loop { let mut buf = String::with_capacity(42); match reader.read_line(&mut buf).await { Ok(_) => { - if let Err(err) = access_list.borrow_mut().insert_from_line(&buf) { + if let Err(err) = new_access_list.insert_from_line(&buf) { ::log::error!( "Couln't parse access list line '{}': {:?}", buf, @@ -36,10 +38,12 @@ pub async fn update_access_list(config: Config, access_list: Rc { ::log::error!("Couldn't open access list file: {:?}", err) } }; } -} +} \ No newline at end of file From d6d5cc78b768da56e99662488d6e895981edcdae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20Frosteg=C3=A5rd?= Date: Wed, 27 Oct 2021 20:49:15 +0200 Subject: [PATCH 37/59] udp, http: move privilege drop code into aquatic_common --- Cargo.lock | 2 +- aquatic_common/Cargo.toml | 1 + aquatic_common/src/lib.rs | 1 + aquatic_common/src/privileges.rs | 59 +++++++++++++++++++++++++++++ aquatic_http/src/lib/config.rs | 25 +----------- aquatic_http/src/lib/glommio/mod.rs | 4 +- aquatic_udp/Cargo.toml | 1 - aquatic_udp/src/lib/config.rs | 23 +---------- aquatic_udp/src/lib/glommio/mod.rs | 4 +- aquatic_udp/src/lib/lib.rs | 35 +---------------- aquatic_udp/src/lib/mio/mod.rs | 4 +- 11 files changed, 72 insertions(+), 87 deletions(-) create mode 100644 aquatic_common/src/privileges.rs diff --git a/Cargo.lock b/Cargo.lock index b8de54a..29bed3f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -79,6 +79,7 @@ dependencies = [ "hashbrown 0.11.2", "hex", "indexmap", + "privdrop", "rand", "serde", ] @@ -179,7 +180,6 @@ dependencies = [ "mimalloc", "mio", "parking_lot", - "privdrop", "quickcheck", "quickcheck_macros", "rand", diff --git a/aquatic_common/Cargo.toml b/aquatic_common/Cargo.toml index 02fbe6e..e86e0d0 100644 --- a/aquatic_common/Cargo.toml +++ b/aquatic_common/Cargo.toml @@ -16,5 +16,6 @@ arc-swap = "1" hashbrown = "0.11.2" hex = "0.4" indexmap = "1" +privdrop = "0.5" rand = { version = "0.8", features = ["small_rng"] } serde = { version = "1", features = ["derive"] } diff --git a/aquatic_common/src/lib.rs b/aquatic_common/src/lib.rs index 4ad9e8e..8866e45 100644 --- a/aquatic_common/src/lib.rs +++ b/aquatic_common/src/lib.rs @@ -6,6 +6,7 @@ use rand::Rng; pub mod access_list; pub mod cpu_pinning; +pub mod privileges; /// Peer or connection valid until this instant /// diff --git a/aquatic_common/src/privileges.rs b/aquatic_common/src/privileges.rs new file mode 100644 index 0000000..67b9e5b --- /dev/null +++ b/aquatic_common/src/privileges.rs @@ -0,0 +1,59 @@ +use std::{sync::{Arc, atomic::{AtomicUsize, Ordering}}, time::Duration}; + +use privdrop::PrivDrop; +use serde::{Serialize, Deserialize}; + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(default)] +pub struct PrivilegeConfig { + /// Chroot and switch user after binding to sockets + pub drop_privileges: bool, + /// Chroot to this path + pub chroot_path: String, + /// User to switch to after chrooting + pub user: String, +} + +impl Default for PrivilegeConfig { + fn default() -> Self { + Self { + drop_privileges: false, + chroot_path: ".".to_string(), + user: "nobody".to_string(), + } + } +} + +pub fn drop_privileges_after_socket_binding( + config: &PrivilegeConfig, + num_bound_sockets: Arc, + target_num: usize, +) -> anyhow::Result<()> { + if config.drop_privileges { + let mut counter = 0usize; + + loop { + let num_bound = num_bound_sockets.load(Ordering::SeqCst); + + if num_bound == target_num { + PrivDrop::default() + .chroot(config.chroot_path.clone()) + .user(config.user.clone()) + .apply()?; + + break; + } + + ::std::thread::sleep(Duration::from_millis(10)); + + counter += 1; + + if counter == 500 { + panic!("Sockets didn't bind in time for privilege drop."); + } + } + } + + Ok(()) +} + diff --git a/aquatic_http/src/lib/config.rs b/aquatic_http/src/lib/config.rs index 0be33ed..33068a4 100644 --- a/aquatic_http/src/lib/config.rs +++ b/aquatic_http/src/lib/config.rs @@ -1,6 +1,6 @@ use std::{net::SocketAddr, path::PathBuf}; -use aquatic_common::access_list::AccessListConfig; +use aquatic_common::{access_list::AccessListConfig, privileges::PrivilegeConfig}; use aquatic_common::cpu_pinning::CpuPinningConfig; use serde::{Deserialize, Serialize}; @@ -94,17 +94,6 @@ pub struct StatisticsConfig { pub interval: u64, } -#[derive(Clone, Debug, Serialize, Deserialize)] -#[serde(default)] -pub struct PrivilegeConfig { - /// Chroot and switch user after binding to sockets - pub drop_privileges: bool, - /// Chroot to this path - pub chroot_path: String, - /// User to switch to after chrooting - pub user: String, -} - impl Default for Config { fn default() -> Self { Self { @@ -118,7 +107,7 @@ impl Default for Config { statistics: StatisticsConfig::default(), privileges: PrivilegeConfig::default(), access_list: AccessListConfig::default(), - cpu_pinning: CpuPinningConfig::default(), + cpu_pinning: Default::default(), } } } @@ -171,16 +160,6 @@ impl Default for StatisticsConfig { } } -impl Default for PrivilegeConfig { - fn default() -> Self { - Self { - drop_privileges: false, - chroot_path: ".".to_string(), - user: "nobody".to_string(), - } - } -} - impl Default for TlsConfig { fn default() -> Self { Self { diff --git a/aquatic_http/src/lib/glommio/mod.rs b/aquatic_http/src/lib/glommio/mod.rs index 5976602..629e239 100644 --- a/aquatic_http/src/lib/glommio/mod.rs +++ b/aquatic_http/src/lib/glommio/mod.rs @@ -4,7 +4,7 @@ use std::{ sync::{atomic::AtomicUsize, Arc}, }; -use aquatic_common::access_list::AccessList; +use aquatic_common::{access_list::AccessList, privileges::drop_privileges_after_socket_binding}; use glommio::{channels::channel_mesh::MeshBuilder, prelude::*}; use crate::config::Config; @@ -94,7 +94,7 @@ pub fn run(config: Config) -> anyhow::Result<()> { executors.push(executor); } - // drop_privileges_after_socket_binding(&config, num_bound_sockets).unwrap(); + drop_privileges_after_socket_binding(&config.privileges, num_bound_sockets, config.socket_workers).unwrap(); for executor in executors { executor diff --git a/aquatic_udp/Cargo.toml b/aquatic_udp/Cargo.toml index 4dfed44..a7bf538 100644 --- a/aquatic_udp/Cargo.toml +++ b/aquatic_udp/Cargo.toml @@ -32,7 +32,6 @@ indexmap = "1" log = "0.4" mimalloc = { version = "0.1", default-features = false } parking_lot = "0.11" -privdrop = "0.5" rand = { version = "0.8", features = ["small_rng"] } serde = { version = "1", features = ["derive"] } diff --git a/aquatic_udp/src/lib/config.rs b/aquatic_udp/src/lib/config.rs index ccf1b37..2f1540a 100644 --- a/aquatic_udp/src/lib/config.rs +++ b/aquatic_udp/src/lib/config.rs @@ -1,6 +1,6 @@ use std::net::SocketAddr; -use aquatic_common::access_list::AccessListConfig; +use aquatic_common::{access_list::AccessListConfig, privileges::PrivilegeConfig}; use aquatic_common::cpu_pinning::CpuPinningConfig; use serde::{Deserialize, Serialize}; @@ -99,17 +99,6 @@ pub struct CleaningConfig { pub max_connection_age: u64, } -#[derive(Clone, Debug, Serialize, Deserialize)] -#[serde(default)] -pub struct PrivilegeConfig { - /// Chroot and switch user after binding to sockets - pub drop_privileges: bool, - /// Chroot to this path - pub chroot_path: String, - /// User to switch to after chrooting - pub user: String, -} - impl Default for Config { fn default() -> Self { Self { @@ -177,13 +166,3 @@ impl Default for CleaningConfig { } } } - -impl Default for PrivilegeConfig { - fn default() -> Self { - Self { - drop_privileges: false, - chroot_path: ".".to_string(), - user: "nobody".to_string(), - } - } -} diff --git a/aquatic_udp/src/lib/glommio/mod.rs b/aquatic_udp/src/lib/glommio/mod.rs index 407bb18..4979d84 100644 --- a/aquatic_udp/src/lib/glommio/mod.rs +++ b/aquatic_udp/src/lib/glommio/mod.rs @@ -1,11 +1,11 @@ use std::sync::{atomic::AtomicUsize, Arc}; use aquatic_common::access_list::AccessList; +use aquatic_common::privileges::drop_privileges_after_socket_binding; use glommio::channels::channel_mesh::MeshBuilder; use glommio::prelude::*; use crate::config::Config; -use crate::drop_privileges_after_socket_binding; mod common; pub mod handlers; @@ -88,7 +88,7 @@ pub fn run(config: Config) -> anyhow::Result<()> { executors.push(executor); } - drop_privileges_after_socket_binding(&config, num_bound_sockets).unwrap(); + drop_privileges_after_socket_binding(&config.privileges, num_bound_sockets, config.socket_workers).unwrap(); for executor in executors { executor diff --git a/aquatic_udp/src/lib/lib.rs b/aquatic_udp/src/lib/lib.rs index e3eea68..e7796d4 100644 --- a/aquatic_udp/src/lib/lib.rs +++ b/aquatic_udp/src/lib/lib.rs @@ -16,7 +16,6 @@ pub mod glommio; pub mod mio; use config::Config; -use privdrop::PrivDrop; pub const APP_NAME: &str = "aquatic_udp: UDP BitTorrent tracker"; @@ -28,36 +27,4 @@ pub fn run(config: Config) -> ::anyhow::Result<()> { mio::run(config) } } -} - -fn drop_privileges_after_socket_binding( - config: &Config, - num_bound_sockets: Arc, -) -> anyhow::Result<()> { - if config.privileges.drop_privileges { - let mut counter = 0usize; - - loop { - let sockets = num_bound_sockets.load(Ordering::SeqCst); - - if sockets == config.socket_workers { - PrivDrop::default() - .chroot(config.privileges.chroot_path.clone()) - .user(config.privileges.user.clone()) - .apply()?; - - break; - } - - ::std::thread::sleep(Duration::from_millis(10)); - - counter += 1; - - if counter == 500 { - panic!("Sockets didn't bind in time for privilege drop."); - } - } - } - - Ok(()) -} +} \ No newline at end of file diff --git a/aquatic_udp/src/lib/mio/mod.rs b/aquatic_udp/src/lib/mio/mod.rs index 0a74702..3127ce2 100644 --- a/aquatic_udp/src/lib/mio/mod.rs +++ b/aquatic_udp/src/lib/mio/mod.rs @@ -6,6 +6,7 @@ use std::{ }; use anyhow::Context; +use aquatic_common::privileges::drop_privileges_after_socket_binding; use crossbeam_channel::unbounded; pub mod common; @@ -16,7 +17,6 @@ pub mod tasks; use aquatic_common::access_list::{AccessListArcSwap, AccessListMode, AccessListQuery}; use crate::config::Config; -use crate::drop_privileges_after_socket_binding; use common::State; @@ -35,7 +35,7 @@ pub fn run(config: Config) -> ::anyhow::Result<()> { start_workers(config.clone(), state.clone(), num_bound_sockets.clone())?; - drop_privileges_after_socket_binding(&config, num_bound_sockets).unwrap(); + drop_privileges_after_socket_binding(&config.privileges, num_bound_sockets, config.socket_workers).unwrap(); loop { ::std::thread::sleep(Duration::from_secs(config.cleaning.interval)); From 30fa96a7f43eb7568b7df1e1fdb6e1885f3b4f58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20Frosteg=C3=A5rd?= Date: Wed, 27 Oct 2021 20:50:55 +0200 Subject: [PATCH 38/59] aquatic_ws: use PrivilegeConfig from aquatic_common --- aquatic_ws/src/lib/config.rs | 23 +---------------------- 1 file changed, 1 insertion(+), 22 deletions(-) diff --git a/aquatic_ws/src/lib/config.rs b/aquatic_ws/src/lib/config.rs index b7ef10c..c0cd032 100644 --- a/aquatic_ws/src/lib/config.rs +++ b/aquatic_ws/src/lib/config.rs @@ -1,6 +1,6 @@ use std::net::SocketAddr; -use aquatic_common::access_list::AccessListConfig; +use aquatic_common::{access_list::AccessListConfig, privileges::PrivilegeConfig}; use serde::{Deserialize, Serialize}; use aquatic_cli_helpers::LogLevel; @@ -84,17 +84,6 @@ pub struct StatisticsConfig { pub interval: u64, } -#[derive(Clone, Debug, Serialize, Deserialize)] -#[serde(default)] -pub struct PrivilegeConfig { - /// Chroot and switch user after binding to sockets - pub drop_privileges: bool, - /// Chroot to this path - pub chroot_path: String, - /// User to switch to after chrooting - pub user: String, -} - impl Default for Config { fn default() -> Self { Self { @@ -162,13 +151,3 @@ impl Default for StatisticsConfig { Self { interval: 0 } } } - -impl Default for PrivilegeConfig { - fn default() -> Self { - Self { - drop_privileges: false, - chroot_path: ".".to_string(), - user: "nobody".to_string(), - } - } -} From 67e82ebad12d34ca9f0e411acfbdf16291c1a197 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20Frosteg=C3=A5rd?= Date: Wed, 27 Oct 2021 20:51:38 +0200 Subject: [PATCH 39/59] Update TODO --- TODO.md | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/TODO.md b/TODO.md index 6ceb42a..d409c36 100644 --- a/TODO.md +++ b/TODO.md @@ -1,13 +1,10 @@ # TODO * aquatic_http glommio: - * cpu pinning - also rename config - * access lists - * privdrop * clean out connections regularly * timeout inside of task for "it took to long to receive request, send response"? * handle panicked/cancelled tasks - * test with load tester with multiple workers + * test with load tester with multiple workers: requires tls load tester.. * get rid of / improve ConnectionMeta stuff in handler * consider better error type for request parsing, so that better error messages can be sent back (e.g., "full scrapes are not supported") From 974aaf03b477832425d27aed6b993a47d7bad721 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20Frosteg=C3=A5rd?= Date: Wed, 27 Oct 2021 20:52:01 +0200 Subject: [PATCH 40/59] aquatic_http: set default impl to glommio for testing --- aquatic_http/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aquatic_http/Cargo.toml b/aquatic_http/Cargo.toml index 49cac1c..7c9a9cc 100644 --- a/aquatic_http/Cargo.toml +++ b/aquatic_http/Cargo.toml @@ -16,7 +16,7 @@ name = "aquatic_http" path = "src/bin/main.rs" [features] -default = ["with-mio"] +default = ["with-glommio"] with-glommio = ["glommio", "futures-lite", "rustls", "rustls-pemfile", "slab"] with-mio = ["crossbeam-channel", "histogram", "mio", "native-tls", "socket2"] From 5d782999d38bedc565ffe11a94c557afc5257455 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20Frosteg=C3=A5rd?= Date: Wed, 27 Oct 2021 21:03:57 +0200 Subject: [PATCH 41/59] CI: use Rust 1.56, ignore plain HTTP result for now --- .github/actions/test-transfer/entrypoint.sh | 7 ++++--- .github/workflows/test-transfer.yml | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/actions/test-transfer/entrypoint.sh b/.github/actions/test-transfer/entrypoint.sh index 1cdc7b0..876fdef 100755 --- a/.github/actions/test-transfer/entrypoint.sh +++ b/.github/actions/test-transfer/entrypoint.sh @@ -50,6 +50,7 @@ $SUDO echo "127.0.0.1 example.com" >> /etc/hosts openssl ecparam -genkey -name prime256v1 -out key.pem openssl req -new -sha256 -key key.pem -out csr.csr -subj "/C=GB/ST=Test/L=Test/O=Test/OU=Test/CN=example.com" openssl req -x509 -sha256 -nodes -days 365 -key key.pem -in csr.csr -out cert.crt +openssl pkcs8 -in key.pem -topk8 -nocrypt -out key.pk8 # rustls $SUDO cp cert.crt /usr/local/share/ca-certificates/snakeoil.crt $SUDO update-ca-certificates @@ -71,8 +72,8 @@ echo "log_level = 'debug' [network] address = '127.0.0.1:3001' use_tls = true -tls_pkcs12_path = './identity.pfx' -tls_pkcs12_password = 'p' +tls_certificate_path = './cert.crt' +tls_private_key_path = './key.pk8' " > tls.toml ./target/debug/aquatic http -c tls.toml > "$HOME/tls.log" 2>&1 & @@ -148,7 +149,7 @@ cd .. # Check for completion -HTTP_IPv4="Failed" +HTTP_IPv4="Ok" # Ignore for now TLS_IPv4="Failed" UDP_IPv4="Failed" WSS_IPv4="Failed" diff --git a/.github/workflows/test-transfer.yml b/.github/workflows/test-transfer.yml index 2679356..ee24cc3 100644 --- a/.github/workflows/test-transfer.yml +++ b/.github/workflows/test-transfer.yml @@ -12,7 +12,7 @@ jobs: name: "Test BitTorrent file transfer over HTTP (with and without TLS), UDP and WSS" timeout-minutes: 20 container: - image: rust:1-bullseye + image: rust:1.56-bullseye options: --ulimit memlock=524288:524288 steps: - name: Checkout From e6d7f78a7add301d16d7e044f7383a76a357122b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20Frosteg=C3=A5rd?= Date: Wed, 27 Oct 2021 21:11:07 +0200 Subject: [PATCH 42/59] aquatic_http: don't use bindings_after_at - it causes CI error --- aquatic_http/src/lib/glommio/network.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/aquatic_http/src/lib/glommio/network.rs b/aquatic_http/src/lib/glommio/network.rs index a756704..085d417 100644 --- a/aquatic_http/src/lib/glommio/network.rs +++ b/aquatic_http/src/lib/glommio/network.rs @@ -346,8 +346,11 @@ impl Connection { let peer_addr = self.get_peer_addr()?; match request { - Request::Announce(request @ AnnounceRequest { info_hash, .. }) => { + Request::Announce(request) => { + let info_hash = request.info_hash; + if self.access_list.borrow().allows(self.config.access_list.mode, &info_hash.0) { + let request = ChannelRequest::Announce { request, connection_id: self.connection_id, From 85412e2976a54e60de57783b6ce2973362570889 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20Frosteg=C3=A5rd?= Date: Wed, 27 Oct 2021 21:21:55 +0200 Subject: [PATCH 43/59] file transfer CI: comment out plain HTTP stuff, use rust:1-bullseye --- .github/actions/test-transfer/action.yml | 4 +- .github/actions/test-transfer/entrypoint.sh | 48 +++++++++++---------- .github/workflows/test-transfer.yml | 6 +-- 3 files changed, 30 insertions(+), 28 deletions(-) diff --git a/.github/actions/test-transfer/action.yml b/.github/actions/test-transfer/action.yml index aa5afef..c04e4e8 100644 --- a/.github/actions/test-transfer/action.yml +++ b/.github/actions/test-transfer/action.yml @@ -1,8 +1,8 @@ name: 'test-transfer' description: 'test aquatic file transfer' outputs: - http_ipv4: - description: 'HTTP IPv4 status' + # http_ipv4: + # description: 'HTTP IPv4 status' http_tls_ipv4: description: 'HTTP IPv4 over TLS status' udp_ipv4: diff --git a/.github/actions/test-transfer/entrypoint.sh b/.github/actions/test-transfer/entrypoint.sh index 876fdef..d6a7eea 100755 --- a/.github/actions/test-transfer/entrypoint.sh +++ b/.github/actions/test-transfer/entrypoint.sh @@ -61,11 +61,11 @@ openssl pkcs12 -export -passout "pass:p" -out identity.pfx -inkey key.pem -in ce cargo build --bin aquatic -echo "log_level = 'debug' - -[network] -address = '127.0.0.1:3000'" > http.toml -./target/debug/aquatic http -c http.toml > "$HOME/http.log" 2>&1 & +# echo "log_level = 'debug' +# +# [network] +# address = '127.0.0.1:3000'" > http.toml +# ./target/debug/aquatic http -c http.toml > "$HOME/http.log" 2>&1 & echo "log_level = 'debug' @@ -101,12 +101,12 @@ mkdir torrents # Create torrents -echo "http-test-ipv4" > seed/http-test-ipv4 +# echo "http-test-ipv4" > seed/http-test-ipv4 echo "tls-test-ipv4" > seed/tls-test-ipv4 echo "udp-test-ipv4" > seed/udp-test-ipv4 echo "wss-test-ipv4" > seed/wss-test-ipv4 -mktorrent -p -o "torrents/http-ipv4.torrent" -a "http://127.0.0.1:3000/announce" "seed/http-test-ipv4" +# mktorrent -p -o "torrents/http-ipv4.torrent" -a "http://127.0.0.1:3000/announce" "seed/http-test-ipv4" mktorrent -p -o "torrents/tls-ipv4.torrent" -a "https://example.com:3001/announce" "seed/tls-test-ipv4" mktorrent -p -o "torrents/udp-ipv4.torrent" -a "udp://127.0.0.1:3000" "seed/udp-test-ipv4" mktorrent -p -o "torrents/wss-ipv4.torrent" -a "wss://example.com:3002" "seed/wss-test-ipv4" @@ -149,7 +149,7 @@ cd .. # Check for completion -HTTP_IPv4="Ok" # Ignore for now +# HTTP_IPv4="Ok" TLS_IPv4="Failed" UDP_IPv4="Failed" WSS_IPv4="Failed" @@ -160,14 +160,14 @@ echo "Watching for finished files.." while [ $i -lt 60 ] do - if test -f "leech/http-test-ipv4"; then - if grep -q "http-test-ipv4" "leech/http-test-ipv4"; then - if [ "$HTTP_IPv4" != "Ok" ]; then - HTTP_IPv4="Ok" - echo "HTTP_IPv4 is Ok" - fi - fi - fi + # if test -f "leech/http-test-ipv4"; then + # if grep -q "http-test-ipv4" "leech/http-test-ipv4"; then + # if [ "$HTTP_IPv4" != "Ok" ]; then + # HTTP_IPv4="Ok" + # echo "HTTP_IPv4 is Ok" + # fi + # fi + # fi if test -f "leech/tls-test-ipv4"; then if grep -q "tls-test-ipv4" "leech/tls-test-ipv4"; then if [ "$TLS_IPv4" != "Ok" ]; then @@ -193,7 +193,8 @@ do fi fi - if [ "$HTTP_IPv4" = "Ok" ] && [ "$TLS_IPv4" = "Ok" ] && [ "$UDP_IPv4" = "Ok" ] && [ "$WSS_IPv4" = "Ok" ]; then + # if [ "$HTTP_IPv4" = "Ok" ] && [ "$TLS_IPv4" = "Ok" ] && [ "$UDP_IPv4" = "Ok" ] && [ "$WSS_IPv4" = "Ok" ]; then + if [ "$TLS_IPv4" = "Ok" ] && [ "$UDP_IPv4" = "Ok" ] && [ "$WSS_IPv4" = "Ok" ]; then break fi @@ -204,14 +205,14 @@ done echo "Waited for $i seconds" -echo "::set-output name=http_ipv4::$HTTP_IPv4" +# echo "::set-output name=http_ipv4::$HTTP_IPv4" echo "::set-output name=http_tls_ipv4::$TLS_IPv4" echo "::set-output name=udp_ipv4::$UDP_IPv4" echo "::set-output name=wss_ipv4::$WSS_IPv4" -echo "" -echo "# --- HTTP log --- #" -cat "http.log" +# echo "" +# echo "# --- HTTP log --- #" +# cat "http.log" sleep 1 @@ -247,11 +248,12 @@ sleep 1 echo "" echo "# --- Test results --- #" -echo "HTTP (IPv4): $HTTP_IPv4" +# echo "HTTP (IPv4): $HTTP_IPv4" echo "HTTP over TLS (IPv4): $TLS_IPv4" echo "UDP (IPv4): $UDP_IPv4" echo "WSS (IPv4): $WSS_IPv4" -if [ "$HTTP_IPv4" != "Ok" ] || [ "$TLS_IPv4" != "Ok" ] || [ "$UDP_IPv4" != "Ok" ] || [ "$WSS_IPv4" != "Ok" ]; then +# if [ "$HTTP_IPv4" != "Ok" ] || [ "$TLS_IPv4" != "Ok" ] || [ "$UDP_IPv4" != "Ok" ] || [ "$WSS_IPv4" != "Ok" ]; then +if [ "$TLS_IPv4" != "Ok" ] || [ "$UDP_IPv4" != "Ok" ] || [ "$WSS_IPv4" != "Ok" ]; then exit 1 fi \ No newline at end of file diff --git a/.github/workflows/test-transfer.yml b/.github/workflows/test-transfer.yml index ee24cc3..c1ae76b 100644 --- a/.github/workflows/test-transfer.yml +++ b/.github/workflows/test-transfer.yml @@ -1,4 +1,4 @@ -name: "Test HTTP, UDP and WSS file transfer" +name: "Test UDP, TLS and WSS file transfer" on: push: @@ -9,10 +9,10 @@ on: jobs: test-transfer-http: runs-on: ubuntu-latest - name: "Test BitTorrent file transfer over HTTP (with and without TLS), UDP and WSS" + name: "Test BitTorrent file transfer over UDP, TLS and WSS" timeout-minutes: 20 container: - image: rust:1.56-bullseye + image: rust:1-bullseye options: --ulimit memlock=524288:524288 steps: - name: Checkout From e79a8c7e68086b1d659b88a97a0834f186a26b06 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20Frosteg=C3=A5rd?= Date: Wed, 27 Oct 2021 21:23:54 +0200 Subject: [PATCH 44/59] Update TODO --- TODO.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/TODO.md b/TODO.md index d409c36..1f85f70 100644 --- a/TODO.md +++ b/TODO.md @@ -1,11 +1,12 @@ # TODO * aquatic_http glommio: + * test with load tester with multiple workers: requires tls load tester + * remove mio version + * get rid of / improve ConnectionMeta stuff in handler * clean out connections regularly * timeout inside of task for "it took to long to receive request, send response"? * handle panicked/cancelled tasks - * test with load tester with multiple workers: requires tls load tester.. - * get rid of / improve ConnectionMeta stuff in handler * consider better error type for request parsing, so that better error messages can be sent back (e.g., "full scrapes are not supported") * Scrape: should stats with only zeroes be sent back for non-registered info hashes? From 13d18bbf03d0cb244e1f9bffabed0484d93b4f1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20Frosteg=C3=A5rd?= Date: Thu, 28 Oct 2021 01:06:13 +0200 Subject: [PATCH 45/59] aquatic_http_load_test: add glommio implementation --- Cargo.lock | 4 + aquatic_http_load_test/Cargo.toml | 5 + aquatic_http_load_test/src/config.rs | 2 +- aquatic_http_load_test/src/glommio.rs | 260 ++++++++++++++++++++++++++ aquatic_http_load_test/src/main.rs | 38 +++- aquatic_http_load_test/src/network.rs | 44 ++++- 6 files changed, 347 insertions(+), 6 deletions(-) create mode 100644 aquatic_http_load_test/src/glommio.rs diff --git a/Cargo.lock b/Cargo.lock index 29bed3f..aa199b5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -127,6 +127,8 @@ dependencies = [ "anyhow", "aquatic_cli_helpers", "aquatic_http_protocol", + "futures-lite", + "glommio", "hashbrown 0.11.2", "mimalloc", "mio", @@ -134,6 +136,8 @@ dependencies = [ "quickcheck_macros", "rand", "rand_distr", + "rustls", + "rustls-pemfile", "serde", ] diff --git a/aquatic_http_load_test/Cargo.toml b/aquatic_http_load_test/Cargo.toml index 7d29e52..0183b75 100644 --- a/aquatic_http_load_test/Cargo.toml +++ b/aquatic_http_load_test/Cargo.toml @@ -18,8 +18,13 @@ mimalloc = { version = "0.1", default-features = false } mio = { version = "0.7", features = ["udp", "os-poll", "os-util"] } rand = { version = "0.8", features = ["small_rng"] } rand_distr = "0.4" +rustls = { version = "0.20", features = ["dangerous_configuration"] } +rustls-pemfile = "0.2" serde = { version = "1", features = ["derive"] } +futures-lite = "1" +glommio = { git = "https://github.com/DataDog/glommio.git", rev = "4e6b14772da2f4325271fbcf12d24cf91ed466e5" } + [dev-dependencies] quickcheck = "1.0" quickcheck_macros = "1.0" diff --git a/aquatic_http_load_test/src/config.rs b/aquatic_http_load_test/src/config.rs index b05655c..eb87bb9 100644 --- a/aquatic_http_load_test/src/config.rs +++ b/aquatic_http_load_test/src/config.rs @@ -1,4 +1,4 @@ -use std::net::SocketAddr; +use std::{net::SocketAddr, path::PathBuf}; use serde::{Deserialize, Serialize}; diff --git a/aquatic_http_load_test/src/glommio.rs b/aquatic_http_load_test/src/glommio.rs new file mode 100644 index 0000000..5783a85 --- /dev/null +++ b/aquatic_http_load_test/src/glommio.rs @@ -0,0 +1,260 @@ +use std::{cell::RefCell, convert::TryInto, io::{Cursor, ErrorKind, Read}, rc::Rc, sync::{Arc, atomic::Ordering}, time::Duration}; + +use aquatic_http_protocol::response::Response; +use futures_lite::{AsyncReadExt, AsyncWriteExt}; +use glommio::{enclose, prelude::*, timer::TimerActionRepeat}; +use glommio::net::TcpStream; +use rand::{SeedableRng, prelude::SmallRng}; +use rustls::ClientConnection; + +use crate::{common::LoadTestState, config::Config, utils::create_random_request}; + +pub async fn run_socket_thread( + config: Config, + tls_config: Arc, + load_test_state: LoadTestState, +) -> anyhow::Result<()> { + let config = Rc::new(config); + let num_active_connections = Rc::new(RefCell::new(0usize)); + + TimerActionRepeat::repeat(enclose!((config, tls_config, load_test_state, num_active_connections) move || { + enclose!((config, tls_config, load_test_state, num_active_connections) move || async move { + if *num_active_connections.borrow() < config.num_connections { + spawn_local(async move { + if let Err(err) = Connection::run(config, tls_config, load_test_state, num_active_connections).await { + eprintln!("connection creation error: {:?}", err); + } + }).detach(); + } + + Some(Duration::from_secs(1)) + })() + })); + + futures_lite::future::pending::().await; + + Ok(()) +} + +struct Connection { + config: Rc, + load_test_state: LoadTestState, + rng: SmallRng, + stream: TcpStream, + tls: ClientConnection, + response_buffer: [u8; 2048], + response_buffer_position: usize, + send_new_request: bool, + queued_responses: usize, +} + +impl Connection { + async fn run( + config: Rc, + tls_config: Arc, + load_test_state: LoadTestState, + num_active_connections: Rc>, + ) -> anyhow::Result<()> { + let stream = TcpStream::connect(config.server_address).await + .map_err(|err| anyhow::anyhow!("connect: {:?}", err))?; + let tls = ClientConnection::new(tls_config, "example.com".try_into().unwrap()).unwrap(); + let rng = SmallRng::from_entropy(); + + let mut connection = Connection { + config, + load_test_state, + rng, + stream, + tls, + response_buffer: [0; 2048], + response_buffer_position: 0, + send_new_request: true, + queued_responses: 0, + }; + + *num_active_connections.borrow_mut() += 1; + + println!("run connection"); + + if let Err(err) = connection.run_connection_loop().await { + eprintln!("connection error: {:?}", err); + } + + *num_active_connections.borrow_mut() -= 1; + + Ok(()) + } + + async fn run_connection_loop(&mut self) -> anyhow::Result<()> { + loop { + if self.send_new_request { + let request = create_random_request(&self.config, &self.load_test_state, &mut self.rng); + + request.write(&mut self.tls.writer())?; + self.queued_responses += 1; + + self.send_new_request = false; + } + + self.write_tls().await?; + self.read_tls().await?; + } + } + + async fn read_tls(&mut self) -> anyhow::Result<()> { + loop { + let mut buf = [0u8; 1024]; + + let bytes_read = self.stream.read(&mut buf).await?; + + if bytes_read == 0 { + return Err(anyhow::anyhow!("Peer has closed connection")); + } + + self + .load_test_state + .statistics + .bytes_received + .fetch_add(bytes_read, Ordering::SeqCst); + + let _ = self.tls.read_tls(&mut &buf[..bytes_read]).unwrap(); + + let io_state = self.tls.process_new_packets()?; + + let mut added_plaintext = false; + + if io_state.plaintext_bytes_to_read() != 0 { + loop { + match self.tls.reader().read(&mut buf) { + Ok(0) => { + break; + } + Ok(amt) => { + let end = self.response_buffer_position + amt; + + if end > self.response_buffer.len() { + return Err(anyhow::anyhow!("response too large")); + } else { + let response_buffer_slice = &mut self.response_buffer[self.response_buffer_position..end]; + + response_buffer_slice.copy_from_slice(&buf[..amt]); + + self.response_buffer_position = end; + + added_plaintext = true; + } + } + Err(err) if err.kind() == ErrorKind::WouldBlock => { + break; + } + Err(err) => { + break; + } + } + } + } + + if added_plaintext { + let interesting_bytes = &self.response_buffer[..self.response_buffer_position]; + + let mut opt_body_start_index = None; + + for (i, chunk) in interesting_bytes.windows(4).enumerate() { + if chunk == b"\r\n\r\n" { + opt_body_start_index = Some(i + 4); + + break; + } + } + + if let Some(body_start_index) = opt_body_start_index { + let interesting_bytes = &interesting_bytes[body_start_index..]; + + match Response::from_bytes(interesting_bytes) { + Ok(response) => { + + match response { + Response::Announce(_) => { + self + .load_test_state + .statistics + .responses_announce + .fetch_add(1, Ordering::SeqCst); + } + Response::Scrape(_) => { + self + .load_test_state + .statistics + .responses_scrape + .fetch_add(1, Ordering::SeqCst); + } + Response::Failure(response) => { + self + .load_test_state + .statistics + .responses_failure + .fetch_add(1, Ordering::SeqCst); + println!( + "failure response: reason: {}", + response.failure_reason + ); + } + } + + self.response_buffer_position = 0; + self.send_new_request = true; + + break; + } + Err(err) => { + eprintln!( + "deserialize response error with {} bytes read: {:?}, text: {}", + self.response_buffer_position, + err, + String::from_utf8_lossy(interesting_bytes) + ); + } + } + } + } + + if self.tls.wants_write() { + break; + } + } + + Ok(()) + } + + async fn write_tls(&mut self) -> anyhow::Result<()> { + if !self.tls.wants_write() { + return Ok(()); + } + + let mut buf = Vec::new(); + let mut buf = Cursor::new(&mut buf); + + while self.tls.wants_write() { + self.tls.write_tls(&mut buf).unwrap(); + } + + let len = buf.get_ref().len(); + + self.stream.write_all(&buf.into_inner()).await?; + self.stream.flush().await?; + + self + .load_test_state + .statistics + .bytes_sent + .fetch_add(len, Ordering::SeqCst); + + if self.queued_responses != 0 { + self.load_test_state.statistics.requests.fetch_add(self.queued_responses, Ordering::SeqCst); + + self.queued_responses = 0; + } + + Ok(()) + } +} \ No newline at end of file diff --git a/aquatic_http_load_test/src/main.rs b/aquatic_http_load_test/src/main.rs index a23be10..efe3f6b 100644 --- a/aquatic_http_load_test/src/main.rs +++ b/aquatic_http_load_test/src/main.rs @@ -1,12 +1,16 @@ +use std::fs::File; +use std::io::BufReader; use std::sync::{atomic::Ordering, Arc}; use std::thread; use std::time::{Duration, Instant}; +use ::glommio::LocalExecutorBuilder; use rand::prelude::*; use rand_distr::Pareto; mod common; mod config; +mod glommio; mod network; mod utils; @@ -53,11 +57,16 @@ fn run(config: Config) -> ::anyhow::Result<()> { // Start socket workers + let tls_config = create_tls_config().unwrap(); + for _ in 0..config.num_workers { let config = config.clone(); + let tls_config = tls_config.clone(); let state = state.clone(); - thread::spawn(move || run_socket_thread(&config, state, 1)); + LocalExecutorBuilder::default().spawn(|| async move { + glommio::run_socket_thread(config, tls_config, state).await.unwrap(); + }).unwrap(); } monitor_statistics(state, &config); @@ -65,6 +74,33 @@ fn run(config: Config) -> ::anyhow::Result<()> { Ok(()) } +struct FakeCertificateVerifier; + +impl rustls::client::ServerCertVerifier for FakeCertificateVerifier { + fn verify_server_cert( + &self, + _end_entity: &rustls::Certificate, + _intermediates: &[rustls::Certificate], + _server_name: &rustls::ServerName, + _scts: &mut dyn Iterator, + _ocsp_response: &[u8], + _now: std::time::SystemTime, + ) -> Result { + Ok(rustls::client::ServerCertVerified::assertion()) + } +} + +fn create_tls_config() -> anyhow::Result> { + let mut config = rustls::ClientConfig::builder() + .with_safe_defaults() + .with_root_certificates(rustls::RootCertStore::empty()) + .with_no_client_auth(); + + config.dangerous().set_certificate_verifier(Arc::new(FakeCertificateVerifier)); + + Ok(Arc::new(config)) +} + fn monitor_statistics(state: LoadTestState, config: &Config) { let start_time = Instant::now(); let mut report_avg_response_vec: Vec = Vec::new(); diff --git a/aquatic_http_load_test/src/network.rs b/aquatic_http_load_test/src/network.rs index 6ddc19a..bbd9b6c 100644 --- a/aquatic_http_load_test/src/network.rs +++ b/aquatic_http_load_test/src/network.rs @@ -1,4 +1,6 @@ +use std::convert::TryInto; use std::io::{Cursor, ErrorKind, Read, Write}; +use std::sync::Arc; use std::sync::atomic::Ordering; use std::time::Duration; @@ -12,7 +14,9 @@ use crate::utils::create_random_request; pub struct Connection { stream: TcpStream, + tls: rustls::ClientConnection, read_buffer: [u8; 4096], + response_buffer: Vec, bytes_read: usize, can_send: bool, } @@ -20,11 +24,13 @@ pub struct Connection { impl Connection { pub fn create_and_register( config: &Config, + tls_config: Arc, connections: &mut ConnectionMap, poll: &mut Poll, token_counter: &mut usize, ) -> anyhow::Result<()> { let mut stream = TcpStream::connect(config.server_address)?; + let tls = rustls::ClientConnection::new(tls_config, "example.com".try_into().unwrap())?; poll.registry() .register(&mut stream, Token(*token_counter), Interest::READABLE) @@ -32,7 +38,9 @@ impl Connection { let connection = Connection { stream, + tls, read_buffer: [0; 4096], + response_buffer: Vec::new(), bytes_read: 0, can_send: true, }; @@ -58,7 +66,23 @@ impl Connection { Ok(bytes_read) => { self.bytes_read += bytes_read; - let interesting_bytes = &self.read_buffer[..self.bytes_read]; + let mut interesting_bytes = &self.read_buffer[..self.bytes_read]; + + self.tls.read_tls(&mut interesting_bytes as &mut dyn std::io::Read).unwrap(); + + let io_state = self.tls.process_new_packets().unwrap(); + + if io_state.plaintext_bytes_to_read() == 0 { + while self.tls.wants_write(){ + self.tls.write_tls(&mut self.stream).unwrap(); + } + + break false; + } + + self.tls.reader().read_to_end(&mut self.response_buffer).unwrap(); + + let interesting_bytes = &self.response_buffer[..]; let mut opt_body_start_index = None; @@ -166,7 +190,13 @@ impl Connection { state: &LoadTestState, request: &[u8], ) -> ::std::io::Result<()> { - let bytes_sent = self.stream.write(request)?; + self.tls.writer().write(request)?; + + let mut bytes_sent = 0; + + while self.tls.wants_write(){ + bytes_sent += self.tls.write_tls(&mut self.stream)?; + } state .statistics @@ -185,7 +215,12 @@ impl Connection { pub type ConnectionMap = HashMap; -pub fn run_socket_thread(config: &Config, state: LoadTestState, num_initial_requests: usize) { +pub fn run_socket_thread( + config: &Config, + tls_config: Arc, + state: LoadTestState, + num_initial_requests: usize +) { let timeout = Duration::from_micros(config.network.poll_timeout_microseconds); let create_conn_interval = 2 ^ config.network.connection_creation_interval; @@ -199,7 +234,7 @@ pub fn run_socket_thread(config: &Config, state: LoadTestState, num_initial_requ let mut token_counter = 0usize; for _ in 0..num_initial_requests { - Connection::create_and_register(config, &mut connections, &mut poll, &mut token_counter) + Connection::create_and_register(config, tls_config.clone(), &mut connections, &mut poll, &mut token_counter) .unwrap(); } @@ -256,6 +291,7 @@ pub fn run_socket_thread(config: &Config, state: LoadTestState, num_initial_requ for _ in 0..num_to_create { let ok = Connection::create_and_register( config, + tls_config.clone(), &mut connections, &mut poll, &mut token_counter, From e458cc54db11f7cb6c13f525f145feb6cc6d9ae4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20Frosteg=C3=A5rd?= Date: Thu, 28 Oct 2021 01:13:18 +0200 Subject: [PATCH 46/59] aquatic_http_load_test: remove mio implementation, clean up --- Cargo.lock | 2 - aquatic_http_load_test/Cargo.toml | 7 +- aquatic_http_load_test/src/config.rs | 22 +- aquatic_http_load_test/src/glommio.rs | 260 -------------- aquatic_http_load_test/src/main.rs | 59 ++-- aquatic_http_load_test/src/network.rs | 478 ++++++++++++-------------- 6 files changed, 242 insertions(+), 586 deletions(-) delete mode 100644 aquatic_http_load_test/src/glommio.rs diff --git a/Cargo.lock b/Cargo.lock index aa199b5..d76d7ea 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -131,13 +131,11 @@ dependencies = [ "glommio", "hashbrown 0.11.2", "mimalloc", - "mio", "quickcheck", "quickcheck_macros", "rand", "rand_distr", "rustls", - "rustls-pemfile", "serde", ] diff --git a/aquatic_http_load_test/Cargo.toml b/aquatic_http_load_test/Cargo.toml index 0183b75..25baab6 100644 --- a/aquatic_http_load_test/Cargo.toml +++ b/aquatic_http_load_test/Cargo.toml @@ -13,18 +13,15 @@ name = "aquatic_http_load_test" anyhow = "1" aquatic_cli_helpers = "0.1.0" aquatic_http_protocol = "0.1.0" +futures-lite = "1" hashbrown = "0.11.2" +glommio = { git = "https://github.com/DataDog/glommio.git", rev = "4e6b14772da2f4325271fbcf12d24cf91ed466e5" } mimalloc = { version = "0.1", default-features = false } -mio = { version = "0.7", features = ["udp", "os-poll", "os-util"] } rand = { version = "0.8", features = ["small_rng"] } rand_distr = "0.4" rustls = { version = "0.20", features = ["dangerous_configuration"] } -rustls-pemfile = "0.2" serde = { version = "1", features = ["derive"] } -futures-lite = "1" -glommio = { git = "https://github.com/DataDog/glommio.git", rev = "4e6b14772da2f4325271fbcf12d24cf91ed466e5" } - [dev-dependencies] quickcheck = "1.0" quickcheck_macros = "1.0" diff --git a/aquatic_http_load_test/src/config.rs b/aquatic_http_load_test/src/config.rs index eb87bb9..1c8456a 100644 --- a/aquatic_http_load_test/src/config.rs +++ b/aquatic_http_load_test/src/config.rs @@ -1,4 +1,4 @@ -use std::{net::SocketAddr, path::PathBuf}; +use std::net::SocketAddr; use serde::{Deserialize, Serialize}; @@ -9,20 +9,11 @@ pub struct Config { pub num_workers: u8, pub num_connections: usize, pub duration: usize, - pub network: NetworkConfig, pub torrents: TorrentConfig, } impl aquatic_cli_helpers::Config for Config {} -#[derive(Clone, Debug, Serialize, Deserialize)] -#[serde(default)] -pub struct NetworkConfig { - pub connection_creation_interval: usize, - pub poll_timeout_microseconds: u64, - pub poll_event_capacity: usize, -} - #[derive(Clone, Debug, Serialize, Deserialize)] #[serde(default)] pub struct TorrentConfig { @@ -48,22 +39,11 @@ impl Default for Config { num_workers: 1, num_connections: 8, duration: 0, - network: NetworkConfig::default(), torrents: TorrentConfig::default(), } } } -impl Default for NetworkConfig { - fn default() -> Self { - Self { - connection_creation_interval: 10, - poll_timeout_microseconds: 197, - poll_event_capacity: 64, - } - } -} - impl Default for TorrentConfig { fn default() -> Self { Self { diff --git a/aquatic_http_load_test/src/glommio.rs b/aquatic_http_load_test/src/glommio.rs deleted file mode 100644 index 5783a85..0000000 --- a/aquatic_http_load_test/src/glommio.rs +++ /dev/null @@ -1,260 +0,0 @@ -use std::{cell::RefCell, convert::TryInto, io::{Cursor, ErrorKind, Read}, rc::Rc, sync::{Arc, atomic::Ordering}, time::Duration}; - -use aquatic_http_protocol::response::Response; -use futures_lite::{AsyncReadExt, AsyncWriteExt}; -use glommio::{enclose, prelude::*, timer::TimerActionRepeat}; -use glommio::net::TcpStream; -use rand::{SeedableRng, prelude::SmallRng}; -use rustls::ClientConnection; - -use crate::{common::LoadTestState, config::Config, utils::create_random_request}; - -pub async fn run_socket_thread( - config: Config, - tls_config: Arc, - load_test_state: LoadTestState, -) -> anyhow::Result<()> { - let config = Rc::new(config); - let num_active_connections = Rc::new(RefCell::new(0usize)); - - TimerActionRepeat::repeat(enclose!((config, tls_config, load_test_state, num_active_connections) move || { - enclose!((config, tls_config, load_test_state, num_active_connections) move || async move { - if *num_active_connections.borrow() < config.num_connections { - spawn_local(async move { - if let Err(err) = Connection::run(config, tls_config, load_test_state, num_active_connections).await { - eprintln!("connection creation error: {:?}", err); - } - }).detach(); - } - - Some(Duration::from_secs(1)) - })() - })); - - futures_lite::future::pending::().await; - - Ok(()) -} - -struct Connection { - config: Rc, - load_test_state: LoadTestState, - rng: SmallRng, - stream: TcpStream, - tls: ClientConnection, - response_buffer: [u8; 2048], - response_buffer_position: usize, - send_new_request: bool, - queued_responses: usize, -} - -impl Connection { - async fn run( - config: Rc, - tls_config: Arc, - load_test_state: LoadTestState, - num_active_connections: Rc>, - ) -> anyhow::Result<()> { - let stream = TcpStream::connect(config.server_address).await - .map_err(|err| anyhow::anyhow!("connect: {:?}", err))?; - let tls = ClientConnection::new(tls_config, "example.com".try_into().unwrap()).unwrap(); - let rng = SmallRng::from_entropy(); - - let mut connection = Connection { - config, - load_test_state, - rng, - stream, - tls, - response_buffer: [0; 2048], - response_buffer_position: 0, - send_new_request: true, - queued_responses: 0, - }; - - *num_active_connections.borrow_mut() += 1; - - println!("run connection"); - - if let Err(err) = connection.run_connection_loop().await { - eprintln!("connection error: {:?}", err); - } - - *num_active_connections.borrow_mut() -= 1; - - Ok(()) - } - - async fn run_connection_loop(&mut self) -> anyhow::Result<()> { - loop { - if self.send_new_request { - let request = create_random_request(&self.config, &self.load_test_state, &mut self.rng); - - request.write(&mut self.tls.writer())?; - self.queued_responses += 1; - - self.send_new_request = false; - } - - self.write_tls().await?; - self.read_tls().await?; - } - } - - async fn read_tls(&mut self) -> anyhow::Result<()> { - loop { - let mut buf = [0u8; 1024]; - - let bytes_read = self.stream.read(&mut buf).await?; - - if bytes_read == 0 { - return Err(anyhow::anyhow!("Peer has closed connection")); - } - - self - .load_test_state - .statistics - .bytes_received - .fetch_add(bytes_read, Ordering::SeqCst); - - let _ = self.tls.read_tls(&mut &buf[..bytes_read]).unwrap(); - - let io_state = self.tls.process_new_packets()?; - - let mut added_plaintext = false; - - if io_state.plaintext_bytes_to_read() != 0 { - loop { - match self.tls.reader().read(&mut buf) { - Ok(0) => { - break; - } - Ok(amt) => { - let end = self.response_buffer_position + amt; - - if end > self.response_buffer.len() { - return Err(anyhow::anyhow!("response too large")); - } else { - let response_buffer_slice = &mut self.response_buffer[self.response_buffer_position..end]; - - response_buffer_slice.copy_from_slice(&buf[..amt]); - - self.response_buffer_position = end; - - added_plaintext = true; - } - } - Err(err) if err.kind() == ErrorKind::WouldBlock => { - break; - } - Err(err) => { - break; - } - } - } - } - - if added_plaintext { - let interesting_bytes = &self.response_buffer[..self.response_buffer_position]; - - let mut opt_body_start_index = None; - - for (i, chunk) in interesting_bytes.windows(4).enumerate() { - if chunk == b"\r\n\r\n" { - opt_body_start_index = Some(i + 4); - - break; - } - } - - if let Some(body_start_index) = opt_body_start_index { - let interesting_bytes = &interesting_bytes[body_start_index..]; - - match Response::from_bytes(interesting_bytes) { - Ok(response) => { - - match response { - Response::Announce(_) => { - self - .load_test_state - .statistics - .responses_announce - .fetch_add(1, Ordering::SeqCst); - } - Response::Scrape(_) => { - self - .load_test_state - .statistics - .responses_scrape - .fetch_add(1, Ordering::SeqCst); - } - Response::Failure(response) => { - self - .load_test_state - .statistics - .responses_failure - .fetch_add(1, Ordering::SeqCst); - println!( - "failure response: reason: {}", - response.failure_reason - ); - } - } - - self.response_buffer_position = 0; - self.send_new_request = true; - - break; - } - Err(err) => { - eprintln!( - "deserialize response error with {} bytes read: {:?}, text: {}", - self.response_buffer_position, - err, - String::from_utf8_lossy(interesting_bytes) - ); - } - } - } - } - - if self.tls.wants_write() { - break; - } - } - - Ok(()) - } - - async fn write_tls(&mut self) -> anyhow::Result<()> { - if !self.tls.wants_write() { - return Ok(()); - } - - let mut buf = Vec::new(); - let mut buf = Cursor::new(&mut buf); - - while self.tls.wants_write() { - self.tls.write_tls(&mut buf).unwrap(); - } - - let len = buf.get_ref().len(); - - self.stream.write_all(&buf.into_inner()).await?; - self.stream.flush().await?; - - self - .load_test_state - .statistics - .bytes_sent - .fetch_add(len, Ordering::SeqCst); - - if self.queued_responses != 0 { - self.load_test_state.statistics.requests.fetch_add(self.queued_responses, Ordering::SeqCst); - - self.queued_responses = 0; - } - - Ok(()) - } -} \ No newline at end of file diff --git a/aquatic_http_load_test/src/main.rs b/aquatic_http_load_test/src/main.rs index efe3f6b..b0d3120 100644 --- a/aquatic_http_load_test/src/main.rs +++ b/aquatic_http_load_test/src/main.rs @@ -1,5 +1,3 @@ -use std::fs::File; -use std::io::BufReader; use std::sync::{atomic::Ordering, Arc}; use std::thread; use std::time::{Duration, Instant}; @@ -10,7 +8,6 @@ use rand_distr::Pareto; mod common; mod config; -mod glommio; mod network; mod utils; @@ -65,7 +62,7 @@ fn run(config: Config) -> ::anyhow::Result<()> { let state = state.clone(); LocalExecutorBuilder::default().spawn(|| async move { - glommio::run_socket_thread(config, tls_config, state).await.unwrap(); + run_socket_thread(config, tls_config, state).await.unwrap(); }).unwrap(); } @@ -74,33 +71,6 @@ fn run(config: Config) -> ::anyhow::Result<()> { Ok(()) } -struct FakeCertificateVerifier; - -impl rustls::client::ServerCertVerifier for FakeCertificateVerifier { - fn verify_server_cert( - &self, - _end_entity: &rustls::Certificate, - _intermediates: &[rustls::Certificate], - _server_name: &rustls::ServerName, - _scts: &mut dyn Iterator, - _ocsp_response: &[u8], - _now: std::time::SystemTime, - ) -> Result { - Ok(rustls::client::ServerCertVerified::assertion()) - } -} - -fn create_tls_config() -> anyhow::Result> { - let mut config = rustls::ClientConfig::builder() - .with_safe_defaults() - .with_root_certificates(rustls::RootCertStore::empty()) - .with_no_client_auth(); - - config.dangerous().set_certificate_verifier(Arc::new(FakeCertificateVerifier)); - - Ok(Arc::new(config)) -} - fn monitor_statistics(state: LoadTestState, config: &Config) { let start_time = Instant::now(); let mut report_avg_response_vec: Vec = Vec::new(); @@ -183,3 +153,30 @@ fn monitor_statistics(state: LoadTestState, config: &Config) { } } } + +struct FakeCertificateVerifier; + +impl rustls::client::ServerCertVerifier for FakeCertificateVerifier { + fn verify_server_cert( + &self, + _end_entity: &rustls::Certificate, + _intermediates: &[rustls::Certificate], + _server_name: &rustls::ServerName, + _scts: &mut dyn Iterator, + _ocsp_response: &[u8], + _now: std::time::SystemTime, + ) -> Result { + Ok(rustls::client::ServerCertVerified::assertion()) + } +} + +fn create_tls_config() -> anyhow::Result> { + let mut config = rustls::ClientConfig::builder() + .with_safe_defaults() + .with_root_certificates(rustls::RootCertStore::empty()) + .with_no_client_auth(); + + config.dangerous().set_certificate_verifier(Arc::new(FakeCertificateVerifier)); + + Ok(Arc::new(config)) +} diff --git a/aquatic_http_load_test/src/network.rs b/aquatic_http_load_test/src/network.rs index bbd9b6c..5783a85 100644 --- a/aquatic_http_load_test/src/network.rs +++ b/aquatic_http_load_test/src/network.rs @@ -1,316 +1,260 @@ -use std::convert::TryInto; -use std::io::{Cursor, ErrorKind, Read, Write}; -use std::sync::Arc; -use std::sync::atomic::Ordering; -use std::time::Duration; +use std::{cell::RefCell, convert::TryInto, io::{Cursor, ErrorKind, Read}, rc::Rc, sync::{Arc, atomic::Ordering}, time::Duration}; -use hashbrown::HashMap; -use mio::{net::TcpStream, Events, Interest, Poll, Token}; -use rand::{prelude::*, rngs::SmallRng}; +use aquatic_http_protocol::response::Response; +use futures_lite::{AsyncReadExt, AsyncWriteExt}; +use glommio::{enclose, prelude::*, timer::TimerActionRepeat}; +use glommio::net::TcpStream; +use rand::{SeedableRng, prelude::SmallRng}; +use rustls::ClientConnection; -use crate::common::*; -use crate::config::*; -use crate::utils::create_random_request; +use crate::{common::LoadTestState, config::Config, utils::create_random_request}; -pub struct Connection { +pub async fn run_socket_thread( + config: Config, + tls_config: Arc, + load_test_state: LoadTestState, +) -> anyhow::Result<()> { + let config = Rc::new(config); + let num_active_connections = Rc::new(RefCell::new(0usize)); + + TimerActionRepeat::repeat(enclose!((config, tls_config, load_test_state, num_active_connections) move || { + enclose!((config, tls_config, load_test_state, num_active_connections) move || async move { + if *num_active_connections.borrow() < config.num_connections { + spawn_local(async move { + if let Err(err) = Connection::run(config, tls_config, load_test_state, num_active_connections).await { + eprintln!("connection creation error: {:?}", err); + } + }).detach(); + } + + Some(Duration::from_secs(1)) + })() + })); + + futures_lite::future::pending::().await; + + Ok(()) +} + +struct Connection { + config: Rc, + load_test_state: LoadTestState, + rng: SmallRng, stream: TcpStream, - tls: rustls::ClientConnection, - read_buffer: [u8; 4096], - response_buffer: Vec, - bytes_read: usize, - can_send: bool, + tls: ClientConnection, + response_buffer: [u8; 2048], + response_buffer_position: usize, + send_new_request: bool, + queued_responses: usize, } impl Connection { - pub fn create_and_register( - config: &Config, + async fn run( + config: Rc, tls_config: Arc, - connections: &mut ConnectionMap, - poll: &mut Poll, - token_counter: &mut usize, + load_test_state: LoadTestState, + num_active_connections: Rc>, ) -> anyhow::Result<()> { - let mut stream = TcpStream::connect(config.server_address)?; - let tls = rustls::ClientConnection::new(tls_config, "example.com".try_into().unwrap())?; + let stream = TcpStream::connect(config.server_address).await + .map_err(|err| anyhow::anyhow!("connect: {:?}", err))?; + let tls = ClientConnection::new(tls_config, "example.com".try_into().unwrap()).unwrap(); + let rng = SmallRng::from_entropy(); - poll.registry() - .register(&mut stream, Token(*token_counter), Interest::READABLE) - .unwrap(); - - let connection = Connection { + let mut connection = Connection { + config, + load_test_state, + rng, stream, tls, - read_buffer: [0; 4096], - response_buffer: Vec::new(), - bytes_read: 0, - can_send: true, + response_buffer: [0; 2048], + response_buffer_position: 0, + send_new_request: true, + queued_responses: 0, }; - connections.insert(*token_counter, connection); + *num_active_connections.borrow_mut() += 1; - *token_counter = token_counter.wrapping_add(1); + println!("run connection"); + + if let Err(err) = connection.run_connection_loop().await { + eprintln!("connection error: {:?}", err); + } + + *num_active_connections.borrow_mut() -= 1; Ok(()) } - pub fn read_response(&mut self, state: &LoadTestState) -> bool { - // bool = remove connection + async fn run_connection_loop(&mut self) -> anyhow::Result<()> { loop { - match self.stream.read(&mut self.read_buffer[self.bytes_read..]) { - Ok(0) => { - if self.bytes_read == self.read_buffer.len() { - eprintln!("read buffer is full"); - } + if self.send_new_request { + let request = create_random_request(&self.config, &self.load_test_state, &mut self.rng); - break true; - } - Ok(bytes_read) => { - self.bytes_read += bytes_read; + request.write(&mut self.tls.writer())?; + self.queued_responses += 1; - let mut interesting_bytes = &self.read_buffer[..self.bytes_read]; + self.send_new_request = false; + } - self.tls.read_tls(&mut interesting_bytes as &mut dyn std::io::Read).unwrap(); + self.write_tls().await?; + self.read_tls().await?; + } + } - let io_state = self.tls.process_new_packets().unwrap(); + async fn read_tls(&mut self) -> anyhow::Result<()> { + loop { + let mut buf = [0u8; 1024]; - if io_state.plaintext_bytes_to_read() == 0 { - while self.tls.wants_write(){ - self.tls.write_tls(&mut self.stream).unwrap(); + let bytes_read = self.stream.read(&mut buf).await?; + + if bytes_read == 0 { + return Err(anyhow::anyhow!("Peer has closed connection")); + } + + self + .load_test_state + .statistics + .bytes_received + .fetch_add(bytes_read, Ordering::SeqCst); + + let _ = self.tls.read_tls(&mut &buf[..bytes_read]).unwrap(); + + let io_state = self.tls.process_new_packets()?; + + let mut added_plaintext = false; + + if io_state.plaintext_bytes_to_read() != 0 { + loop { + match self.tls.reader().read(&mut buf) { + Ok(0) => { + break; } + Ok(amt) => { + let end = self.response_buffer_position + amt; - break false; - } + if end > self.response_buffer.len() { + return Err(anyhow::anyhow!("response too large")); + } else { + let response_buffer_slice = &mut self.response_buffer[self.response_buffer_position..end]; - self.tls.reader().read_to_end(&mut self.response_buffer).unwrap(); + response_buffer_slice.copy_from_slice(&buf[..amt]); - let interesting_bytes = &self.response_buffer[..]; - - let mut opt_body_start_index = None; - - for (i, chunk) in interesting_bytes.windows(4).enumerate() { - if chunk == b"\r\n\r\n" { - opt_body_start_index = Some(i + 4); + self.response_buffer_position = end; + added_plaintext = true; + } + } + Err(err) if err.kind() == ErrorKind::WouldBlock => { + break; + } + Err(err) => { break; } } + } + } - if let Some(body_start_index) = opt_body_start_index { - let interesting_bytes = &interesting_bytes[body_start_index..]; + if added_plaintext { + let interesting_bytes = &self.response_buffer[..self.response_buffer_position]; - match Response::from_bytes(interesting_bytes) { - Ok(response) => { - state - .statistics - .bytes_received - .fetch_add(self.bytes_read, Ordering::SeqCst); + let mut opt_body_start_index = None; - match response { - Response::Announce(_) => { - state - .statistics - .responses_announce - .fetch_add(1, Ordering::SeqCst); - } - Response::Scrape(_) => { - state - .statistics - .responses_scrape - .fetch_add(1, Ordering::SeqCst); - } - Response::Failure(response) => { - state - .statistics - .responses_failure - .fetch_add(1, Ordering::SeqCst); - println!( - "failure response: reason: {}", - response.failure_reason - ); - } + for (i, chunk) in interesting_bytes.windows(4).enumerate() { + if chunk == b"\r\n\r\n" { + opt_body_start_index = Some(i + 4); + + break; + } + } + + if let Some(body_start_index) = opt_body_start_index { + let interesting_bytes = &interesting_bytes[body_start_index..]; + + match Response::from_bytes(interesting_bytes) { + Ok(response) => { + + match response { + Response::Announce(_) => { + self + .load_test_state + .statistics + .responses_announce + .fetch_add(1, Ordering::SeqCst); } + Response::Scrape(_) => { + self + .load_test_state + .statistics + .responses_scrape + .fetch_add(1, Ordering::SeqCst); + } + Response::Failure(response) => { + self + .load_test_state + .statistics + .responses_failure + .fetch_add(1, Ordering::SeqCst); + println!( + "failure response: reason: {}", + response.failure_reason + ); + } + } - self.bytes_read = 0; - self.can_send = true; - } - Err(err) => { - eprintln!( - "deserialize response error with {} bytes read: {:?}, text: {}", - self.bytes_read, - err, - String::from_utf8_lossy(interesting_bytes) - ); - } + self.response_buffer_position = 0; + self.send_new_request = true; + + break; + } + Err(err) => { + eprintln!( + "deserialize response error with {} bytes read: {:?}, text: {}", + self.response_buffer_position, + err, + String::from_utf8_lossy(interesting_bytes) + ); } } } - Err(err) if err.kind() == ErrorKind::WouldBlock => { - break false; - } - Err(_) => { - self.bytes_read = 0; + } - break true; - } + if self.tls.wants_write() { + break; } } - } - - pub fn send_request( - &mut self, - config: &Config, - state: &LoadTestState, - rng: &mut impl Rng, - request_buffer: &mut Cursor<&mut [u8]>, - ) -> bool { - // bool = remove connection - if !self.can_send { - return false; - } - - let request = create_random_request(&config, &state, rng); - - request_buffer.set_position(0); - request.write(request_buffer).unwrap(); - let position = request_buffer.position() as usize; - - match self.send_request_inner(state, &request_buffer.get_mut()[..position]) { - Ok(()) => { - state.statistics.requests.fetch_add(1, Ordering::SeqCst); - - self.can_send = false; - - false - } - Err(_) => true, - } - } - - fn send_request_inner( - &mut self, - state: &LoadTestState, - request: &[u8], - ) -> ::std::io::Result<()> { - self.tls.writer().write(request)?; - - let mut bytes_sent = 0; - - while self.tls.wants_write(){ - bytes_sent += self.tls.write_tls(&mut self.stream)?; - } - - state - .statistics - .bytes_sent - .fetch_add(bytes_sent, Ordering::SeqCst); - - self.stream.flush()?; Ok(()) } - fn deregister(&mut self, poll: &mut Poll) -> ::std::io::Result<()> { - poll.registry().deregister(&mut self.stream) + async fn write_tls(&mut self) -> anyhow::Result<()> { + if !self.tls.wants_write() { + return Ok(()); + } + + let mut buf = Vec::new(); + let mut buf = Cursor::new(&mut buf); + + while self.tls.wants_write() { + self.tls.write_tls(&mut buf).unwrap(); + } + + let len = buf.get_ref().len(); + + self.stream.write_all(&buf.into_inner()).await?; + self.stream.flush().await?; + + self + .load_test_state + .statistics + .bytes_sent + .fetch_add(len, Ordering::SeqCst); + + if self.queued_responses != 0 { + self.load_test_state.statistics.requests.fetch_add(self.queued_responses, Ordering::SeqCst); + + self.queued_responses = 0; + } + + Ok(()) } -} - -pub type ConnectionMap = HashMap; - -pub fn run_socket_thread( - config: &Config, - tls_config: Arc, - state: LoadTestState, - num_initial_requests: usize -) { - let timeout = Duration::from_micros(config.network.poll_timeout_microseconds); - let create_conn_interval = 2 ^ config.network.connection_creation_interval; - - let mut connections: ConnectionMap = HashMap::with_capacity(config.num_connections); - let mut poll = Poll::new().expect("create poll"); - let mut events = Events::with_capacity(config.network.poll_event_capacity); - let mut rng = SmallRng::from_entropy(); - let mut request_buffer = [0u8; 1024]; - let mut request_buffer = Cursor::new(&mut request_buffer[..]); - - let mut token_counter = 0usize; - - for _ in 0..num_initial_requests { - Connection::create_and_register(config, tls_config.clone(), &mut connections, &mut poll, &mut token_counter) - .unwrap(); - } - - let mut iter_counter = 0usize; - let mut num_to_create = 0usize; - - let mut drop_connections = Vec::with_capacity(config.num_connections); - - loop { - poll.poll(&mut events, Some(timeout)) - .expect("failed polling"); - - for event in events.iter() { - if event.is_readable() { - let token = event.token(); - - if let Some(connection) = connections.get_mut(&token.0) { - // Note that this does not indicate successfully reading - // response - if connection.read_response(&state) { - remove_connection(&mut poll, &mut connections, token.0); - - num_to_create += 1; - } - } else { - eprintln!("connection not found: {:?}", token); - } - } - } - - for (k, connection) in connections.iter_mut() { - let remove_connection = - connection.send_request(config, &state, &mut rng, &mut request_buffer); - - if remove_connection { - drop_connections.push(*k); - } - } - - for k in drop_connections.drain(..) { - remove_connection(&mut poll, &mut connections, k); - - num_to_create += 1; - } - - let max_new = config.num_connections - connections.len(); - - if iter_counter % create_conn_interval == 0 { - num_to_create += 1; - } - - num_to_create = num_to_create.min(max_new); - - for _ in 0..num_to_create { - let ok = Connection::create_and_register( - config, - tls_config.clone(), - &mut connections, - &mut poll, - &mut token_counter, - ) - .is_ok(); - - if ok { - num_to_create -= 1; - } - } - - iter_counter = iter_counter.wrapping_add(1); - } -} - -fn remove_connection(poll: &mut Poll, connections: &mut ConnectionMap, connection_id: usize) { - if let Some(mut connection) = connections.remove(&connection_id) { - if let Err(err) = connection.deregister(poll) { - eprintln!("couldn't deregister connection: {}", err); - } - } -} +} \ No newline at end of file From 80447d9d1b30f942296dd9a0d9e2a3ffa7f964c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20Frosteg=C3=A5rd?= Date: Thu, 28 Oct 2021 01:21:47 +0200 Subject: [PATCH 47/59] aquatic_http_load_test: refactor timer code; panic on an error case --- aquatic_http_load_test/src/network.rs | 40 +++++++++++++++++---------- 1 file changed, 25 insertions(+), 15 deletions(-) diff --git a/aquatic_http_load_test/src/network.rs b/aquatic_http_load_test/src/network.rs index 5783a85..5c38dc2 100644 --- a/aquatic_http_load_test/src/network.rs +++ b/aquatic_http_load_test/src/network.rs @@ -2,7 +2,7 @@ use std::{cell::RefCell, convert::TryInto, io::{Cursor, ErrorKind, Read}, rc::Rc use aquatic_http_protocol::response::Response; use futures_lite::{AsyncReadExt, AsyncWriteExt}; -use glommio::{enclose, prelude::*, timer::TimerActionRepeat}; +use glommio::{prelude::*, timer::TimerActionRepeat}; use glommio::net::TcpStream; use rand::{SeedableRng, prelude::SmallRng}; use rustls::ClientConnection; @@ -17,25 +17,35 @@ pub async fn run_socket_thread( let config = Rc::new(config); let num_active_connections = Rc::new(RefCell::new(0usize)); - TimerActionRepeat::repeat(enclose!((config, tls_config, load_test_state, num_active_connections) move || { - enclose!((config, tls_config, load_test_state, num_active_connections) move || async move { - if *num_active_connections.borrow() < config.num_connections { - spawn_local(async move { - if let Err(err) = Connection::run(config, tls_config, load_test_state, num_active_connections).await { - eprintln!("connection creation error: {:?}", err); - } - }).detach(); - } - - Some(Duration::from_secs(1)) - })() - })); + TimerActionRepeat::repeat(move || periodically_open_connections( + config.clone(), + tls_config.clone(), + load_test_state.clone(), + num_active_connections.clone()) + ); futures_lite::future::pending::().await; Ok(()) } +async fn periodically_open_connections( + config: Rc, + tls_config: Arc, + load_test_state: LoadTestState, + num_active_connections: Rc>, +) -> Option { + if *num_active_connections.borrow() < config.num_connections { + spawn_local(async move { + if let Err(err) = Connection::run(config, tls_config, load_test_state, num_active_connections).await { + eprintln!("connection creation error: {:?}", err); + } + }).detach(); + } + + Some(Duration::from_secs(1)) +} + struct Connection { config: Rc, load_test_state: LoadTestState, @@ -148,7 +158,7 @@ impl Connection { break; } Err(err) => { - break; + panic!("tls.reader().read: {}", err); } } } From 49ed4371e7e7784e24aab28f9e6e0e4c46978342 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20Frosteg=C3=A5rd?= Date: Thu, 28 Oct 2021 01:23:43 +0200 Subject: [PATCH 48/59] Run cargo fmt, clean up imports --- aquatic_common/src/cpu_pinning.rs | 2 +- aquatic_common/src/privileges.rs | 11 +++-- aquatic_http/src/lib/config.rs | 2 +- aquatic_http/src/lib/glommio/common.rs | 9 ++-- aquatic_http/src/lib/glommio/mod.rs | 10 ++-- aquatic_http/src/lib/glommio/network.rs | 64 +++++++++++++----------- aquatic_http_load_test/src/main.rs | 16 +++--- aquatic_http_load_test/src/network.rs | 66 +++++++++++++++---------- aquatic_udp/src/lib/config.rs | 2 +- aquatic_udp/src/lib/glommio/common.rs | 7 ++- aquatic_udp/src/lib/glommio/mod.rs | 10 ++-- aquatic_udp/src/lib/lib.rs | 2 +- aquatic_udp/src/lib/mio/mod.rs | 7 ++- 13 files changed, 125 insertions(+), 83 deletions(-) diff --git a/aquatic_common/src/cpu_pinning.rs b/aquatic_common/src/cpu_pinning.rs index 386e52b..a13fc73 100644 --- a/aquatic_common/src/cpu_pinning.rs +++ b/aquatic_common/src/cpu_pinning.rs @@ -1,4 +1,4 @@ -use serde::{Serialize, Deserialize}; +use serde::{Deserialize, Serialize}; #[derive(Clone, Debug, Serialize, Deserialize)] #[serde(default)] diff --git a/aquatic_common/src/privileges.rs b/aquatic_common/src/privileges.rs index 67b9e5b..a898969 100644 --- a/aquatic_common/src/privileges.rs +++ b/aquatic_common/src/privileges.rs @@ -1,7 +1,13 @@ -use std::{sync::{Arc, atomic::{AtomicUsize, Ordering}}, time::Duration}; +use std::{ + sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, + }, + time::Duration, +}; use privdrop::PrivDrop; -use serde::{Serialize, Deserialize}; +use serde::{Deserialize, Serialize}; #[derive(Clone, Debug, Serialize, Deserialize)] #[serde(default)] @@ -56,4 +62,3 @@ pub fn drop_privileges_after_socket_binding( Ok(()) } - diff --git a/aquatic_http/src/lib/config.rs b/aquatic_http/src/lib/config.rs index 33068a4..1e60db5 100644 --- a/aquatic_http/src/lib/config.rs +++ b/aquatic_http/src/lib/config.rs @@ -1,7 +1,7 @@ use std::{net::SocketAddr, path::PathBuf}; -use aquatic_common::{access_list::AccessListConfig, privileges::PrivilegeConfig}; use aquatic_common::cpu_pinning::CpuPinningConfig; +use aquatic_common::{access_list::AccessListConfig, privileges::PrivilegeConfig}; use serde::{Deserialize, Serialize}; use aquatic_cli_helpers::LogLevel; diff --git a/aquatic_http/src/lib/glommio/common.rs b/aquatic_http/src/lib/glommio/common.rs index 7495165..3887b9f 100644 --- a/aquatic_http/src/lib/glommio/common.rs +++ b/aquatic_http/src/lib/glommio/common.rs @@ -1,7 +1,7 @@ use std::borrow::Borrow; use std::cell::RefCell; -use std::rc::Rc; use std::net::SocketAddr; +use std::rc::Rc; use aquatic_common::access_list::AccessList; use futures_lite::AsyncBufReadExt; @@ -66,7 +66,10 @@ impl ChannelResponse { } } -pub async fn update_access_list>(config: C, access_list: Rc>) { +pub async fn update_access_list>( + config: C, + access_list: Rc>, +) { if config.borrow().access_list.mode.is_on() { match BufferedFile::open(&config.borrow().access_list.path).await { Ok(file) => { @@ -104,5 +107,3 @@ pub async fn update_access_list>(config: C, access_list: Rc anyhow::Result<()> { let mut builder = LocalExecutorBuilder::default(); if config.cpu_pinning.active { - builder = - builder.pin_to_cpu(config.cpu_pinning.offset + 1 + config.socket_workers + i); + builder = builder.pin_to_cpu(config.cpu_pinning.offset + 1 + config.socket_workers + i); } let executor = builder.spawn(|| async move { @@ -94,7 +93,12 @@ pub fn run(config: Config) -> anyhow::Result<()> { executors.push(executor); } - drop_privileges_after_socket_binding(&config.privileges, num_bound_sockets, config.socket_workers).unwrap(); + drop_privileges_after_socket_binding( + &config.privileges, + num_bound_sockets, + config.socket_workers, + ) + .unwrap(); for executor in executors { executor diff --git a/aquatic_http/src/lib/glommio/network.rs b/aquatic_http/src/lib/glommio/network.rs index 085d417..4efbab7 100644 --- a/aquatic_http/src/lib/glommio/network.rs +++ b/aquatic_http/src/lib/glommio/network.rs @@ -9,7 +9,7 @@ use std::time::Duration; use aquatic_common::access_list::AccessList; use aquatic_http_protocol::common::InfoHash; -use aquatic_http_protocol::request::{AnnounceRequest, Request, RequestParseError, ScrapeRequest}; +use aquatic_http_protocol::request::{Request, RequestParseError, ScrapeRequest}; use aquatic_http_protocol::response::{ FailureResponse, Response, ScrapeResponse, ScrapeStatistics, }; @@ -19,9 +19,9 @@ use glommio::channels::channel_mesh::{MeshBuilder, Partial, Role, Senders}; use glommio::channels::local_channel::{new_bounded, LocalReceiver, LocalSender}; use glommio::channels::shared_channel::ConnectedReceiver; use glommio::net::{TcpListener, TcpStream}; -use glommio::{enclose, prelude::*}; use glommio::task::JoinHandle; use glommio::timer::TimerActionRepeat; +use glommio::{enclose, prelude::*}; use rustls::ServerConnection; use slab::Slab; @@ -89,24 +89,26 @@ pub async fn run_socket_worker( })); // Periodically remove closed connections - TimerActionRepeat::repeat(enclose!((config, connection_slab, connections_to_remove) move || { - enclose!((config, connection_slab, connections_to_remove) move || async move { - let connections_to_remove = connections_to_remove.replace(Vec::new()); + TimerActionRepeat::repeat( + enclose!((config, connection_slab, connections_to_remove) move || { + enclose!((config, connection_slab, connections_to_remove) move || async move { + let connections_to_remove = connections_to_remove.replace(Vec::new()); - for connection_id in connections_to_remove { - if let Some(_) = connection_slab.borrow_mut().try_remove(connection_id) { - ::log::debug!("removed connection with id {}", connection_id); - } else { - ::log::error!( - "couldn't remove connection with id {}, it is not in connection slab", - connection_id - ); + for connection_id in connections_to_remove { + if let Some(_) = connection_slab.borrow_mut().try_remove(connection_id) { + ::log::debug!("removed connection with id {}", connection_id); + } else { + ::log::error!( + "couldn't remove connection with id {}, it is not in connection slab", + connection_id + ); + } } - } - Some(Duration::from_secs(config.cleaning.interval)) - })() - })); + Some(Duration::from_secs(config.cleaning.interval)) + })() + }), + ); for (_, response_receiver) in response_receivers.streams() { spawn_local(receive_responses( @@ -148,7 +150,8 @@ pub async fn run_socket_worker( } connections_to_remove.borrow_mut().push(key); - }).detach(); + }) + .detach(); let connection_reference = ConnectionReference { response_sender, @@ -188,15 +191,12 @@ impl Connection { match self.read_tls().await? { Some(Either::Left(request)) => { let response = match self.handle_request(request).await? { - Some(Either::Left(response)) => { - response - } + Some(Either::Left(response)) => response, Some(Either::Right(pending_scrape_response)) => { - self.wait_for_response(Some(pending_scrape_response)).await? - }, - None => { - self.wait_for_response(None).await? + self.wait_for_response(Some(pending_scrape_response)) + .await? } + None => self.wait_for_response(None).await?, }; self.queue_response(&response)?; @@ -257,7 +257,8 @@ impl Connection { if end > self.request_buffer.len() { return Err(anyhow::anyhow!("request too large")); } else { - let request_buffer_slice = &mut self.request_buffer[self.request_buffer_position..end]; + let request_buffer_slice = + &mut self.request_buffer[self.request_buffer_position..end]; request_buffer_slice.copy_from_slice(&buf[..amt]); @@ -341,7 +342,7 @@ impl Connection { /// relevant request workers, and return PendingScrapeResponse struct. async fn handle_request( &self, - request: Request + request: Request, ) -> anyhow::Result>> { let peer_addr = self.get_peer_addr()?; @@ -349,8 +350,11 @@ impl Connection { Request::Announce(request) => { let info_hash = request.info_hash; - if self.access_list.borrow().allows(self.config.access_list.mode, &info_hash.0) { - + if self + .access_list + .borrow() + .allows(self.config.access_list.mode, &info_hash.0) + { let request = ChannelRequest::Announce { request, connection_id: self.connection_id, @@ -417,7 +421,7 @@ impl Connection { /// return full response async fn wait_for_response( &self, - mut opt_pending_scrape_response: Option + mut opt_pending_scrape_response: Option, ) -> anyhow::Result { loop { if let Some(channel_response) = self.response_receiver.recv().await { diff --git a/aquatic_http_load_test/src/main.rs b/aquatic_http_load_test/src/main.rs index b0d3120..e719f77 100644 --- a/aquatic_http_load_test/src/main.rs +++ b/aquatic_http_load_test/src/main.rs @@ -61,9 +61,11 @@ fn run(config: Config) -> ::anyhow::Result<()> { let tls_config = tls_config.clone(); let state = state.clone(); - LocalExecutorBuilder::default().spawn(|| async move { - run_socket_thread(config, tls_config, state).await.unwrap(); - }).unwrap(); + LocalExecutorBuilder::default() + .spawn(|| async move { + run_socket_thread(config, tls_config, state).await.unwrap(); + }) + .unwrap(); } monitor_statistics(state, &config); @@ -175,8 +177,10 @@ fn create_tls_config() -> anyhow::Result> { .with_safe_defaults() .with_root_certificates(rustls::RootCertStore::empty()) .with_no_client_auth(); - - config.dangerous().set_certificate_verifier(Arc::new(FakeCertificateVerifier)); - + + config + .dangerous() + .set_certificate_verifier(Arc::new(FakeCertificateVerifier)); + Ok(Arc::new(config)) } diff --git a/aquatic_http_load_test/src/network.rs b/aquatic_http_load_test/src/network.rs index 5c38dc2..aaa15c0 100644 --- a/aquatic_http_load_test/src/network.rs +++ b/aquatic_http_load_test/src/network.rs @@ -1,10 +1,17 @@ -use std::{cell::RefCell, convert::TryInto, io::{Cursor, ErrorKind, Read}, rc::Rc, sync::{Arc, atomic::Ordering}, time::Duration}; +use std::{ + cell::RefCell, + convert::TryInto, + io::{Cursor, ErrorKind, Read}, + rc::Rc, + sync::{atomic::Ordering, Arc}, + time::Duration, +}; use aquatic_http_protocol::response::Response; use futures_lite::{AsyncReadExt, AsyncWriteExt}; -use glommio::{prelude::*, timer::TimerActionRepeat}; use glommio::net::TcpStream; -use rand::{SeedableRng, prelude::SmallRng}; +use glommio::{prelude::*, timer::TimerActionRepeat}; +use rand::{prelude::SmallRng, SeedableRng}; use rustls::ClientConnection; use crate::{common::LoadTestState, config::Config, utils::create_random_request}; @@ -17,12 +24,14 @@ pub async fn run_socket_thread( let config = Rc::new(config); let num_active_connections = Rc::new(RefCell::new(0usize)); - TimerActionRepeat::repeat(move || periodically_open_connections( - config.clone(), - tls_config.clone(), - load_test_state.clone(), - num_active_connections.clone()) - ); + TimerActionRepeat::repeat(move || { + periodically_open_connections( + config.clone(), + tls_config.clone(), + load_test_state.clone(), + num_active_connections.clone(), + ) + }); futures_lite::future::pending::().await; @@ -37,10 +46,13 @@ async fn periodically_open_connections( ) -> Option { if *num_active_connections.borrow() < config.num_connections { spawn_local(async move { - if let Err(err) = Connection::run(config, tls_config, load_test_state, num_active_connections).await { + if let Err(err) = + Connection::run(config, tls_config, load_test_state, num_active_connections).await + { eprintln!("connection creation error: {:?}", err); } - }).detach(); + }) + .detach(); } Some(Duration::from_secs(1)) @@ -65,7 +77,8 @@ impl Connection { load_test_state: LoadTestState, num_active_connections: Rc>, ) -> anyhow::Result<()> { - let stream = TcpStream::connect(config.server_address).await + let stream = TcpStream::connect(config.server_address) + .await .map_err(|err| anyhow::anyhow!("connect: {:?}", err))?; let tls = ClientConnection::new(tls_config, "example.com".try_into().unwrap()).unwrap(); let rng = SmallRng::from_entropy(); @@ -98,7 +111,8 @@ impl Connection { async fn run_connection_loop(&mut self) -> anyhow::Result<()> { loop { if self.send_new_request { - let request = create_random_request(&self.config, &self.load_test_state, &mut self.rng); + let request = + create_random_request(&self.config, &self.load_test_state, &mut self.rng); request.write(&mut self.tls.writer())?; self.queued_responses += 1; @@ -121,8 +135,7 @@ impl Connection { return Err(anyhow::anyhow!("Peer has closed connection")); } - self - .load_test_state + self.load_test_state .statistics .bytes_received .fetch_add(bytes_read, Ordering::SeqCst); @@ -145,7 +158,8 @@ impl Connection { if end > self.response_buffer.len() { return Err(anyhow::anyhow!("response too large")); } else { - let response_buffer_slice = &mut self.response_buffer[self.response_buffer_position..end]; + let response_buffer_slice = + &mut self.response_buffer[self.response_buffer_position..end]; response_buffer_slice.copy_from_slice(&buf[..amt]); @@ -182,25 +196,21 @@ impl Connection { match Response::from_bytes(interesting_bytes) { Ok(response) => { - match response { Response::Announce(_) => { - self - .load_test_state + self.load_test_state .statistics .responses_announce .fetch_add(1, Ordering::SeqCst); } Response::Scrape(_) => { - self - .load_test_state + self.load_test_state .statistics .responses_scrape .fetch_add(1, Ordering::SeqCst); } Response::Failure(response) => { - self - .load_test_state + self.load_test_state .statistics .responses_failure .fetch_add(1, Ordering::SeqCst); @@ -253,18 +263,20 @@ impl Connection { self.stream.write_all(&buf.into_inner()).await?; self.stream.flush().await?; - self - .load_test_state + self.load_test_state .statistics .bytes_sent .fetch_add(len, Ordering::SeqCst); if self.queued_responses != 0 { - self.load_test_state.statistics.requests.fetch_add(self.queued_responses, Ordering::SeqCst); + self.load_test_state + .statistics + .requests + .fetch_add(self.queued_responses, Ordering::SeqCst); self.queued_responses = 0; } Ok(()) } -} \ No newline at end of file +} diff --git a/aquatic_udp/src/lib/config.rs b/aquatic_udp/src/lib/config.rs index 2f1540a..dbda176 100644 --- a/aquatic_udp/src/lib/config.rs +++ b/aquatic_udp/src/lib/config.rs @@ -1,7 +1,7 @@ use std::net::SocketAddr; -use aquatic_common::{access_list::AccessListConfig, privileges::PrivilegeConfig}; use aquatic_common::cpu_pinning::CpuPinningConfig; +use aquatic_common::{access_list::AccessListConfig, privileges::PrivilegeConfig}; use serde::{Deserialize, Serialize}; use aquatic_cli_helpers::LogLevel; diff --git a/aquatic_udp/src/lib/glommio/common.rs b/aquatic_udp/src/lib/glommio/common.rs index 393038d..4b3d80d 100644 --- a/aquatic_udp/src/lib/glommio/common.rs +++ b/aquatic_udp/src/lib/glommio/common.rs @@ -9,7 +9,10 @@ use glommio::prelude::*; use crate::common::*; use crate::config::Config; -pub async fn update_access_list>(config: C, access_list: Rc>) { +pub async fn update_access_list>( + config: C, + access_list: Rc>, +) { if config.borrow().access_list.mode.is_on() { match BufferedFile::open(&config.borrow().access_list.path).await { Ok(file) => { @@ -46,4 +49,4 @@ pub async fn update_access_list>(config: C, access_list: Rc anyhow::Result<()> { let mut builder = LocalExecutorBuilder::default(); if config.cpu_pinning.active { - builder = - builder.pin_to_cpu(config.cpu_pinning.offset + 1 + config.socket_workers + i); + builder = builder.pin_to_cpu(config.cpu_pinning.offset + 1 + config.socket_workers + i); } let executor = builder.spawn(|| async move { @@ -88,7 +87,12 @@ pub fn run(config: Config) -> anyhow::Result<()> { executors.push(executor); } - drop_privileges_after_socket_binding(&config.privileges, num_bound_sockets, config.socket_workers).unwrap(); + drop_privileges_after_socket_binding( + &config.privileges, + num_bound_sockets, + config.socket_workers, + ) + .unwrap(); for executor in executors { executor diff --git a/aquatic_udp/src/lib/lib.rs b/aquatic_udp/src/lib/lib.rs index e7796d4..78676c2 100644 --- a/aquatic_udp/src/lib/lib.rs +++ b/aquatic_udp/src/lib/lib.rs @@ -27,4 +27,4 @@ pub fn run(config: Config) -> ::anyhow::Result<()> { mio::run(config) } } -} \ No newline at end of file +} diff --git a/aquatic_udp/src/lib/mio/mod.rs b/aquatic_udp/src/lib/mio/mod.rs index 3127ce2..1b8e656 100644 --- a/aquatic_udp/src/lib/mio/mod.rs +++ b/aquatic_udp/src/lib/mio/mod.rs @@ -35,7 +35,12 @@ pub fn run(config: Config) -> ::anyhow::Result<()> { start_workers(config.clone(), state.clone(), num_bound_sockets.clone())?; - drop_privileges_after_socket_binding(&config.privileges, num_bound_sockets, config.socket_workers).unwrap(); + drop_privileges_after_socket_binding( + &config.privileges, + num_bound_sockets, + config.socket_workers, + ) + .unwrap(); loop { ::std::thread::sleep(Duration::from_secs(config.cleaning.interval)); From a935a3bbf0f32cf76c86587e7dc16e67f3fc5501 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20Frosteg=C3=A5rd?= Date: Thu, 28 Oct 2021 01:33:23 +0200 Subject: [PATCH 49/59] Update TODO --- TODO.md | 1 - 1 file changed, 1 deletion(-) diff --git a/TODO.md b/TODO.md index 1f85f70..3eb6ebc 100644 --- a/TODO.md +++ b/TODO.md @@ -1,7 +1,6 @@ # TODO * aquatic_http glommio: - * test with load tester with multiple workers: requires tls load tester * remove mio version * get rid of / improve ConnectionMeta stuff in handler * clean out connections regularly From 130377b8f4219b508f7f687aad293278495d928d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20Frosteg=C3=A5rd?= Date: Thu, 28 Oct 2021 01:48:32 +0200 Subject: [PATCH 50/59] aquatic_http: remove mio implementation --- Cargo.lock | 5 - TODO.md | 3 +- aquatic_http/Cargo.toml | 24 +- .../src/lib/{common/mod.rs => common.rs} | 109 ++++- aquatic_http/src/lib/glommio/common.rs | 109 ----- aquatic_http/src/lib/glommio/handlers.rs | 149 ------- aquatic_http/src/lib/glommio/mod.rs | 140 ------- aquatic_http/src/lib/{common => }/handlers.rs | 147 ++++++- aquatic_http/src/lib/lib.rs | 150 ++++++- aquatic_http/src/lib/mio/common.rs | 57 --- aquatic_http/src/lib/mio/handler.rs | 97 ----- aquatic_http/src/lib/mio/mod.rs | 150 ------- .../src/lib/mio/network/connection.rs | 260 ------------ aquatic_http/src/lib/mio/network/mod.rs | 388 ------------------ aquatic_http/src/lib/mio/network/stream.rs | 69 ---- aquatic_http/src/lib/mio/network/utils.rs | 63 --- aquatic_http/src/lib/mio/tasks.rs | 53 --- aquatic_http/src/lib/{glommio => }/network.rs | 0 aquatic_udp/src/lib/lib.rs | 8 - 19 files changed, 394 insertions(+), 1587 deletions(-) rename aquatic_http/src/lib/{common/mod.rs => common.rs} (61%) delete mode 100644 aquatic_http/src/lib/glommio/common.rs delete mode 100644 aquatic_http/src/lib/glommio/handlers.rs delete mode 100644 aquatic_http/src/lib/glommio/mod.rs rename aquatic_http/src/lib/{common => }/handlers.rs (59%) delete mode 100644 aquatic_http/src/lib/mio/common.rs delete mode 100644 aquatic_http/src/lib/mio/handler.rs delete mode 100644 aquatic_http/src/lib/mio/mod.rs delete mode 100644 aquatic_http/src/lib/mio/network/connection.rs delete mode 100644 aquatic_http/src/lib/mio/network/mod.rs delete mode 100644 aquatic_http/src/lib/mio/network/stream.rs delete mode 100644 aquatic_http/src/lib/mio/network/utils.rs delete mode 100644 aquatic_http/src/lib/mio/tasks.rs rename aquatic_http/src/lib/{glommio => }/network.rs (100%) diff --git a/Cargo.lock b/Cargo.lock index d76d7ea..826d684 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -94,19 +94,15 @@ dependencies = [ "aquatic_http_protocol", "cfg-if", "core_affinity", - "crossbeam-channel", "either", "futures-lite", "glommio", "hashbrown 0.11.2", - "histogram", "indexmap", "itoa", "log", "memchr", "mimalloc", - "mio", - "native-tls", "parking_lot", "privdrop", "quickcheck", @@ -117,7 +113,6 @@ dependencies = [ "serde", "slab", "smartstring", - "socket2 0.4.2", ] [[package]] diff --git a/TODO.md b/TODO.md index 3eb6ebc..fcb5aa3 100644 --- a/TODO.md +++ b/TODO.md @@ -1,8 +1,7 @@ # TODO * aquatic_http glommio: - * remove mio version - * get rid of / improve ConnectionMeta stuff in handler + * get rid of / improve ConnectionMeta stuff in handler * clean out connections regularly * timeout inside of task for "it took to long to receive request, send response"? * handle panicked/cancelled tasks diff --git a/aquatic_http/Cargo.toml b/aquatic_http/Cargo.toml index 7c9a9cc..e969d83 100644 --- a/aquatic_http/Cargo.toml +++ b/aquatic_http/Cargo.toml @@ -15,11 +15,6 @@ path = "src/lib/lib.rs" name = "aquatic_http" path = "src/bin/main.rs" -[features] -default = ["with-glommio"] -with-glommio = ["glommio", "futures-lite", "rustls", "rustls-pemfile", "slab"] -with-mio = ["crossbeam-channel", "histogram", "mio", "native-tls", "socket2"] - [dependencies] anyhow = "1" aquatic_cli_helpers = "0.1.0" @@ -28,6 +23,8 @@ aquatic_http_protocol = "0.1.0" cfg-if = "1" core_affinity = "0.5" either = "1" +futures-lite = "1" +glommio = { git = "https://github.com/DataDog/glommio.git", rev = "4e6b14772da2f4325271fbcf12d24cf91ed466e5" } hashbrown = "0.11.2" indexmap = "1" itoa = "0.4" @@ -37,23 +34,12 @@ memchr = "2" parking_lot = "0.11" privdrop = "0.5" rand = { version = "0.8", features = ["small_rng"] } +rustls = "0.20" +rustls-pemfile = "0.2" serde = { version = "1", features = ["derive"] } +slab = "0.4" smartstring = "0.2" -# mio -crossbeam-channel = { version = "0.5", optional = true } -histogram = { version = "0.6", optional = true } -mio = { version = "0.7", features = ["tcp", "os-poll", "os-util"], optional = true } -native-tls = { version = "0.2", optional = true } -socket2 = { version = "0.4.1", features = ["all"], optional = true } - -# glommio -futures-lite = { version = "1", optional = true } -glommio = { git = "https://github.com/DataDog/glommio.git", rev = "4e6b14772da2f4325271fbcf12d24cf91ed466e5", optional = true } -rustls = { version = "0.20", optional = true } -rustls-pemfile = { version = "0.2", optional = true } -slab = { version = "0.4", optional = true } - [dev-dependencies] quickcheck = "1.0" quickcheck_macros = "1.0" diff --git a/aquatic_http/src/lib/common/mod.rs b/aquatic_http/src/lib/common.rs similarity index 61% rename from aquatic_http/src/lib/common/mod.rs rename to aquatic_http/src/lib/common.rs index b9c26b7..67edb55 100644 --- a/aquatic_http/src/lib/common/mod.rs +++ b/aquatic_http/src/lib/common.rs @@ -14,7 +14,112 @@ use aquatic_http_protocol::response::ResponsePeer; use crate::config::Config; -pub mod handlers; +use std::borrow::Borrow; +use std::cell::RefCell; +use std::rc::Rc; + +use futures_lite::AsyncBufReadExt; +use glommio::io::{BufferedFile, StreamReaderBuilder}; +use glommio::prelude::*; + +use aquatic_http_protocol::{ + request::{AnnounceRequest, ScrapeRequest}, + response::{AnnounceResponse, ScrapeResponse}, +}; + + +#[derive(Copy, Clone, Debug)] +pub struct ConsumerId(pub usize); + +#[derive(Clone, Copy, Debug)] +pub struct ConnectionId(pub usize); + +#[derive(Debug)] +pub enum ChannelRequest { + Announce { + request: AnnounceRequest, + peer_addr: SocketAddr, + connection_id: ConnectionId, + response_consumer_id: ConsumerId, + }, + Scrape { + request: ScrapeRequest, + peer_addr: SocketAddr, + connection_id: ConnectionId, + response_consumer_id: ConsumerId, + }, +} + +#[derive(Debug)] +pub enum ChannelResponse { + Announce { + response: AnnounceResponse, + peer_addr: SocketAddr, + connection_id: ConnectionId, + }, + Scrape { + response: ScrapeResponse, + peer_addr: SocketAddr, + connection_id: ConnectionId, + }, +} + +impl ChannelResponse { + pub fn get_connection_id(&self) -> ConnectionId { + match self { + Self::Announce { connection_id, .. } => *connection_id, + Self::Scrape { connection_id, .. } => *connection_id, + } + } + pub fn get_peer_addr(&self) -> SocketAddr { + match self { + Self::Announce { peer_addr, .. } => *peer_addr, + Self::Scrape { peer_addr, .. } => *peer_addr, + } + } +} + +pub async fn update_access_list>( + config: C, + access_list: Rc>, +) { + if config.borrow().access_list.mode.is_on() { + match BufferedFile::open(&config.borrow().access_list.path).await { + Ok(file) => { + let mut reader = StreamReaderBuilder::new(file).build(); + let mut new_access_list = AccessList::default(); + + loop { + let mut buf = String::with_capacity(42); + + match reader.read_line(&mut buf).await { + Ok(_) => { + if let Err(err) = new_access_list.insert_from_line(&buf) { + ::log::error!( + "Couln't parse access list line '{}': {:?}", + buf, + err + ); + } + } + Err(err) => { + ::log::error!("Couln't read access list line {:?}", err); + + break; + } + } + + yield_if_needed().await; + } + + *access_list.borrow_mut() = new_access_list; + } + Err(err) => { + ::log::error!("Couldn't open access list file: {:?}", err) + } + }; + } +} pub trait Ip: ::std::fmt::Debug + Copy + Eq + ::std::hash::Hash {} @@ -186,4 +291,4 @@ mod tests { assert_eq!(f(101), 3); assert_eq!(f(1000), 4); } -} +} \ No newline at end of file diff --git a/aquatic_http/src/lib/glommio/common.rs b/aquatic_http/src/lib/glommio/common.rs deleted file mode 100644 index 3887b9f..0000000 --- a/aquatic_http/src/lib/glommio/common.rs +++ /dev/null @@ -1,109 +0,0 @@ -use std::borrow::Borrow; -use std::cell::RefCell; -use std::net::SocketAddr; -use std::rc::Rc; - -use aquatic_common::access_list::AccessList; -use futures_lite::AsyncBufReadExt; -use glommio::io::{BufferedFile, StreamReaderBuilder}; -use glommio::prelude::*; - -use aquatic_http_protocol::{ - request::{AnnounceRequest, ScrapeRequest}, - response::{AnnounceResponse, ScrapeResponse}, -}; - -use crate::config::Config; - -#[derive(Copy, Clone, Debug)] -pub struct ConsumerId(pub usize); - -#[derive(Clone, Copy, Debug)] -pub struct ConnectionId(pub usize); - -#[derive(Debug)] -pub enum ChannelRequest { - Announce { - request: AnnounceRequest, - peer_addr: SocketAddr, - connection_id: ConnectionId, - response_consumer_id: ConsumerId, - }, - Scrape { - request: ScrapeRequest, - peer_addr: SocketAddr, - connection_id: ConnectionId, - response_consumer_id: ConsumerId, - }, -} - -#[derive(Debug)] -pub enum ChannelResponse { - Announce { - response: AnnounceResponse, - peer_addr: SocketAddr, - connection_id: ConnectionId, - }, - Scrape { - response: ScrapeResponse, - peer_addr: SocketAddr, - connection_id: ConnectionId, - }, -} - -impl ChannelResponse { - pub fn get_connection_id(&self) -> ConnectionId { - match self { - Self::Announce { connection_id, .. } => *connection_id, - Self::Scrape { connection_id, .. } => *connection_id, - } - } - pub fn get_peer_addr(&self) -> SocketAddr { - match self { - Self::Announce { peer_addr, .. } => *peer_addr, - Self::Scrape { peer_addr, .. } => *peer_addr, - } - } -} - -pub async fn update_access_list>( - config: C, - access_list: Rc>, -) { - if config.borrow().access_list.mode.is_on() { - match BufferedFile::open(&config.borrow().access_list.path).await { - Ok(file) => { - let mut reader = StreamReaderBuilder::new(file).build(); - let mut new_access_list = AccessList::default(); - - loop { - let mut buf = String::with_capacity(42); - - match reader.read_line(&mut buf).await { - Ok(_) => { - if let Err(err) = new_access_list.insert_from_line(&buf) { - ::log::error!( - "Couln't parse access list line '{}': {:?}", - buf, - err - ); - } - } - Err(err) => { - ::log::error!("Couln't read access list line {:?}", err); - - break; - } - } - - yield_if_needed().await; - } - - *access_list.borrow_mut() = new_access_list; - } - Err(err) => { - ::log::error!("Couldn't open access list file: {:?}", err) - } - }; - } -} diff --git a/aquatic_http/src/lib/glommio/handlers.rs b/aquatic_http/src/lib/glommio/handlers.rs deleted file mode 100644 index 21a4daf..0000000 --- a/aquatic_http/src/lib/glommio/handlers.rs +++ /dev/null @@ -1,149 +0,0 @@ -use std::cell::RefCell; -use std::rc::Rc; -use std::time::Duration; - -use aquatic_common::access_list::AccessList; -use futures_lite::{Stream, StreamExt}; -use glommio::channels::channel_mesh::{MeshBuilder, Partial, Role, Senders}; -use glommio::timer::TimerActionRepeat; -use glommio::{enclose, prelude::*}; -use rand::prelude::SmallRng; -use rand::SeedableRng; - -use crate::common::handlers::handle_announce_request; -use crate::common::handlers::*; -use crate::common::*; -use crate::config::Config; - -use super::common::*; - -pub async fn run_request_worker( - config: Config, - request_mesh_builder: MeshBuilder, - response_mesh_builder: MeshBuilder, - access_list: AccessList, -) { - let (_, mut request_receivers) = request_mesh_builder.join(Role::Consumer).await.unwrap(); - let (response_senders, _) = response_mesh_builder.join(Role::Producer).await.unwrap(); - - let response_senders = Rc::new(response_senders); - - let torrents = Rc::new(RefCell::new(TorrentMaps::default())); - let access_list = Rc::new(RefCell::new(access_list)); - - // Periodically clean torrents and update access list - TimerActionRepeat::repeat(enclose!((config, torrents, access_list) move || { - enclose!((config, torrents, access_list) move || async move { - update_access_list(&config, access_list.clone()).await; - - torrents.borrow_mut().clean(&config, &*access_list.borrow()); - - Some(Duration::from_secs(config.cleaning.interval)) - })() - })); - - let mut handles = Vec::new(); - - for (_, receiver) in request_receivers.streams() { - let handle = spawn_local(handle_request_stream( - config.clone(), - torrents.clone(), - response_senders.clone(), - receiver, - )) - .detach(); - - handles.push(handle); - } - - for handle in handles { - handle.await; - } -} - -async fn handle_request_stream( - config: Config, - torrents: Rc>, - response_senders: Rc>, - mut stream: S, -) where - S: Stream + ::std::marker::Unpin, -{ - let mut rng = SmallRng::from_entropy(); - - let max_peer_age = config.cleaning.max_peer_age; - let peer_valid_until = Rc::new(RefCell::new(ValidUntil::new(max_peer_age))); - - TimerActionRepeat::repeat(enclose!((peer_valid_until) move || { - enclose!((peer_valid_until) move || async move { - *peer_valid_until.borrow_mut() = ValidUntil::new(max_peer_age); - - Some(Duration::from_secs(1)) - })() - })); - - while let Some(channel_request) = stream.next().await { - let (response, consumer_id) = match channel_request { - ChannelRequest::Announce { - request, - peer_addr, - response_consumer_id, - connection_id, - } => { - let meta = ConnectionMeta { - worker_index: response_consumer_id.0, - poll_token: connection_id.0, - peer_addr, - }; - - let response = handle_announce_request( - &config, - &mut rng, - &mut torrents.borrow_mut(), - peer_valid_until.borrow().to_owned(), - meta, - request, - ); - - let response = ChannelResponse::Announce { - response, - peer_addr, - connection_id, - }; - - (response, response_consumer_id) - } - ChannelRequest::Scrape { - request, - peer_addr, - response_consumer_id, - connection_id, - } => { - let meta = ConnectionMeta { - worker_index: response_consumer_id.0, - poll_token: connection_id.0, - peer_addr, - }; - - let response = - handle_scrape_request(&config, &mut torrents.borrow_mut(), meta, request); - - let response = ChannelResponse::Scrape { - response, - peer_addr, - connection_id, - }; - - (response, response_consumer_id) - } - }; - - ::log::debug!("preparing to send response to channel: {:?}", response); - - if let Err(err) = response_senders.try_send_to(consumer_id.0, response) { - ::log::warn!("response_sender.try_send: {:?}", err); - } - - yield_if_needed().await; - } -} diff --git a/aquatic_http/src/lib/glommio/mod.rs b/aquatic_http/src/lib/glommio/mod.rs deleted file mode 100644 index 573004e..0000000 --- a/aquatic_http/src/lib/glommio/mod.rs +++ /dev/null @@ -1,140 +0,0 @@ -use std::{ - fs::File, - io::BufReader, - sync::{atomic::AtomicUsize, Arc}, -}; - -use aquatic_common::{access_list::AccessList, privileges::drop_privileges_after_socket_binding}; -use glommio::{channels::channel_mesh::MeshBuilder, prelude::*}; - -use crate::config::Config; - -mod common; -mod handlers; -mod network; - -const SHARED_CHANNEL_SIZE: usize = 1024; - -pub fn run(config: Config) -> anyhow::Result<()> { - if config.cpu_pinning.active { - core_affinity::set_for_current(core_affinity::CoreId { - id: config.cpu_pinning.offset, - }); - } - - let access_list = if config.access_list.mode.is_on() { - AccessList::create_from_path(&config.access_list.path).expect("Load access list") - } else { - AccessList::default() - }; - - let num_peers = config.socket_workers + config.request_workers; - - let request_mesh_builder = MeshBuilder::partial(num_peers, SHARED_CHANNEL_SIZE); - let response_mesh_builder = MeshBuilder::partial(num_peers, SHARED_CHANNEL_SIZE); - - let num_bound_sockets = Arc::new(AtomicUsize::new(0)); - - let tls_config = Arc::new(create_tls_config(&config).unwrap()); - - let mut executors = Vec::new(); - - for i in 0..(config.socket_workers) { - let config = config.clone(); - let tls_config = tls_config.clone(); - let request_mesh_builder = request_mesh_builder.clone(); - let response_mesh_builder = response_mesh_builder.clone(); - let num_bound_sockets = num_bound_sockets.clone(); - let access_list = access_list.clone(); - - let mut builder = LocalExecutorBuilder::default(); - - if config.cpu_pinning.active { - builder = builder.pin_to_cpu(config.cpu_pinning.offset + 1 + i); - } - - let executor = builder.spawn(|| async move { - network::run_socket_worker( - config, - tls_config, - request_mesh_builder, - response_mesh_builder, - num_bound_sockets, - access_list, - ) - .await - }); - - executors.push(executor); - } - - for i in 0..(config.request_workers) { - let config = config.clone(); - let request_mesh_builder = request_mesh_builder.clone(); - let response_mesh_builder = response_mesh_builder.clone(); - let access_list = access_list.clone(); - - let mut builder = LocalExecutorBuilder::default(); - - if config.cpu_pinning.active { - builder = builder.pin_to_cpu(config.cpu_pinning.offset + 1 + config.socket_workers + i); - } - - let executor = builder.spawn(|| async move { - handlers::run_request_worker( - config, - request_mesh_builder, - response_mesh_builder, - access_list, - ) - .await - }); - - executors.push(executor); - } - - drop_privileges_after_socket_binding( - &config.privileges, - num_bound_sockets, - config.socket_workers, - ) - .unwrap(); - - for executor in executors { - executor - .expect("failed to spawn local executor") - .join() - .unwrap(); - } - - Ok(()) -} - -fn create_tls_config(config: &Config) -> anyhow::Result { - let certs = { - let f = File::open(&config.network.tls.tls_certificate_path)?; - let mut f = BufReader::new(f); - - rustls_pemfile::certs(&mut f)? - .into_iter() - .map(|bytes| rustls::Certificate(bytes)) - .collect() - }; - - let private_key = { - let f = File::open(&config.network.tls.tls_private_key_path)?; - let mut f = BufReader::new(f); - - rustls_pemfile::pkcs8_private_keys(&mut f)? - .first() - .map(|bytes| rustls::PrivateKey(bytes.clone())) - .ok_or(anyhow::anyhow!("No private keys in file"))? - }; - - let tls_config = rustls::ServerConfig::builder() - .with_safe_defaults() - .with_no_client_auth() - .with_single_cert(certs, private_key)?; - - Ok(tls_config) -} diff --git a/aquatic_http/src/lib/common/handlers.rs b/aquatic_http/src/lib/handlers.rs similarity index 59% rename from aquatic_http/src/lib/common/handlers.rs rename to aquatic_http/src/lib/handlers.rs index 01f7bc0..b636fff 100644 --- a/aquatic_http/src/lib/common/handlers.rs +++ b/aquatic_http/src/lib/handlers.rs @@ -8,9 +8,152 @@ use aquatic_common::extract_response_peers; use aquatic_http_protocol::request::*; use aquatic_http_protocol::response::*; -use super::*; +use std::cell::RefCell; +use std::rc::Rc; +use std::time::Duration; + +use aquatic_common::access_list::AccessList; +use futures_lite::{Stream, StreamExt}; +use glommio::channels::channel_mesh::{MeshBuilder, Partial, Role, Senders}; +use glommio::timer::TimerActionRepeat; +use glommio::{enclose, prelude::*}; +use rand::prelude::SmallRng; +use rand::SeedableRng; + +use crate::common::*; use crate::config::Config; +pub async fn run_request_worker( + config: Config, + request_mesh_builder: MeshBuilder, + response_mesh_builder: MeshBuilder, + access_list: AccessList, +) { + let (_, mut request_receivers) = request_mesh_builder.join(Role::Consumer).await.unwrap(); + let (response_senders, _) = response_mesh_builder.join(Role::Producer).await.unwrap(); + + let response_senders = Rc::new(response_senders); + + let torrents = Rc::new(RefCell::new(TorrentMaps::default())); + let access_list = Rc::new(RefCell::new(access_list)); + + // Periodically clean torrents and update access list + TimerActionRepeat::repeat(enclose!((config, torrents, access_list) move || { + enclose!((config, torrents, access_list) move || async move { + update_access_list(&config, access_list.clone()).await; + + torrents.borrow_mut().clean(&config, &*access_list.borrow()); + + Some(Duration::from_secs(config.cleaning.interval)) + })() + })); + + let mut handles = Vec::new(); + + for (_, receiver) in request_receivers.streams() { + let handle = spawn_local(handle_request_stream( + config.clone(), + torrents.clone(), + response_senders.clone(), + receiver, + )) + .detach(); + + handles.push(handle); + } + + for handle in handles { + handle.await; + } +} + +async fn handle_request_stream( + config: Config, + torrents: Rc>, + response_senders: Rc>, + mut stream: S, +) where + S: Stream + ::std::marker::Unpin, +{ + let mut rng = SmallRng::from_entropy(); + + let max_peer_age = config.cleaning.max_peer_age; + let peer_valid_until = Rc::new(RefCell::new(ValidUntil::new(max_peer_age))); + + TimerActionRepeat::repeat(enclose!((peer_valid_until) move || { + enclose!((peer_valid_until) move || async move { + *peer_valid_until.borrow_mut() = ValidUntil::new(max_peer_age); + + Some(Duration::from_secs(1)) + })() + })); + + while let Some(channel_request) = stream.next().await { + let (response, consumer_id) = match channel_request { + ChannelRequest::Announce { + request, + peer_addr, + response_consumer_id, + connection_id, + } => { + let meta = ConnectionMeta { + worker_index: response_consumer_id.0, + poll_token: connection_id.0, + peer_addr, + }; + + let response = handle_announce_request( + &config, + &mut rng, + &mut torrents.borrow_mut(), + peer_valid_until.borrow().to_owned(), + meta, + request, + ); + + let response = ChannelResponse::Announce { + response, + peer_addr, + connection_id, + }; + + (response, response_consumer_id) + } + ChannelRequest::Scrape { + request, + peer_addr, + response_consumer_id, + connection_id, + } => { + let meta = ConnectionMeta { + worker_index: response_consumer_id.0, + poll_token: connection_id.0, + peer_addr, + }; + + let response = + handle_scrape_request(&config, &mut torrents.borrow_mut(), meta, request); + + let response = ChannelResponse::Scrape { + response, + peer_addr, + connection_id, + }; + + (response, response_consumer_id) + } + }; + + ::log::debug!("preparing to send response to channel: {:?}", response); + + if let Err(err) = response_senders.try_send_to(consumer_id.0, response) { + ::log::warn!("response_sender.try_send: {:?}", err); + } + + yield_if_needed().await; + } +} + pub fn handle_announce_request( config: &Config, rng: &mut impl Rng, @@ -214,4 +357,4 @@ pub fn handle_scrape_request( }; response -} +} \ No newline at end of file diff --git a/aquatic_http/src/lib/lib.rs b/aquatic_http/src/lib/lib.rs index 10cb683..62ac41f 100644 --- a/aquatic_http/src/lib/lib.rs +++ b/aquatic_http/src/lib/lib.rs @@ -1,21 +1,143 @@ -use cfg_if::cfg_if; +use std::{ + fs::File, + io::BufReader, + sync::{atomic::AtomicUsize, Arc}, +}; + +use aquatic_common::{access_list::AccessList, privileges::drop_privileges_after_socket_binding}; +use glommio::{channels::channel_mesh::MeshBuilder, prelude::*}; + +use crate::config::Config; -pub mod common; pub mod config; - -#[cfg(all(feature = "with-glommio", target_os = "linux"))] -pub mod glommio; -#[cfg(feature = "with-mio")] -pub mod mio; +mod common; +mod handlers; +mod network; pub const APP_NAME: &str = "aquatic_http: HTTP/TLS BitTorrent tracker"; -pub fn run(config: config::Config) -> ::anyhow::Result<()> { - cfg_if! { - if #[cfg(all(feature = "with-glommio", target_os = "linux"))] { - glommio::run(config) - } else { - mio::run(config) - } +const SHARED_CHANNEL_SIZE: usize = 1024; + +pub fn run(config: Config) -> anyhow::Result<()> { + if config.cpu_pinning.active { + core_affinity::set_for_current(core_affinity::CoreId { + id: config.cpu_pinning.offset, + }); } + + let access_list = if config.access_list.mode.is_on() { + AccessList::create_from_path(&config.access_list.path).expect("Load access list") + } else { + AccessList::default() + }; + + let num_peers = config.socket_workers + config.request_workers; + + let request_mesh_builder = MeshBuilder::partial(num_peers, SHARED_CHANNEL_SIZE); + let response_mesh_builder = MeshBuilder::partial(num_peers, SHARED_CHANNEL_SIZE); + + let num_bound_sockets = Arc::new(AtomicUsize::new(0)); + + let tls_config = Arc::new(create_tls_config(&config).unwrap()); + + let mut executors = Vec::new(); + + for i in 0..(config.socket_workers) { + let config = config.clone(); + let tls_config = tls_config.clone(); + let request_mesh_builder = request_mesh_builder.clone(); + let response_mesh_builder = response_mesh_builder.clone(); + let num_bound_sockets = num_bound_sockets.clone(); + let access_list = access_list.clone(); + + let mut builder = LocalExecutorBuilder::default(); + + if config.cpu_pinning.active { + builder = builder.pin_to_cpu(config.cpu_pinning.offset + 1 + i); + } + + let executor = builder.spawn(|| async move { + network::run_socket_worker( + config, + tls_config, + request_mesh_builder, + response_mesh_builder, + num_bound_sockets, + access_list, + ) + .await + }); + + executors.push(executor); + } + + for i in 0..(config.request_workers) { + let config = config.clone(); + let request_mesh_builder = request_mesh_builder.clone(); + let response_mesh_builder = response_mesh_builder.clone(); + let access_list = access_list.clone(); + + let mut builder = LocalExecutorBuilder::default(); + + if config.cpu_pinning.active { + builder = builder.pin_to_cpu(config.cpu_pinning.offset + 1 + config.socket_workers + i); + } + + let executor = builder.spawn(|| async move { + handlers::run_request_worker( + config, + request_mesh_builder, + response_mesh_builder, + access_list, + ) + .await + }); + + executors.push(executor); + } + + drop_privileges_after_socket_binding( + &config.privileges, + num_bound_sockets, + config.socket_workers, + ) + .unwrap(); + + for executor in executors { + executor + .expect("failed to spawn local executor") + .join() + .unwrap(); + } + + Ok(()) } + +fn create_tls_config(config: &Config) -> anyhow::Result { + let certs = { + let f = File::open(&config.network.tls.tls_certificate_path)?; + let mut f = BufReader::new(f); + + rustls_pemfile::certs(&mut f)? + .into_iter() + .map(|bytes| rustls::Certificate(bytes)) + .collect() + }; + + let private_key = { + let f = File::open(&config.network.tls.tls_private_key_path)?; + let mut f = BufReader::new(f); + + rustls_pemfile::pkcs8_private_keys(&mut f)? + .first() + .map(|bytes| rustls::PrivateKey(bytes.clone())) + .ok_or(anyhow::anyhow!("No private keys in file"))? + }; + + let tls_config = rustls::ServerConfig::builder() + .with_safe_defaults() + .with_no_client_auth() + .with_single_cert(certs, private_key)?; + + Ok(tls_config) +} \ No newline at end of file diff --git a/aquatic_http/src/lib/mio/common.rs b/aquatic_http/src/lib/mio/common.rs deleted file mode 100644 index ef224d6..0000000 --- a/aquatic_http/src/lib/mio/common.rs +++ /dev/null @@ -1,57 +0,0 @@ -use std::sync::Arc; - -use aquatic_common::access_list::AccessListArcSwap; -use crossbeam_channel::{Receiver, Sender}; -use log::error; -use mio::Token; -use parking_lot::Mutex; - -pub use aquatic_common::{convert_ipv4_mapped_ipv6, ValidUntil}; - -use aquatic_http_protocol::request::Request; -use aquatic_http_protocol::response::Response; - -use crate::common::*; - -pub const LISTENER_TOKEN: Token = Token(0); -pub const CHANNEL_TOKEN: Token = Token(1); - -#[derive(Clone)] -pub struct State { - pub access_list: Arc, - pub torrent_maps: Arc>, -} - -impl Default for State { - fn default() -> Self { - Self { - access_list: Arc::new(Default::default()), - torrent_maps: Arc::new(Mutex::new(TorrentMaps::default())), - } - } -} - -pub type RequestChannelSender = Sender<(ConnectionMeta, Request)>; -pub type RequestChannelReceiver = Receiver<(ConnectionMeta, Request)>; -pub type ResponseChannelReceiver = Receiver<(ConnectionMeta, Response)>; - -#[derive(Clone)] -pub struct ResponseChannelSender { - senders: Vec>, -} - -impl ResponseChannelSender { - pub fn new(senders: Vec>) -> Self { - Self { senders } - } - - #[inline] - pub fn send(&self, meta: ConnectionMeta, message: Response) { - if let Err(err) = self.senders[meta.worker_index].send((meta, message)) { - error!("ResponseChannelSender: couldn't send message: {:?}", err); - } - } -} - -pub type SocketWorkerStatus = Option>; -pub type SocketWorkerStatuses = Arc>>; diff --git a/aquatic_http/src/lib/mio/handler.rs b/aquatic_http/src/lib/mio/handler.rs deleted file mode 100644 index fcdc4e9..0000000 --- a/aquatic_http/src/lib/mio/handler.rs +++ /dev/null @@ -1,97 +0,0 @@ -use std::sync::Arc; -use std::time::Duration; - -use mio::Waker; -use parking_lot::MutexGuard; -use rand::{rngs::SmallRng, SeedableRng}; - -use aquatic_http_protocol::request::*; -use aquatic_http_protocol::response::*; - -use super::common::*; -use crate::common::handlers::{handle_announce_request, handle_scrape_request}; -use crate::common::*; -use crate::config::Config; - -pub fn run_request_worker( - config: Config, - state: State, - request_channel_receiver: RequestChannelReceiver, - response_channel_sender: ResponseChannelSender, - wakers: Vec>, -) { - let mut wake_socket_workers: Vec = (0..config.socket_workers).map(|_| false).collect(); - - let mut announce_requests = Vec::new(); - let mut scrape_requests = Vec::new(); - - let mut rng = SmallRng::from_entropy(); - - let timeout = Duration::from_micros(config.handlers.channel_recv_timeout_microseconds); - - loop { - let mut opt_torrent_maps: Option> = None; - - // If torrent state mutex is locked, just keep collecting requests - // and process them later. This can happen with either multiple - // request workers or while cleaning is underway. - for i in 0..config.handlers.max_requests_per_iter { - let opt_in_message = if i == 0 { - request_channel_receiver.recv().ok() - } else { - request_channel_receiver.recv_timeout(timeout).ok() - }; - - match opt_in_message { - Some((meta, Request::Announce(r))) => { - announce_requests.push((meta, r)); - } - Some((meta, Request::Scrape(r))) => { - scrape_requests.push((meta, r)); - } - None => { - if let Some(torrent_guard) = state.torrent_maps.try_lock() { - opt_torrent_maps = Some(torrent_guard); - - break; - } - } - } - } - - let mut torrent_maps = opt_torrent_maps.unwrap_or_else(|| state.torrent_maps.lock()); - - let valid_until = ValidUntil::new(config.cleaning.max_peer_age); - - for (meta, request) in announce_requests.drain(..) { - let response = handle_announce_request( - &config, - &mut rng, - &mut torrent_maps, - valid_until, - meta, - request, - ); - - response_channel_sender.send(meta, Response::Announce(response)); - wake_socket_workers[meta.worker_index] = true; - } - - for (meta, request) in scrape_requests.drain(..) { - let response = handle_scrape_request(&config, &mut torrent_maps, meta, request); - - response_channel_sender.send(meta, Response::Scrape(response)); - wake_socket_workers[meta.worker_index] = true; - } - - for (worker_index, wake) in wake_socket_workers.iter_mut().enumerate() { - if *wake { - if let Err(err) = wakers[worker_index].wake() { - ::log::error!("request handler couldn't wake poll: {:?}", err); - } - - *wake = false; - } - } - } -} diff --git a/aquatic_http/src/lib/mio/mod.rs b/aquatic_http/src/lib/mio/mod.rs deleted file mode 100644 index ea61019..0000000 --- a/aquatic_http/src/lib/mio/mod.rs +++ /dev/null @@ -1,150 +0,0 @@ -use std::sync::Arc; -use std::thread::Builder; -use std::time::Duration; - -use anyhow::Context; -use mio::{Poll, Waker}; -use parking_lot::Mutex; -use privdrop::PrivDrop; - -pub mod common; -pub mod handler; -pub mod network; -pub mod tasks; - -use crate::config::Config; -use common::*; -use network::utils::create_tls_acceptor; - -pub fn run(config: Config) -> anyhow::Result<()> { - let state = State::default(); - - tasks::update_access_list(&config, &state); - - start_workers(config.clone(), state.clone())?; - - loop { - ::std::thread::sleep(Duration::from_secs(config.cleaning.interval)); - - tasks::update_access_list(&config, &state); - - state - .torrent_maps - .lock() - .clean(&config, &state.access_list.load_full()); - } -} - -pub fn start_workers(config: Config, state: State) -> anyhow::Result<()> { - let opt_tls_acceptor = create_tls_acceptor(&config.network.tls)?; - - let (request_channel_sender, request_channel_receiver) = ::crossbeam_channel::unbounded(); - - let mut out_message_senders = Vec::new(); - let mut wakers = Vec::new(); - - let socket_worker_statuses: SocketWorkerStatuses = { - let mut statuses = Vec::new(); - - for _ in 0..config.socket_workers { - statuses.push(None); - } - - Arc::new(Mutex::new(statuses)) - }; - - for i in 0..config.socket_workers { - let config = config.clone(); - let state = state.clone(); - let socket_worker_statuses = socket_worker_statuses.clone(); - let request_channel_sender = request_channel_sender.clone(); - let opt_tls_acceptor = opt_tls_acceptor.clone(); - let poll = Poll::new().expect("create poll"); - let waker = Arc::new(Waker::new(poll.registry(), CHANNEL_TOKEN).expect("create waker")); - - let (response_channel_sender, response_channel_receiver) = ::crossbeam_channel::unbounded(); - - out_message_senders.push(response_channel_sender); - wakers.push(waker); - - Builder::new() - .name(format!("socket-{:02}", i + 1)) - .spawn(move || { - network::run_socket_worker( - config, - state, - i, - socket_worker_statuses, - request_channel_sender, - response_channel_receiver, - opt_tls_acceptor, - poll, - ); - })?; - } - - // Wait for socket worker statuses. On error from any, quit program. - // On success from all, drop privileges if corresponding setting is set - // and continue program. - loop { - ::std::thread::sleep(::std::time::Duration::from_millis(10)); - - if let Some(statuses) = socket_worker_statuses.try_lock() { - for opt_status in statuses.iter() { - if let Some(Err(err)) = opt_status { - return Err(::anyhow::anyhow!(err.to_owned())); - } - } - - if statuses.iter().all(Option::is_some) { - if config.privileges.drop_privileges { - PrivDrop::default() - .chroot(config.privileges.chroot_path.clone()) - .user(config.privileges.user.clone()) - .apply() - .context("Couldn't drop root privileges")?; - } - - break; - } - } - } - - let response_channel_sender = ResponseChannelSender::new(out_message_senders); - - for i in 0..config.request_workers { - let config = config.clone(); - let state = state.clone(); - let request_channel_receiver = request_channel_receiver.clone(); - let response_channel_sender = response_channel_sender.clone(); - let wakers = wakers.clone(); - - Builder::new() - .name(format!("request-{:02}", i + 1)) - .spawn(move || { - handler::run_request_worker( - config, - state, - request_channel_receiver, - response_channel_sender, - wakers, - ); - })?; - } - - if config.statistics.interval != 0 { - let state = state.clone(); - let config = config.clone(); - - Builder::new() - .name("statistics".to_string()) - .spawn(move || loop { - ::std::thread::sleep(Duration::from_secs(config.statistics.interval)); - - tasks::print_statistics(&state); - }) - .expect("spawn statistics thread"); - } - - Ok(()) -} diff --git a/aquatic_http/src/lib/mio/network/connection.rs b/aquatic_http/src/lib/mio/network/connection.rs deleted file mode 100644 index 7ac3aa6..0000000 --- a/aquatic_http/src/lib/mio/network/connection.rs +++ /dev/null @@ -1,260 +0,0 @@ -use std::io::ErrorKind; -use std::io::{Read, Write}; -use std::net::SocketAddr; -use std::sync::Arc; - -use hashbrown::HashMap; -use mio::net::TcpStream; -use mio::{Poll, Token}; -use native_tls::{MidHandshakeTlsStream, TlsAcceptor}; - -use aquatic_http_protocol::request::{Request, RequestParseError}; - -use crate::common::num_digits_in_usize; -use crate::mio::common::*; - -use super::stream::Stream; - -#[derive(Debug)] -pub enum RequestReadError { - NeedMoreData, - StreamEnded, - Parse(anyhow::Error), - Io(::std::io::Error), -} - -pub struct EstablishedConnection { - stream: Stream, - pub peer_addr: SocketAddr, - buf: Vec, - bytes_read: usize, -} - -impl EstablishedConnection { - #[inline] - fn new(stream: Stream) -> Self { - let peer_addr = stream.get_peer_addr(); - - Self { - stream, - peer_addr, - buf: Vec::new(), - bytes_read: 0, - } - } - - pub fn read_request(&mut self) -> Result { - if (self.buf.len() - self.bytes_read < 512) & (self.buf.len() <= 3072) { - self.buf.extend_from_slice(&[0; 1024]); - } - - match self.stream.read(&mut self.buf[self.bytes_read..]) { - Ok(0) => { - self.clear_buffer(); - - return Err(RequestReadError::StreamEnded); - } - Ok(bytes_read) => { - self.bytes_read += bytes_read; - - ::log::debug!("read_request read {} bytes", bytes_read); - } - Err(err) if err.kind() == ErrorKind::WouldBlock => { - return Err(RequestReadError::NeedMoreData); - } - Err(err) => { - self.clear_buffer(); - - return Err(RequestReadError::Io(err)); - } - } - - match Request::from_bytes(&self.buf[..self.bytes_read]) { - Ok(request) => { - self.clear_buffer(); - - Ok(request) - } - Err(RequestParseError::NeedMoreData) => Err(RequestReadError::NeedMoreData), - Err(RequestParseError::Invalid(err)) => { - self.clear_buffer(); - - Err(RequestReadError::Parse(err)) - } - } - } - - pub fn send_response(&mut self, body: &[u8]) -> ::std::io::Result<()> { - let content_len = body.len() + 2; // 2 is for newlines at end - let content_len_num_digits = num_digits_in_usize(content_len); - - let mut response = Vec::with_capacity(39 + content_len_num_digits + body.len()); - - response.extend_from_slice(b"HTTP/1.1 200 OK\r\nContent-Length: "); - ::itoa::write(&mut response, content_len)?; - response.extend_from_slice(b"\r\n\r\n"); - response.extend_from_slice(body); - response.extend_from_slice(b"\r\n"); - - let bytes_written = self.stream.write(&response)?; - - if bytes_written != response.len() { - ::log::error!( - "send_response: only {} out of {} bytes written", - bytes_written, - response.len() - ); - } - - self.stream.flush()?; - - Ok(()) - } - - #[inline] - pub fn clear_buffer(&mut self) { - self.bytes_read = 0; - self.buf = Vec::new(); - } -} - -pub enum TlsHandshakeMachineError { - WouldBlock(TlsHandshakeMachine), - Failure(native_tls::Error), -} - -enum TlsHandshakeMachineInner { - TcpStream(TcpStream), - TlsMidHandshake(MidHandshakeTlsStream), -} - -pub struct TlsHandshakeMachine { - tls_acceptor: Arc, - inner: TlsHandshakeMachineInner, -} - -impl<'a> TlsHandshakeMachine { - #[inline] - fn new(tls_acceptor: Arc, tcp_stream: TcpStream) -> Self { - Self { - tls_acceptor, - inner: TlsHandshakeMachineInner::TcpStream(tcp_stream), - } - } - - /// Attempt to establish a TLS connection. On a WouldBlock error, return - /// the machine wrapped in an error for later attempts. - pub fn establish_tls(self) -> Result { - let handshake_result = match self.inner { - TlsHandshakeMachineInner::TcpStream(stream) => self.tls_acceptor.accept(stream), - TlsHandshakeMachineInner::TlsMidHandshake(handshake) => handshake.handshake(), - }; - - match handshake_result { - Ok(stream) => { - let established = EstablishedConnection::new(Stream::TlsStream(stream)); - - ::log::debug!("established tls connection"); - - Ok(established) - } - Err(native_tls::HandshakeError::WouldBlock(handshake)) => { - let inner = TlsHandshakeMachineInner::TlsMidHandshake(handshake); - - let machine = Self { - tls_acceptor: self.tls_acceptor, - inner, - }; - - Err(TlsHandshakeMachineError::WouldBlock(machine)) - } - Err(native_tls::HandshakeError::Failure(err)) => { - Err(TlsHandshakeMachineError::Failure(err)) - } - } - } -} - -enum ConnectionInner { - Established(EstablishedConnection), - InProgress(TlsHandshakeMachine), -} - -pub struct Connection { - pub valid_until: ValidUntil, - inner: ConnectionInner, -} - -impl Connection { - #[inline] - pub fn new( - opt_tls_acceptor: &Option>, - valid_until: ValidUntil, - tcp_stream: TcpStream, - ) -> Self { - // Setup handshake machine if TLS is requested - let inner = if let Some(tls_acceptor) = opt_tls_acceptor { - ConnectionInner::InProgress(TlsHandshakeMachine::new(tls_acceptor.clone(), tcp_stream)) - } else { - ::log::debug!("established tcp connection"); - - ConnectionInner::Established(EstablishedConnection::new(Stream::TcpStream(tcp_stream))) - }; - - Self { valid_until, inner } - } - - #[inline] - pub fn from_established(valid_until: ValidUntil, established: EstablishedConnection) -> Self { - Self { - valid_until, - inner: ConnectionInner::Established(established), - } - } - - #[inline] - pub fn from_in_progress(valid_until: ValidUntil, machine: TlsHandshakeMachine) -> Self { - Self { - valid_until, - inner: ConnectionInner::InProgress(machine), - } - } - - #[inline] - pub fn get_established(&mut self) -> Option<&mut EstablishedConnection> { - if let ConnectionInner::Established(ref mut established) = self.inner { - Some(established) - } else { - None - } - } - - /// Takes ownership since TlsStream needs ownership of TcpStream - #[inline] - pub fn get_in_progress(self) -> Option { - if let ConnectionInner::InProgress(machine) = self.inner { - Some(machine) - } else { - None - } - } - - pub fn deregister(&mut self, poll: &mut Poll) -> ::std::io::Result<()> { - match &mut self.inner { - ConnectionInner::Established(established) => match &mut established.stream { - Stream::TcpStream(ref mut stream) => poll.registry().deregister(stream), - Stream::TlsStream(ref mut stream) => poll.registry().deregister(stream.get_mut()), - }, - ConnectionInner::InProgress(TlsHandshakeMachine { inner, .. }) => match inner { - TlsHandshakeMachineInner::TcpStream(ref mut stream) => { - poll.registry().deregister(stream) - } - TlsHandshakeMachineInner::TlsMidHandshake(ref mut mid_handshake) => { - poll.registry().deregister(mid_handshake.get_mut()) - } - }, - } - } -} - -pub type ConnectionMap = HashMap; diff --git a/aquatic_http/src/lib/mio/network/mod.rs b/aquatic_http/src/lib/mio/network/mod.rs deleted file mode 100644 index fa0c9bd..0000000 --- a/aquatic_http/src/lib/mio/network/mod.rs +++ /dev/null @@ -1,388 +0,0 @@ -use std::io::{Cursor, ErrorKind}; -use std::sync::Arc; -use std::time::{Duration, Instant}; -use std::vec::Drain; - -use aquatic_common::access_list::AccessListQuery; -use aquatic_http_protocol::request::Request; -use hashbrown::HashMap; -use log::{debug, error, info}; -use mio::net::TcpListener; -use mio::{Events, Interest, Poll, Token}; -use native_tls::TlsAcceptor; - -use aquatic_http_protocol::response::*; - -use crate::common::*; -use crate::config::Config; -use crate::mio::common::*; - -pub mod connection; -pub mod stream; -pub mod utils; - -use connection::*; -use utils::*; - -const CONNECTION_CLEAN_INTERVAL: usize = 2 ^ 22; - -pub fn run_socket_worker( - config: Config, - state: State, - socket_worker_index: usize, - socket_worker_statuses: SocketWorkerStatuses, - request_channel_sender: RequestChannelSender, - response_channel_receiver: ResponseChannelReceiver, - opt_tls_acceptor: Option, - poll: Poll, -) { - match create_listener(config.network.address, config.network.ipv6_only) { - Ok(listener) => { - socket_worker_statuses.lock()[socket_worker_index] = Some(Ok(())); - - run_poll_loop( - config, - &state, - socket_worker_index, - request_channel_sender, - response_channel_receiver, - listener, - opt_tls_acceptor, - poll, - ); - } - Err(err) => { - socket_worker_statuses.lock()[socket_worker_index] = - Some(Err(format!("Couldn't open socket: {:#}", err))); - } - } -} - -pub fn run_poll_loop( - config: Config, - state: &State, - socket_worker_index: usize, - request_channel_sender: RequestChannelSender, - response_channel_receiver: ResponseChannelReceiver, - listener: ::std::net::TcpListener, - opt_tls_acceptor: Option, - mut poll: Poll, -) { - let poll_timeout = Duration::from_micros(config.network.poll_timeout_microseconds); - - let mut listener = TcpListener::from_std(listener); - let mut events = Events::with_capacity(config.network.poll_event_capacity); - - poll.registry() - .register(&mut listener, Token(0), Interest::READABLE) - .unwrap(); - - let mut connections: ConnectionMap = HashMap::new(); - let opt_tls_acceptor = opt_tls_acceptor.map(Arc::new); - - let mut poll_token_counter = Token(0usize); - let mut iter_counter = 0usize; - - let mut response_buffer = [0u8; 4096]; - let mut response_buffer = Cursor::new(&mut response_buffer[..]); - let mut local_responses = Vec::new(); - - loop { - poll.poll(&mut events, Some(poll_timeout)) - .expect("failed polling"); - - for event in events.iter() { - let token = event.token(); - - if token == LISTENER_TOKEN { - accept_new_streams( - &config, - &mut listener, - &mut poll, - &mut connections, - &mut poll_token_counter, - &opt_tls_acceptor, - ); - } else if token != CHANNEL_TOKEN { - handle_connection_read_event( - &config, - &state, - socket_worker_index, - &mut poll, - &request_channel_sender, - &mut local_responses, - &mut connections, - token, - ); - } - - // Send responses for each event. Channel token is not interesting - // by itself, but is just for making sure responses are sent even - // if no new connects / requests come in. - send_responses( - &config, - &mut poll, - &mut response_buffer, - local_responses.drain(..), - &response_channel_receiver, - &mut connections, - ); - } - - // Remove inactive connections, but not every iteration - if iter_counter % CONNECTION_CLEAN_INTERVAL == 0 { - remove_inactive_connections(&mut poll, &mut connections); - } - - iter_counter = iter_counter.wrapping_add(1); - } -} - -fn accept_new_streams( - config: &Config, - listener: &mut TcpListener, - poll: &mut Poll, - connections: &mut ConnectionMap, - poll_token_counter: &mut Token, - opt_tls_acceptor: &Option>, -) { - let valid_until = ValidUntil::new(config.cleaning.max_connection_age); - - loop { - match listener.accept() { - Ok((mut stream, _)) => { - poll_token_counter.0 = poll_token_counter.0.wrapping_add(1); - - // Skip listener and channel tokens - if poll_token_counter.0 < 2 { - poll_token_counter.0 = 2; - } - - let token = *poll_token_counter; - - // Remove connection if it exists (which is unlikely) - remove_connection(poll, connections, poll_token_counter); - - poll.registry() - .register(&mut stream, token, Interest::READABLE) - .unwrap(); - - let connection = Connection::new(opt_tls_acceptor, valid_until, stream); - - connections.insert(token, connection); - } - Err(err) => { - if err.kind() == ErrorKind::WouldBlock { - break; - } - - info!("error while accepting streams: {}", err); - } - } - } -} - -/// On the stream given by poll_token, get TLS up and running if requested, -/// then read requests and pass on through channel. -pub fn handle_connection_read_event( - config: &Config, - state: &State, - socket_worker_index: usize, - poll: &mut Poll, - request_channel_sender: &RequestChannelSender, - local_responses: &mut Vec<(ConnectionMeta, Response)>, - connections: &mut ConnectionMap, - poll_token: Token, -) { - let valid_until = ValidUntil::new(config.cleaning.max_connection_age); - let access_list_mode = config.access_list.mode; - - loop { - // Get connection, updating valid_until - let connection = if let Some(c) = connections.get_mut(&poll_token) { - c - } else { - // If there is no connection, there is no stream, so there - // shouldn't be any (relevant) poll events. In other words, it's - // safe to return here - return; - }; - - connection.valid_until = valid_until; - - if let Some(established) = connection.get_established() { - match established.read_request() { - Ok(Request::Announce(ref r)) - if !state.access_list.allows(access_list_mode, &r.info_hash.0) => - { - let meta = ConnectionMeta { - worker_index: socket_worker_index, - poll_token: poll_token.0, - peer_addr: established.peer_addr, - }; - let response = FailureResponse::new("Info hash not allowed"); - - debug!("read disallowed request, sending back error response"); - - local_responses.push((meta, Response::Failure(response))); - - break; - } - Ok(request) => { - let meta = ConnectionMeta { - worker_index: socket_worker_index, - poll_token: poll_token.0, - peer_addr: established.peer_addr, - }; - - debug!("read allowed request, sending on to channel"); - - if let Err(err) = request_channel_sender.send((meta, request)) { - error!("RequestChannelSender: couldn't send message: {:?}", err); - } - - break; - } - Err(RequestReadError::NeedMoreData) => { - info!("need more data"); - - // Stop reading data (defer to later events) - break; - } - Err(RequestReadError::Parse(err)) => { - info!("error reading request (invalid): {:#?}", err); - - let meta = ConnectionMeta { - worker_index: socket_worker_index, - poll_token: poll_token.0, - peer_addr: established.peer_addr, - }; - - let response = FailureResponse::new("Invalid request"); - - local_responses.push((meta, Response::Failure(response))); - - break; - } - Err(RequestReadError::StreamEnded) => { - ::log::debug!("stream ended"); - - remove_connection(poll, connections, &poll_token); - - break; - } - Err(RequestReadError::Io(err)) => { - ::log::info!("error reading request (io): {}", err); - - remove_connection(poll, connections, &poll_token); - - break; - } - } - } else if let Some(handshake_machine) = connections - .remove(&poll_token) - .and_then(Connection::get_in_progress) - { - match handshake_machine.establish_tls() { - Ok(established) => { - let connection = Connection::from_established(valid_until, established); - - connections.insert(poll_token, connection); - } - Err(TlsHandshakeMachineError::WouldBlock(machine)) => { - let connection = Connection::from_in_progress(valid_until, machine); - - connections.insert(poll_token, connection); - - // Break and wait for more data - break; - } - Err(TlsHandshakeMachineError::Failure(err)) => { - info!("tls handshake error: {}", err); - - // TLS negotiation failed - break; - } - } - } - } -} - -/// Read responses from channel, send to peers -pub fn send_responses( - config: &Config, - poll: &mut Poll, - buffer: &mut Cursor<&mut [u8]>, - local_responses: Drain<(ConnectionMeta, Response)>, - channel_responses: &ResponseChannelReceiver, - connections: &mut ConnectionMap, -) { - let channel_responses_len = channel_responses.len(); - let channel_responses_drain = channel_responses.try_iter().take(channel_responses_len); - - for (meta, response) in local_responses.chain(channel_responses_drain) { - if let Some(established) = connections - .get_mut(&Token(meta.poll_token)) - .and_then(Connection::get_established) - { - if established.peer_addr != meta.peer_addr { - info!("socket worker error: peer socket addrs didn't match"); - - continue; - } - - buffer.set_position(0); - - let bytes_written = response.write(buffer).unwrap(); - - match established.send_response(&buffer.get_mut()[..bytes_written]) { - Ok(()) => { - ::log::debug!( - "sent response: {:?} with response string {}", - response, - String::from_utf8_lossy(&buffer.get_ref()[..bytes_written]) - ); - - if !config.network.keep_alive { - remove_connection(poll, connections, &Token(meta.poll_token)); - } - } - Err(err) if err.kind() == ErrorKind::WouldBlock => { - debug!("send response: would block"); - } - Err(err) => { - info!("error sending response: {}", err); - - remove_connection(poll, connections, &Token(meta.poll_token)); - } - } - } - } -} - -// Close and remove inactive connections -pub fn remove_inactive_connections(poll: &mut Poll, connections: &mut ConnectionMap) { - let now = Instant::now(); - - connections.retain(|_, connection| { - let keep = connection.valid_until.0 >= now; - - if !keep { - if let Err(err) = connection.deregister(poll) { - ::log::error!("deregister connection error: {}", err); - } - } - - keep - }); - - connections.shrink_to_fit(); -} - -fn remove_connection(poll: &mut Poll, connections: &mut ConnectionMap, connection_token: &Token) { - if let Some(mut connection) = connections.remove(connection_token) { - if let Err(err) = connection.deregister(poll) { - ::log::error!("deregister connection error: {}", err); - } - } -} diff --git a/aquatic_http/src/lib/mio/network/stream.rs b/aquatic_http/src/lib/mio/network/stream.rs deleted file mode 100644 index 0104f5a..0000000 --- a/aquatic_http/src/lib/mio/network/stream.rs +++ /dev/null @@ -1,69 +0,0 @@ -use std::io::{Read, Write}; -use std::net::SocketAddr; - -use mio::net::TcpStream; -use native_tls::TlsStream; - -pub enum Stream { - TcpStream(TcpStream), - TlsStream(TlsStream), -} - -impl Stream { - #[inline] - pub fn get_peer_addr(&self) -> SocketAddr { - match self { - Self::TcpStream(stream) => stream.peer_addr().unwrap(), - Self::TlsStream(stream) => stream.get_ref().peer_addr().unwrap(), - } - } -} - -impl Read for Stream { - #[inline] - fn read(&mut self, buf: &mut [u8]) -> Result { - match self { - Self::TcpStream(stream) => stream.read(buf), - Self::TlsStream(stream) => stream.read(buf), - } - } - - /// Not used but provided for completeness - #[inline] - fn read_vectored( - &mut self, - bufs: &mut [::std::io::IoSliceMut<'_>], - ) -> ::std::io::Result { - match self { - Self::TcpStream(stream) => stream.read_vectored(bufs), - Self::TlsStream(stream) => stream.read_vectored(bufs), - } - } -} - -impl Write for Stream { - #[inline] - fn write(&mut self, buf: &[u8]) -> ::std::io::Result { - match self { - Self::TcpStream(stream) => stream.write(buf), - Self::TlsStream(stream) => stream.write(buf), - } - } - - /// Not used but provided for completeness - #[inline] - fn write_vectored(&mut self, bufs: &[::std::io::IoSlice<'_>]) -> ::std::io::Result { - match self { - Self::TcpStream(stream) => stream.write_vectored(bufs), - Self::TlsStream(stream) => stream.write_vectored(bufs), - } - } - - #[inline] - fn flush(&mut self) -> ::std::io::Result<()> { - match self { - Self::TcpStream(stream) => stream.flush(), - Self::TlsStream(stream) => stream.flush(), - } - } -} diff --git a/aquatic_http/src/lib/mio/network/utils.rs b/aquatic_http/src/lib/mio/network/utils.rs deleted file mode 100644 index 9349f98..0000000 --- a/aquatic_http/src/lib/mio/network/utils.rs +++ /dev/null @@ -1,63 +0,0 @@ -use std::fs::File; -use std::io::Read; -use std::net::SocketAddr; - -use anyhow::Context; -use native_tls::{Identity, TlsAcceptor}; -use socket2::{Domain, Protocol, Socket, Type}; - -use crate::config::TlsConfig; - -pub fn create_tls_acceptor(config: &TlsConfig) -> anyhow::Result> { - if config.use_tls { - let mut identity_bytes = Vec::new(); - let mut file = - File::open(&config.tls_pkcs12_path).context("Couldn't open pkcs12 identity file")?; - - file.read_to_end(&mut identity_bytes) - .context("Couldn't read pkcs12 identity file")?; - - let identity = Identity::from_pkcs12(&identity_bytes[..], &config.tls_pkcs12_password) - .context("Couldn't parse pkcs12 identity file")?; - - let acceptor = TlsAcceptor::new(identity) - .context("Couldn't create TlsAcceptor from pkcs12 identity")?; - - Ok(Some(acceptor)) - } else { - Ok(None) - } -} - -pub fn create_listener( - address: SocketAddr, - ipv6_only: bool, -) -> ::anyhow::Result<::std::net::TcpListener> { - let builder = if address.is_ipv4() { - Socket::new(Domain::IPV4, Type::STREAM, Some(Protocol::TCP)) - } else { - Socket::new(Domain::IPV6, Type::STREAM, Some(Protocol::TCP)) - } - .context("Couldn't create socket2::Socket")?; - - if ipv6_only { - builder - .set_only_v6(true) - .context("Couldn't put socket in ipv6 only mode")? - } - - builder - .set_nonblocking(true) - .context("Couldn't put socket in non-blocking mode")?; - builder - .set_reuse_port(true) - .context("Couldn't put socket in reuse_port mode")?; - builder - .bind(&address.into()) - .with_context(|| format!("Couldn't bind socket to address {}", address))?; - builder - .listen(128) - .context("Couldn't listen for connections on socket")?; - - Ok(builder.into()) -} diff --git a/aquatic_http/src/lib/mio/tasks.rs b/aquatic_http/src/lib/mio/tasks.rs deleted file mode 100644 index 0106e4e..0000000 --- a/aquatic_http/src/lib/mio/tasks.rs +++ /dev/null @@ -1,53 +0,0 @@ -use histogram::Histogram; - -use aquatic_common::access_list::{AccessListMode, AccessListQuery}; - -use super::common::*; -use crate::config::Config; - -pub fn update_access_list(config: &Config, state: &State) { - match config.access_list.mode { - AccessListMode::White | AccessListMode::Black => { - if let Err(err) = state.access_list.update_from_path(&config.access_list.path) { - ::log::error!("Couldn't update access list: {:?}", err); - } - } - AccessListMode::Off => {} - } -} - -pub fn print_statistics(state: &State) { - let mut peers_per_torrent = Histogram::new(); - - { - let torrents = &mut state.torrent_maps.lock(); - - for torrent in torrents.ipv4.values() { - let num_peers = (torrent.num_seeders + torrent.num_leechers) as u64; - - if let Err(err) = peers_per_torrent.increment(num_peers) { - eprintln!("error incrementing peers_per_torrent histogram: {}", err) - } - } - for torrent in torrents.ipv6.values() { - let num_peers = (torrent.num_seeders + torrent.num_leechers) as u64; - - if let Err(err) = peers_per_torrent.increment(num_peers) { - eprintln!("error incrementing peers_per_torrent histogram: {}", err) - } - } - } - - if peers_per_torrent.entries() != 0 { - println!( - "peers per torrent: min: {}, p50: {}, p75: {}, p90: {}, p99: {}, p999: {}, max: {}", - peers_per_torrent.minimum().unwrap(), - peers_per_torrent.percentile(50.0).unwrap(), - peers_per_torrent.percentile(75.0).unwrap(), - peers_per_torrent.percentile(90.0).unwrap(), - peers_per_torrent.percentile(99.0).unwrap(), - peers_per_torrent.percentile(99.9).unwrap(), - peers_per_torrent.maximum().unwrap(), - ); - } -} diff --git a/aquatic_http/src/lib/glommio/network.rs b/aquatic_http/src/lib/network.rs similarity index 100% rename from aquatic_http/src/lib/glommio/network.rs rename to aquatic_http/src/lib/network.rs diff --git a/aquatic_udp/src/lib/lib.rs b/aquatic_udp/src/lib/lib.rs index 78676c2..34e25a8 100644 --- a/aquatic_udp/src/lib/lib.rs +++ b/aquatic_udp/src/lib/lib.rs @@ -1,11 +1,3 @@ -use std::{ - sync::{ - atomic::{AtomicUsize, Ordering}, - Arc, - }, - time::Duration, -}; - use cfg_if::cfg_if; pub mod common; From f60631b29e4c99343f5e7af02b6cdce271797fbb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20Frosteg=C3=A5rd?= Date: Thu, 28 Oct 2021 01:49:37 +0200 Subject: [PATCH 51/59] Run cargo fmt --- aquatic_http/src/lib/common.rs | 3 +-- aquatic_http/src/lib/handlers.rs | 2 +- aquatic_http/src/lib/lib.rs | 4 ++-- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/aquatic_http/src/lib/common.rs b/aquatic_http/src/lib/common.rs index 67edb55..b64d30e 100644 --- a/aquatic_http/src/lib/common.rs +++ b/aquatic_http/src/lib/common.rs @@ -27,7 +27,6 @@ use aquatic_http_protocol::{ response::{AnnounceResponse, ScrapeResponse}, }; - #[derive(Copy, Clone, Debug)] pub struct ConsumerId(pub usize); @@ -291,4 +290,4 @@ mod tests { assert_eq!(f(101), 3); assert_eq!(f(1000), 4); } -} \ No newline at end of file +} diff --git a/aquatic_http/src/lib/handlers.rs b/aquatic_http/src/lib/handlers.rs index b636fff..dd2d6ed 100644 --- a/aquatic_http/src/lib/handlers.rs +++ b/aquatic_http/src/lib/handlers.rs @@ -357,4 +357,4 @@ pub fn handle_scrape_request( }; response -} \ No newline at end of file +} diff --git a/aquatic_http/src/lib/lib.rs b/aquatic_http/src/lib/lib.rs index 62ac41f..7ec1224 100644 --- a/aquatic_http/src/lib/lib.rs +++ b/aquatic_http/src/lib/lib.rs @@ -9,8 +9,8 @@ use glommio::{channels::channel_mesh::MeshBuilder, prelude::*}; use crate::config::Config; -pub mod config; mod common; +pub mod config; mod handlers; mod network; @@ -140,4 +140,4 @@ fn create_tls_config(config: &Config) -> anyhow::Result { .with_single_cert(certs, private_key)?; Ok(tls_config) -} \ No newline at end of file +} From cccee914d65c66257304ff229dc3f8b3a1afe733 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20Frosteg=C3=A5rd?= Date: Thu, 28 Oct 2021 01:57:16 +0200 Subject: [PATCH 52/59] aquatic_http: remove config fields previously used for mio impl --- aquatic_http/src/lib/config.rs | 68 ++-------------------------------- aquatic_http/src/lib/lib.rs | 4 +- 2 files changed, 6 insertions(+), 66 deletions(-) diff --git a/aquatic_http/src/lib/config.rs b/aquatic_http/src/lib/config.rs index 1e60db5..e2e3131 100644 --- a/aquatic_http/src/lib/config.rs +++ b/aquatic_http/src/lib/config.rs @@ -19,9 +19,7 @@ pub struct Config { pub log_level: LogLevel, pub network: NetworkConfig, pub protocol: ProtocolConfig, - pub handlers: HandlerConfig, pub cleaning: CleaningConfig, - pub statistics: StatisticsConfig, pub privileges: PrivilegeConfig, pub access_list: AccessListConfig, pub cpu_pinning: CpuPinningConfig, @@ -33,27 +31,15 @@ impl aquatic_cli_helpers::Config for Config { } } -#[derive(Clone, Debug, Serialize, Deserialize)] -#[serde(default)] -pub struct TlsConfig { - pub use_tls: bool, - pub tls_pkcs12_path: String, - pub tls_pkcs12_password: String, - pub tls_certificate_path: PathBuf, - pub tls_private_key_path: PathBuf, -} - #[derive(Clone, Debug, Serialize, Deserialize)] #[serde(default)] pub struct NetworkConfig { /// Bind to this address pub address: SocketAddr, + pub tls_certificate_path: PathBuf, + pub tls_private_key_path: PathBuf, pub ipv6_only: bool, - #[serde(flatten)] - pub tls: TlsConfig, pub keep_alive: bool, - pub poll_event_capacity: usize, - pub poll_timeout_microseconds: u64, } #[derive(Clone, Debug, Serialize, Deserialize)] @@ -67,15 +53,6 @@ pub struct ProtocolConfig { pub peer_announce_interval: usize, } -#[derive(Clone, Debug, Serialize, Deserialize)] -#[serde(default)] -pub struct HandlerConfig { - /// Maximum number of requests to receive from channel before locking - /// mutex and starting work - pub max_requests_per_iter: usize, - pub channel_recv_timeout_microseconds: u64, -} - #[derive(Clone, Debug, Serialize, Deserialize)] #[serde(default)] pub struct CleaningConfig { @@ -87,13 +64,6 @@ pub struct CleaningConfig { pub max_connection_age: u64, } -#[derive(Clone, Debug, Serialize, Deserialize)] -#[serde(default)] -pub struct StatisticsConfig { - /// Print statistics this often (seconds). Don't print when set to zero. - pub interval: u64, -} - impl Default for Config { fn default() -> Self { Self { @@ -102,9 +72,7 @@ impl Default for Config { log_level: LogLevel::default(), network: NetworkConfig::default(), protocol: ProtocolConfig::default(), - handlers: HandlerConfig::default(), cleaning: CleaningConfig::default(), - statistics: StatisticsConfig::default(), privileges: PrivilegeConfig::default(), access_list: AccessListConfig::default(), cpu_pinning: Default::default(), @@ -116,11 +84,10 @@ impl Default for NetworkConfig { fn default() -> Self { Self { address: SocketAddr::from(([0, 0, 0, 0], 3000)), + tls_certificate_path: "".into(), + tls_private_key_path: "".into(), ipv6_only: false, - tls: TlsConfig::default(), keep_alive: true, - poll_event_capacity: 4096, - poll_timeout_microseconds: 200_000, } } } @@ -135,15 +102,6 @@ impl Default for ProtocolConfig { } } -impl Default for HandlerConfig { - fn default() -> Self { - Self { - max_requests_per_iter: 10_000, - channel_recv_timeout_microseconds: 200, - } - } -} - impl Default for CleaningConfig { fn default() -> Self { Self { @@ -153,21 +111,3 @@ impl Default for CleaningConfig { } } } - -impl Default for StatisticsConfig { - fn default() -> Self { - Self { interval: 0 } - } -} - -impl Default for TlsConfig { - fn default() -> Self { - Self { - use_tls: false, - tls_pkcs12_path: "".into(), - tls_pkcs12_password: "".into(), - tls_certificate_path: "".into(), - tls_private_key_path: "".into(), - } - } -} diff --git a/aquatic_http/src/lib/lib.rs b/aquatic_http/src/lib/lib.rs index 7ec1224..5a1d900 100644 --- a/aquatic_http/src/lib/lib.rs +++ b/aquatic_http/src/lib/lib.rs @@ -115,7 +115,7 @@ pub fn run(config: Config) -> anyhow::Result<()> { fn create_tls_config(config: &Config) -> anyhow::Result { let certs = { - let f = File::open(&config.network.tls.tls_certificate_path)?; + let f = File::open(&config.network.tls_certificate_path)?; let mut f = BufReader::new(f); rustls_pemfile::certs(&mut f)? @@ -125,7 +125,7 @@ fn create_tls_config(config: &Config) -> anyhow::Result { }; let private_key = { - let f = File::open(&config.network.tls.tls_private_key_path)?; + let f = File::open(&config.network.tls_private_key_path)?; let mut f = BufReader::new(f); rustls_pemfile::pkcs8_private_keys(&mut f)? From a24a10c518f9648bdd7bd06a2ef58f78989d6d7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20Frosteg=C3=A5rd?= Date: Thu, 28 Oct 2021 17:36:11 +0200 Subject: [PATCH 53/59] README: write implementation list as table --- README.md | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 6249884..017494d 100644 --- a/README.md +++ b/README.md @@ -5,10 +5,20 @@ Blazingly fast, multi-threaded BitTorrent tracker written in Rust. Consists of three sub-implementations for different protocols: - * `aquatic_udp`: BitTorrent over UDP. Implementation achieves 45% higher throughput - than opentracker (see benchmarks below) - * `aquatic_http`: BitTorrent over HTTP/TLS (slightly experimental) - * `aquatic_ws`: WebTorrent (experimental) + +[BitTorrent over UDP]: https://libtorrent.org/udp_tracker_protocol.html +[BitTorrent over HTTP]: https://wiki.theory.org/index.php/BitTorrentSpecification#Tracker_HTTP.2FHTTPS_Protocol +[WebTorrent]: https://github.com/webtorrent +[rustls]: https://github.com/rustls/rustls +[native-tls]: https://github.com/sfackler/rust-native-tls +[mio]: https://github.com/tokio-rs/mio +[glommio]: https://github.com/DataDog/glommio + +| Name | Protocol | OS requirements | +|--------------|-----------------------------------------------|-----------------------------------------------------------------| +| aquatic_udp | [BitTorrent over UDP] | Cross-platform with [mio] (default) / Linux 5.8+ with [glommio] | +| aquatic_http | [BitTorrent over HTTP] with TLS ([rustls]) | Linux 5.8+ | +| aquatic_ws | [WebTorrent], plain / with TLS ([native-tls]) | Cross-platform | ## Copyright and license From 109a3e34f34dcc017fb1aef0599d296d36e714e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20Frosteg=C3=A5rd?= Date: Thu, 28 Oct 2021 18:15:12 +0200 Subject: [PATCH 54/59] README: rewrite and improve --- README.md | 116 +++++++++++++++++++++++++++++------------------------- 1 file changed, 62 insertions(+), 54 deletions(-) diff --git a/README.md b/README.md index 017494d..3a68021 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ Blazingly fast, multi-threaded BitTorrent tracker written in Rust. -Consists of three sub-implementations for different protocols: +Consists of sub-implementations for different protocols: [BitTorrent over UDP]: https://libtorrent.org/udp_tracker_protocol.html [BitTorrent over HTTP]: https://wiki.theory.org/index.php/BitTorrentSpecification#Tracker_HTTP.2FHTTPS_Protocol @@ -14,11 +14,11 @@ Consists of three sub-implementations for different protocols: [mio]: https://github.com/tokio-rs/mio [glommio]: https://github.com/DataDog/glommio -| Name | Protocol | OS requirements | -|--------------|-----------------------------------------------|-----------------------------------------------------------------| -| aquatic_udp | [BitTorrent over UDP] | Cross-platform with [mio] (default) / Linux 5.8+ with [glommio] | -| aquatic_http | [BitTorrent over HTTP] with TLS ([rustls]) | Linux 5.8+ | -| aquatic_ws | [WebTorrent], plain / with TLS ([native-tls]) | Cross-platform | +| Name | Protocol | OS requirements | +|--------------|------------------------------------------------|-----------------------------------------------------------------| +| aquatic_udp | [BitTorrent over UDP] | Cross-platform with [mio] (default) / Linux 5.8+ with [glommio] | +| aquatic_http | [BitTorrent over HTTP] with TLS ([rustls]) | Linux 5.8+ | +| aquatic_ws | [WebTorrent], plain or with TLS ([native-tls]) | Cross-platform | ## Copyright and license @@ -26,48 +26,52 @@ Copyright (c) 2020-2021 Joakim Frostegård Distributed under Apache 2.0 license (details in `LICENSE` file.) -## Installation prerequisites +## Building + +### Prerequisites - Install Rust with [rustup](https://rustup.rs/) (stable is recommended) - Install cmake with your package manager (e.g., `apt-get install cmake`) -- On GNU/Linux, also install the OpenSSL components necessary for dynamic - linking (e.g., `apt-get install libssl-dev`) -- Clone the git repository and refer to the next section. +- If you want to run aquatic_ws and are on a Unix-like OS, install the OpenSSL + components necessary for dynamic linking (e.g., `apt-get install libssl-dev`) +- Clone this git repository and enter it -## Compile and run +### Compiling -To compile the master executable for all protocols, run: +Compile the implementations that you are interested in: ```sh -./scripts/build-aquatic.sh +cargo build --release -p aquatic_udp +cargo build --release -p aquatic_udp --features "with-glommio" --no-default-features +cargo build --release -p aquatic_http +cargo build --release -p aquatic_ws ``` -To start the tracker for a protocol with default settings, run: +## Running + +To start a tracker with default configuration, run any of: ```sh -./target/release/aquatic udp -./target/release/aquatic http -./target/release/aquatic ws +./target/release/aquatic_udp +./target/release/aquatic_http +./target/release/aquatic_ws ``` -To print default settings to standard output, pass the "-p" flag to the binary: +To adjust the configuration, begin by generating configuration files. They +differ between protocols. ```sh -./target/release/aquatic udp -p -./target/release/aquatic http -p -./target/release/aquatic ws -p +./target/release/aquatic_udp -p > "aquatic-udp-config.toml" +./target/release/aquatic_http -p > "aquatic-http-config.toml" +./target/release/aquatic_ws -p > "aquatic-ws-config.toml" ``` -Note that the configuration files differ between protocols. - -To adjust the settings, save the output of the relevant previous command to a -file and make your changes. Then run `aquatic` with a "-c" argument pointing to -the file, e.g.: +Make adjustments to the files. Then run the tracker with: ```sh -./target/release/aquatic udp -c "/path/to/aquatic-udp-config.toml" -./target/release/aquatic http -c "/path/to/aquatic-http-config.toml" -./target/release/aquatic ws -c "/path/to/aquatic-ws-config.toml" +./target/release/aquatic_udp -c "aquatic-udp-config.toml" +./target/release/aquatic_http -c "aquatic-http-config.toml" +./target/release/aquatic_ws -c "aquatic-ws-config.toml" ``` The configuration file values you will most likely want to adjust are @@ -83,7 +87,7 @@ mode = 'off' # Change to 'black' (blacklist) or 'white' (whitelist) path = '' # Path to text file with newline-delimited hex-encoded info hashes ``` -Some more documentation of configuration file values might be available under +More documentation of configuration file values might be available under `src/lib/config.rs` in crates `aquatic_udp`, `aquatic_http`, `aquatic_ws`. ## Details on implementations @@ -131,25 +135,39 @@ There is an alternative implementation that utilizes [io_uring] by running on [glommio]. It only runs on Linux and requires a recent kernel (version 5.8 or later). In some cases, it performs even better than the cross-platform implementation. -To use it, pass the `with-glommio` feature when building, e.g.: - -```sh -cargo build -p aquatic_udp --features "with-glommio" --no-default-features -./target/release/aquatic_udp -``` - ### aquatic_http: HTTP BitTorrent tracker -Aims for compatibility with the HTTP BitTorrent protocol, as described -[here](https://wiki.theory.org/index.php/BitTorrentSpecification#Tracker_HTTP.2FHTTPS_Protocol), -including TLS and scrape request support. There are some exceptions: +[HTTP BitTorrent protocol]: https://wiki.theory.org/index.php/BitTorrentSpecification#Tracker_HTTP.2FHTTPS_Protocol - * Doesn't track of the number of torrent downloads (0 is always sent). +Aims for compatibility with the [HTTP BitTorrent protocol], with some exceptions: + + * Only runs over TLS + * Doesn't track of the number of torrent downloads (0 is always sent) * Doesn't allow full scrapes, i.e. of all registered info hashes `aquatic_http` has not been tested as much as `aquatic_udp` but likely works fine. +A TLS certificate file (DER-encoded X.509) and a corresponding private key file +(DER-encoded ASN.1 in either PKCS#8 or PKCS#1 format) are required. Set their +paths in the configuration file, e.g.: + +```toml +[network] +address = '0.0.0.0:3000' +tls_certificate_path = './cert.crt' +tls_private_key_path = './key.pk8' +``` + +### aquatic_ws: WebTorrent tracker + +Aims for compatibility with [WebTorrent](https://github.com/webtorrent) +clients, including `wss` protocol support (WebSockets over TLS), with some +exceptions: + + * Doesn't track of the number of torrent downloads (0 is always sent). + * Doesn't allow full scrapes, i.e. of all registered info hashes + #### TLS To run over TLS, a pkcs12 file (`.pkx`) is needed. It can be generated from @@ -164,24 +182,14 @@ Enter a password when prompted. Then move `identity.pfx` somewhere suitable, and enter the path into the tracker configuration field `tls_pkcs12_path`. Set the password in the field `tls_pkcs12_password` and set `use_tls` to true. -### aquatic_ws: WebTorrent tracker - -Aims for compatibility with [WebTorrent](https://github.com/webtorrent) -clients, including `wss` protocol support (WebSockets over TLS), with some -exceptions: - - * Doesn't track of the number of torrent downloads (0 is always sent). - * Doesn't allow full scrapes, i.e. of all registered info hashes - -For information about running over TLS, please refer to the TLS subsection -of the `aquatic_http` section above. - #### Benchmarks [wt-tracker]: https://github.com/Novage/wt-tracker [bittorrent-tracker]: https://github.com/webtorrent/bittorrent-tracker -The following benchmark is not very realistic, as it simulates a small number of clients, each sending a large number of requests. Nonetheless, I think that it gives a useful indication of relative performance. +The following benchmark is not very realistic, as it simulates a small number +of clients, each sending a large number of requests. Nonetheless, I think that +it gives a useful indication of relative performance. Server responses per second, best result in bold: From 1fca54bfe93ab38629e53c84d9811f386a43b818 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20Frosteg=C3=A5rd?= Date: Thu, 28 Oct 2021 18:25:51 +0200 Subject: [PATCH 55/59] Improve README --- README.md | 39 ++++++++++++++++++--------------------- 1 file changed, 18 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 3a68021..864df13 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ Copyright (c) 2020-2021 Joakim Frostegård Distributed under Apache 2.0 license (details in `LICENSE` file.) -## Building +## Usage ### Prerequisites @@ -47,18 +47,9 @@ cargo build --release -p aquatic_http cargo build --release -p aquatic_ws ``` -## Running +### Running -To start a tracker with default configuration, run any of: - -```sh -./target/release/aquatic_udp -./target/release/aquatic_http -./target/release/aquatic_ws -``` - -To adjust the configuration, begin by generating configuration files. They -differ between protocols. +Begin by generating configuration files. They differ between protocols. ```sh ./target/release/aquatic_udp -p > "aquatic-udp-config.toml" @@ -66,7 +57,15 @@ differ between protocols. ./target/release/aquatic_ws -p > "aquatic-ws-config.toml" ``` -Make adjustments to the files. Then run the tracker with: +Make adjustments to the files. The values you will most likely want to adjust +are `socket_workers` (number of threads reading from and writing to sockets) +and `address` under the `network` section (listening address). This goes for +all three protocols. + +`aquatic_http` requires configuring a TLS certificate file and a private key file +to run. More information is available futher down in this document. + +Once done, run the tracker: ```sh ./target/release/aquatic_udp -c "aquatic-udp-config.toml" @@ -74,12 +73,13 @@ Make adjustments to the files. Then run the tracker with: ./target/release/aquatic_ws -c "aquatic-ws-config.toml" ``` -The configuration file values you will most likely want to adjust are -`socket_workers` (number of threads reading from and writing to sockets) and -`address` under the `network` section (listening address). This goes for all -three protocols. +More documentation of configuration file values might be available under +`src/lib/config.rs` in crates `aquatic_udp`, `aquatic_http`, `aquatic_ws`. -Access control by info hash is supported for all protocols. Relevant part of configuration: +#### General settings + +Access control by info hash is supported for all protocols. The relevant part +of configuration is: ```toml [access_list] @@ -87,9 +87,6 @@ mode = 'off' # Change to 'black' (blacklist) or 'white' (whitelist) path = '' # Path to text file with newline-delimited hex-encoded info hashes ``` -More documentation of configuration file values might be available under -`src/lib/config.rs` in crates `aquatic_udp`, `aquatic_http`, `aquatic_ws`. - ## Details on implementations ### aquatic_udp: UDP BitTorrent tracker From b938a6c723914f48734af3c91b1979897a709911 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20Frosteg=C3=A5rd?= Date: Thu, 28 Oct 2021 18:29:26 +0200 Subject: [PATCH 56/59] Improve README --- README.md | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 864df13..a9981f3 100644 --- a/README.md +++ b/README.md @@ -2,9 +2,8 @@ [![CargoBuildAndTest](https://github.com/greatest-ape/aquatic/actions/workflows/cargo-build-and-test.yml/badge.svg)](https://github.com/greatest-ape/aquatic/actions/workflows/cargo-build-and-test.yml) [![Test HTTP, UDP and WSS file transfer](https://github.com/greatest-ape/aquatic/actions/workflows/test-transfer.yml/badge.svg)](https://github.com/greatest-ape/aquatic/actions/workflows/test-transfer.yml) -Blazingly fast, multi-threaded BitTorrent tracker written in Rust. - -Consists of sub-implementations for different protocols: +Blazingly fast, multi-threaded BitTorrent tracker written in Rust, consisting +of sub-implementations for different protocols: [BitTorrent over UDP]: https://libtorrent.org/udp_tracker_protocol.html [BitTorrent over HTTP]: https://wiki.theory.org/index.php/BitTorrentSpecification#Tracker_HTTP.2FHTTPS_Protocol @@ -20,12 +19,6 @@ Consists of sub-implementations for different protocols: | aquatic_http | [BitTorrent over HTTP] with TLS ([rustls]) | Linux 5.8+ | | aquatic_ws | [WebTorrent], plain or with TLS ([native-tls]) | Cross-platform | -## Copyright and license - -Copyright (c) 2020-2021 Joakim Frostegård - -Distributed under Apache 2.0 license (details in `LICENSE` file.) - ## Usage ### Prerequisites @@ -145,6 +138,8 @@ Aims for compatibility with the [HTTP BitTorrent protocol], with some exceptions `aquatic_http` has not been tested as much as `aquatic_udp` but likely works fine. +#### TLS + A TLS certificate file (DER-encoded X.509) and a corresponding private key file (DER-encoded ASN.1 in either PKCS#8 or PKCS#1 format) are required. Set their paths in the configuration file, e.g.: @@ -235,6 +230,12 @@ This design means little waiting for locks on internal state occurs, while network work can be efficiently distributed over multiple threads, making use of SO_REUSEPORT setting. +## Copyright and license + +Copyright (c) 2020-2021 Joakim Frostegård + +Distributed under Apache 2.0 license (details in `LICENSE` file.) + ## Trivia The tracker is called aquatic because it thrives under a torrent of bits ;-) From 2deadb2fff5d1667e495167f7e6c17aee3963852 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20Frosteg=C3=A5rd?= Date: Thu, 28 Oct 2021 18:37:41 +0200 Subject: [PATCH 57/59] README: improve info on libssl-dev installation --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a9981f3..9bc7e20 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ of sub-implementations for different protocols: - Install Rust with [rustup](https://rustup.rs/) (stable is recommended) - Install cmake with your package manager (e.g., `apt-get install cmake`) -- If you want to run aquatic_ws and are on a Unix-like OS, install the OpenSSL +- If you want to run aquatic_ws and are on Linux or BSD, install OpenSSL components necessary for dynamic linking (e.g., `apt-get install libssl-dev`) - Clone this git repository and enter it From af0761418e51a87ff00b360b23277bc0e9330a92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20Frosteg=C3=A5rd?= Date: Thu, 28 Oct 2021 19:30:04 +0200 Subject: [PATCH 58/59] aquatic_http: improve ConnectionMeta field names and types --- TODO.md | 1 - aquatic_http/src/lib/common.rs | 9 +++++---- aquatic_http/src/lib/handlers.rs | 16 ++++++++-------- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/TODO.md b/TODO.md index fcb5aa3..2dc446a 100644 --- a/TODO.md +++ b/TODO.md @@ -1,7 +1,6 @@ # TODO * aquatic_http glommio: - * get rid of / improve ConnectionMeta stuff in handler * clean out connections regularly * timeout inside of task for "it took to long to receive request, send response"? * handle panicked/cancelled tasks diff --git a/aquatic_http/src/lib/common.rs b/aquatic_http/src/lib/common.rs index b64d30e..e814c36 100644 --- a/aquatic_http/src/lib/common.rs +++ b/aquatic_http/src/lib/common.rs @@ -129,15 +129,16 @@ impl Ip for Ipv6Addr {} pub struct ConnectionMeta { /// Index of socket worker responsible for this connection. Required for /// sending back response through correct channel to correct worker. - pub worker_index: usize, // Or response consumer id in glommio + pub response_consumer_id: ConsumerId, pub peer_addr: SocketAddr, - pub poll_token: usize, // Or connection id in glommio + /// Connection id local to socket worker + pub connection_id: ConnectionId, } #[derive(Clone, Copy, Debug)] pub struct PeerConnectionMeta { - pub worker_index: usize, - pub poll_token: usize, + pub response_consumer_id: ConsumerId, + pub connection_id: ConnectionId, pub peer_ip_address: I, } diff --git a/aquatic_http/src/lib/handlers.rs b/aquatic_http/src/lib/handlers.rs index dd2d6ed..d30fe64 100644 --- a/aquatic_http/src/lib/handlers.rs +++ b/aquatic_http/src/lib/handlers.rs @@ -97,8 +97,8 @@ async fn handle_request_stream( connection_id, } => { let meta = ConnectionMeta { - worker_index: response_consumer_id.0, - poll_token: connection_id.0, + response_consumer_id, + connection_id, peer_addr, }; @@ -126,8 +126,8 @@ async fn handle_request_stream( connection_id, } => { let meta = ConnectionMeta { - worker_index: response_consumer_id.0, - poll_token: connection_id.0, + response_consumer_id, + connection_id, peer_addr, }; @@ -172,8 +172,8 @@ pub fn handle_announce_request( torrent_maps.ipv4.entry(request.info_hash).or_default(); let peer_connection_meta = PeerConnectionMeta { - worker_index: meta.worker_index, - poll_token: meta.poll_token, + response_consumer_id: meta.response_consumer_id, + connection_id: meta.connection_id, peer_ip_address, }; @@ -201,8 +201,8 @@ pub fn handle_announce_request( torrent_maps.ipv6.entry(request.info_hash).or_default(); let peer_connection_meta = PeerConnectionMeta { - worker_index: meta.worker_index, - poll_token: meta.poll_token, + response_consumer_id: meta.response_consumer_id, + connection_id: meta.connection_id, peer_ip_address, }; From 4f055c7c6f706cdbb7b722cef510b4d42728de3f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20Frosteg=C3=A5rd?= Date: Thu, 28 Oct 2021 20:23:56 +0200 Subject: [PATCH 59/59] Update TODO --- TODO.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/TODO.md b/TODO.md index 2dc446a..1a1d3ce 100644 --- a/TODO.md +++ b/TODO.md @@ -1,6 +1,9 @@ # TODO * aquatic_http glommio: + * optimize? + * get_peer_addr only once (takes 1.2% of runtime) + * queue response: allocating takes 2.8% of runtime * clean out connections regularly * timeout inside of task for "it took to long to receive request, send response"? * handle panicked/cancelled tasks