Move all crates to new crates dir

crates/http/src/common.rs

@@ -0,0 +1,37 @@
+use std::sync::Arc;
+
+use aquatic_common::access_list::AccessListArcSwap;
+use aquatic_common::CanonicalSocketAddr;
+
+pub use aquatic_common::ValidUntil;
+
+use aquatic_http_protocol::{
+    request::{AnnounceRequest, ScrapeRequest},
+    response::{AnnounceResponse, ScrapeResponse},
+};
+use glommio::channels::shared_channel::SharedSender;
+
+#[derive(Copy, Clone, Debug)]
+pub struct ConsumerId(pub usize);
+
+#[derive(Clone, Copy, Debug)]
+pub struct ConnectionId(pub usize);
+
+#[derive(Debug)]
+pub enum ChannelRequest {
+    Announce {
+        request: AnnounceRequest,
+        peer_addr: CanonicalSocketAddr,
+        response_sender: SharedSender<AnnounceResponse>,
+    },
+    Scrape {
+        request: ScrapeRequest,
+        peer_addr: CanonicalSocketAddr,
+        response_sender: SharedSender<ScrapeResponse>,
+    },
+}
+
+#[derive(Default, Clone)]
+pub struct State {
+    pub access_list: Arc<AccessListArcSwap>,
+}

crates/http/src/config.rs

@@ -0,0 +1,163 @@
+use std::{net::SocketAddr, path::PathBuf};
+
+use aquatic_common::{
+    access_list::AccessListConfig, cpu_pinning::asc::CpuPinningConfigAsc,
+    privileges::PrivilegeConfig,
+};
+use aquatic_toml_config::TomlConfig;
+use serde::Deserialize;
+
+use aquatic_common::cli::LogLevel;
+
+/// aquatic_http configuration
+///
+/// Does not support running behind a reverse proxy.
+#[derive(Clone, Debug, PartialEq, TomlConfig, Deserialize)]
+#[serde(default, deny_unknown_fields)]
+pub struct Config {
+    /// Socket workers receive requests from the socket, parse them and send
+    /// them on to the swarm workers. They then receive responses from the
+    /// swarm workers, encode them and send them back over the socket.
+    pub socket_workers: usize,
+    /// Swarm workers receive a number of requests from socket workers,
+    /// generate responses and send them back to the socket workers.
+    pub swarm_workers: usize,
+    pub log_level: LogLevel,
+    pub network: NetworkConfig,
+    pub protocol: ProtocolConfig,
+    pub cleaning: CleaningConfig,
+    pub privileges: PrivilegeConfig,
+    pub access_list: AccessListConfig,
+    pub cpu_pinning: CpuPinningConfigAsc,
+    #[cfg(feature = "metrics")]
+    pub metrics: MetricsConfig,
+}
+
+impl Default for Config {
+    fn default() -> Self {
+        Self {
+            socket_workers: 1,
+            swarm_workers: 1,
+            log_level: LogLevel::default(),
+            network: NetworkConfig::default(),
+            protocol: ProtocolConfig::default(),
+            cleaning: CleaningConfig::default(),
+            privileges: PrivilegeConfig::default(),
+            access_list: AccessListConfig::default(),
+            cpu_pinning: Default::default(),
+            #[cfg(feature = "metrics")]
+            metrics: Default::default(),
+        }
+    }
+}
+
+impl aquatic_common::cli::Config for Config {
+    fn get_log_level(&self) -> Option<LogLevel> {
+        Some(self.log_level)
+    }
+}
+
+#[derive(Clone, Debug, PartialEq, TomlConfig, Deserialize)]
+#[serde(default, deny_unknown_fields)]
+pub struct NetworkConfig {
+    /// Bind to this address
+    pub address: SocketAddr,
+    /// Only allow access over IPv6
+    pub only_ipv6: bool,
+    /// Maximum number of pending TCP connections
+    pub tcp_backlog: i32,
+    /// Path to TLS certificate (DER-encoded X.509)
+    pub tls_certificate_path: PathBuf,
+    /// Path to TLS private key (DER-encoded ASN.1 in PKCS#8 or PKCS#1 format)
+    pub tls_private_key_path: PathBuf,
+    /// Keep connections alive after sending a response
+    pub keep_alive: bool,
+}
+
+impl Default for NetworkConfig {
+    fn default() -> Self {
+        Self {
+            address: SocketAddr::from(([0, 0, 0, 0], 3000)),
+            tls_certificate_path: "".into(),
+            tls_private_key_path: "".into(),
+            only_ipv6: false,
+            tcp_backlog: 1024,
+            keep_alive: true,
+        }
+    }
+}
+
+#[derive(Clone, Debug, PartialEq, TomlConfig, Deserialize)]
+#[serde(default, deny_unknown_fields)]
+pub struct ProtocolConfig {
+    /// Maximum number of torrents to accept in scrape request
+    pub max_scrape_torrents: usize,
+    /// Maximum number of requested peers to accept in announce request
+    pub max_peers: usize,
+    /// Ask peers to announce this often (seconds)
+    pub peer_announce_interval: usize,
+}
+
+impl Default for ProtocolConfig {
+    fn default() -> Self {
+        Self {
+            max_scrape_torrents: 100,
+            max_peers: 50,
+            peer_announce_interval: 120,
+        }
+    }
+}
+
+#[derive(Clone, Debug, PartialEq, TomlConfig, Deserialize)]
+#[serde(default, deny_unknown_fields)]
+pub struct CleaningConfig {
+    /// Clean peers this often (seconds)
+    pub torrent_cleaning_interval: u64,
+    /// Clean connections this often (seconds)
+    pub connection_cleaning_interval: u64,
+    /// Remove peers that have not announced for this long (seconds)
+    pub max_peer_age: u32,
+    /// Remove connections that haven't seen valid requests for this long (seconds)
+    pub max_connection_idle: u32,
+}
+
+impl Default for CleaningConfig {
+    fn default() -> Self {
+        Self {
+            torrent_cleaning_interval: 30,
+            connection_cleaning_interval: 60,
+            max_peer_age: 1800,
+            max_connection_idle: 180,
+        }
+    }
+}
+
+#[cfg(feature = "metrics")]
+#[derive(Clone, Debug, PartialEq, TomlConfig, Deserialize)]
+#[serde(default, deny_unknown_fields)]
+pub struct MetricsConfig {
+    /// Run a prometheus endpoint
+    pub run_prometheus_endpoint: bool,
+    /// Address to run prometheus endpoint on
+    pub prometheus_endpoint_address: SocketAddr,
+    /// Update metrics for torrent count this often (seconds)
+    pub torrent_count_update_interval: u64,
+}
+
+#[cfg(feature = "metrics")]
+impl Default for MetricsConfig {
+    fn default() -> Self {
+        Self {
+            run_prometheus_endpoint: false,
+            prometheus_endpoint_address: SocketAddr::from(([0, 0, 0, 0], 9000)),
+            torrent_count_update_interval: 10,
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::Config;
+
+    ::aquatic_toml_config::gen_serialize_deserialize_test!(Config);
+}

crates/http/src/lib.rs

@@ -0,0 +1,160 @@
+use anyhow::Context;
+use aquatic_common::{
+    access_list::update_access_list,
+    cpu_pinning::{
+        glommio::{get_worker_placement, set_affinity_for_util_worker},
+        WorkerIndex,
+    },
+    privileges::PrivilegeDropper,
+    rustls_config::create_rustls_config,
+    PanicSentinelWatcher, ServerStartInstant,
+};
+use common::State;
+use glommio::{channels::channel_mesh::MeshBuilder, prelude::*};
+use signal_hook::{
+    consts::{SIGTERM, SIGUSR1},
+    iterator::Signals,
+};
+use std::sync::Arc;
+
+use crate::config::Config;
+
+mod common;
+pub mod config;
+mod workers;
+
+pub const APP_NAME: &str = "aquatic_http: BitTorrent tracker (HTTP over TLS)";
+pub const APP_VERSION: &str = env!("CARGO_PKG_VERSION");
+
+const SHARED_CHANNEL_SIZE: usize = 1024;
+
+pub fn run(config: Config) -> ::anyhow::Result<()> {
+    let mut signals = Signals::new([SIGUSR1, SIGTERM])?;
+
+    #[cfg(feature = "prometheus")]
+    if config.metrics.run_prometheus_endpoint {
+        use metrics_exporter_prometheus::PrometheusBuilder;
+
+        PrometheusBuilder::new()
+            .with_http_listener(config.metrics.prometheus_endpoint_address)
+            .install()
+            .with_context(|| {
+                format!(
+                    "Install prometheus endpoint on {}",
+                    config.metrics.prometheus_endpoint_address
+                )
+            })?;
+    }
+
+    let state = State::default();
+
+    update_access_list(&config.access_list, &state.access_list)?;
+
+    let num_peers = config.socket_workers + config.swarm_workers;
+
+    let request_mesh_builder = MeshBuilder::partial(num_peers, SHARED_CHANNEL_SIZE);
+
+    let (sentinel_watcher, sentinel) = PanicSentinelWatcher::create_with_sentinel();
+    let priv_dropper = PrivilegeDropper::new(config.privileges.clone(), config.socket_workers);
+
+    let tls_config = Arc::new(create_rustls_config(
+        &config.network.tls_certificate_path,
+        &config.network.tls_private_key_path,
+    )?);
+
+    let server_start_instant = ServerStartInstant::new();
+
+    let mut executors = Vec::new();
+
+    for i in 0..(config.socket_workers) {
+        let sentinel = sentinel.clone();
+        let config = config.clone();
+        let state = state.clone();
+        let tls_config = tls_config.clone();
+        let request_mesh_builder = request_mesh_builder.clone();
+        let priv_dropper = priv_dropper.clone();
+
+        let placement = get_worker_placement(
+            &config.cpu_pinning,
+            config.socket_workers,
+            config.swarm_workers,
+            WorkerIndex::SocketWorker(i),
+        )?;
+        let builder = LocalExecutorBuilder::new(placement).name(&format!("socket-{:02}", i + 1));
+
+        let executor = builder
+            .spawn(move || async move {
+                workers::socket::run_socket_worker(
+                    sentinel,
+                    config,
+                    state,
+                    tls_config,
+                    request_mesh_builder,
+                    priv_dropper,
+                    server_start_instant,
+                    i,
+                )
+                .await
+            })
+            .map_err(|err| anyhow::anyhow!("Spawning executor failed: {:#}", err))?;
+
+        executors.push(executor);
+    }
+
+    for i in 0..(config.swarm_workers) {
+        let sentinel = sentinel.clone();
+        let config = config.clone();
+        let state = state.clone();
+        let request_mesh_builder = request_mesh_builder.clone();
+
+        let placement = get_worker_placement(
+            &config.cpu_pinning,
+            config.socket_workers,
+            config.swarm_workers,
+            WorkerIndex::SwarmWorker(i),
+        )?;
+        let builder = LocalExecutorBuilder::new(placement).name(&format!("swarm-{:02}", i + 1));
+
+        let executor = builder
+            .spawn(move || async move {
+                workers::swarm::run_swarm_worker(
+                    sentinel,
+                    config,
+                    state,
+                    request_mesh_builder,
+                    server_start_instant,
+                    i,
+                )
+                .await
+            })
+            .map_err(|err| anyhow::anyhow!("Spawning executor failed: {:#}", err))?;
+
+        executors.push(executor);
+    }
+
+    if config.cpu_pinning.active {
+        set_affinity_for_util_worker(
+            &config.cpu_pinning,
+            config.socket_workers,
+            config.swarm_workers,
+        )?;
+    }
+
+    for signal in &mut signals {
+        match signal {
+            SIGUSR1 => {
+                let _ = update_access_list(&config.access_list, &state.access_list);
+            }
+            SIGTERM => {
+                if sentinel_watcher.panic_was_triggered() {
+                    return Err(anyhow::anyhow!("worker thread panicked"));
+                } else {
+                    return Ok(());
+                }
+            }
+            _ => unreachable!(),
+        }
+    }
+
+    Ok(())
+}

crates/http/src/main.rs

@@ -0,0 +1,14 @@
+use aquatic_common::cli::run_app_with_cli_and_config;
+use aquatic_http::config::Config;
+
+#[global_allocator]
+static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc;
+
+fn main() {
+    run_app_with_cli_and_config::<Config>(
+        aquatic_http::APP_NAME,
+        aquatic_http::APP_VERSION,
+        aquatic_http::run,
+        None,
+    )
+}

crates/http/src/workers/mod.rs

@@ -0,0 +1,2 @@
+pub mod socket;
+pub mod swarm;

crates/http/src/workers/socket.rs

@@ -0,0 +1,575 @@
+use std::cell::RefCell;
+use std::collections::BTreeMap;
+use std::os::unix::prelude::{FromRawFd, IntoRawFd};
+use std::rc::Rc;
+use std::sync::Arc;
+use std::time::Duration;
+
+use anyhow::Context;
+use aquatic_common::access_list::{create_access_list_cache, AccessListArcSwap, AccessListCache};
+use aquatic_common::privileges::PrivilegeDropper;
+use aquatic_common::rustls_config::RustlsConfig;
+use aquatic_common::{CanonicalSocketAddr, PanicSentinel, ServerStartInstant};
+use aquatic_http_protocol::common::InfoHash;
+use aquatic_http_protocol::request::{Request, RequestParseError, ScrapeRequest};
+use aquatic_http_protocol::response::{
+    FailureResponse, Response, ScrapeResponse, ScrapeStatistics,
+};
+use either::Either;
+use futures::stream::FuturesUnordered;
+use futures_lite::{AsyncReadExt, AsyncWriteExt, StreamExt};
+use futures_rustls::server::TlsStream;
+use futures_rustls::TlsAcceptor;
+use glommio::channels::channel_mesh::{MeshBuilder, Partial, Role, Senders};
+use glommio::channels::shared_channel::{self, SharedReceiver};
+use glommio::net::{TcpListener, TcpStream};
+use glommio::task::JoinHandle;
+use glommio::timer::TimerActionRepeat;
+use glommio::{enclose, prelude::*};
+use once_cell::sync::Lazy;
+use slab::Slab;
+
+use crate::common::*;
+use crate::config::Config;
+
+const REQUEST_BUFFER_SIZE: usize = 2048;
+const RESPONSE_BUFFER_SIZE: usize = 4096;
+
+const RESPONSE_HEADER_A: &[u8] = b"HTTP/1.1 200 OK\r\nContent-Length: ";
+const RESPONSE_HEADER_B: &[u8] = b"        ";
+const RESPONSE_HEADER_C: &[u8] = b"\r\n\r\n";
+
+#[cfg(feature = "metrics")]
+thread_local! { static WORKER_INDEX: ::std::cell::Cell<usize> = Default::default() }
+
+static RESPONSE_HEADER: Lazy<Vec<u8>> =
+    Lazy::new(|| [RESPONSE_HEADER_A, RESPONSE_HEADER_B, RESPONSE_HEADER_C].concat());
+
+struct PendingScrapeResponse {
+    pending_worker_responses: usize,
+    stats: BTreeMap<InfoHash, ScrapeStatistics>,
+}
+
+struct ConnectionReference {
+    task_handle: Option<JoinHandle<()>>,
+    valid_until: ValidUntil,
+}
+
+pub async fn run_socket_worker(
+    _sentinel: PanicSentinel,
+    config: Config,
+    state: State,
+    tls_config: Arc<RustlsConfig>,
+    request_mesh_builder: MeshBuilder<ChannelRequest, Partial>,
+    priv_dropper: PrivilegeDropper,
+    server_start_instant: ServerStartInstant,
+    worker_index: usize,
+) {
+    #[cfg(feature = "metrics")]
+    WORKER_INDEX.with(|index| index.set(worker_index));
+
+    let config = Rc::new(config);
+    let access_list = state.access_list;
+
+    let listener = create_tcp_listener(&config, priv_dropper).expect("create tcp listener");
+
+    let (request_senders, _) = request_mesh_builder.join(Role::Producer).await.unwrap();
+    let request_senders = Rc::new(request_senders);
+
+    let connection_slab = Rc::new(RefCell::new(Slab::new()));
+
+    TimerActionRepeat::repeat(enclose!((config, connection_slab) move || {
+        clean_connections(
+            config.clone(),
+            connection_slab.clone(),
+            server_start_instant,
+        )
+    }));
+
+    let mut incoming = listener.incoming();
+
+    while let Some(stream) = incoming.next().await {
+        match stream {
+            Ok(stream) => {
+                let key = connection_slab.borrow_mut().insert(ConnectionReference {
+                    task_handle: None,
+                    valid_until: ValidUntil::new(
+                        server_start_instant,
+                        config.cleaning.max_connection_idle,
+                    ),
+                });
+
+                let task_handle = spawn_local(enclose!((config, access_list, request_senders, tls_config, connection_slab) async move {
+                    let result = match stream.peer_addr() {
+                        Ok(peer_addr) => {
+                            let peer_addr = CanonicalSocketAddr::new(peer_addr);
+
+                            #[cfg(feature = "metrics")]
+                            let ip_version_str = peer_addr_to_ip_version_str(&peer_addr);
+
+                            #[cfg(feature = "metrics")]
+                            ::metrics::increment_gauge!(
+                                "aquatic_active_connections",
+                                1.0,
+                                "ip_version" => ip_version_str,
+                                "worker_index" => worker_index.to_string(),
+                            );
+
+                            let result = Connection::run(
+                                config,
+                                access_list,
+                                request_senders,
+                                server_start_instant,
+                                ConnectionId(key),
+                                tls_config,
+                                connection_slab.clone(),
+                                stream,
+                                peer_addr
+                            ).await;
+
+                            #[cfg(feature = "metrics")]
+                            ::metrics::decrement_gauge!(
+                                "aquatic_active_connections",
+                                1.0,
+                                "ip_version" => ip_version_str,
+                                "worker_index" => worker_index.to_string(),
+                            );
+
+                            result
+                        }
+                        Err(err) => {
+                            Err(anyhow::anyhow!("Couldn't get peer addr: {:?}", err))
+                        }
+                    };
+
+                    if let Err(err) = result {
+                        ::log::debug!("Connection::run() error: {:?}", err);
+                    }
+
+                    connection_slab.borrow_mut().try_remove(key);
+                }))
+                .detach();
+
+                if let Some(reference) = connection_slab.borrow_mut().get_mut(key) {
+                    reference.task_handle = Some(task_handle);
+                }
+            }
+            Err(err) => {
+                ::log::error!("accept connection: {:?}", err);
+            }
+        }
+    }
+}
+
+async fn clean_connections(
+    config: Rc<Config>,
+    connection_slab: Rc<RefCell<Slab<ConnectionReference>>>,
+    server_start_instant: ServerStartInstant,
+) -> Option<Duration> {
+    let now = server_start_instant.seconds_elapsed();
+
+    connection_slab.borrow_mut().retain(|_, reference| {
+        if reference.valid_until.valid(now) {
+            true
+        } else {
+            if let Some(ref handle) = reference.task_handle {
+                handle.cancel();
+            }
+
+            false
+        }
+    });
+
+    connection_slab.borrow_mut().shrink_to_fit();
+
+    Some(Duration::from_secs(
+        config.cleaning.connection_cleaning_interval,
+    ))
+}
+
+struct Connection {
+    config: Rc<Config>,
+    access_list_cache: AccessListCache,
+    request_senders: Rc<Senders<ChannelRequest>>,
+    connection_slab: Rc<RefCell<Slab<ConnectionReference>>>,
+    server_start_instant: ServerStartInstant,
+    stream: TlsStream<TcpStream>,
+    peer_addr: CanonicalSocketAddr,
+    connection_id: ConnectionId,
+    request_buffer: [u8; REQUEST_BUFFER_SIZE],
+    request_buffer_position: usize,
+    response_buffer: [u8; RESPONSE_BUFFER_SIZE],
+}
+
+impl Connection {
+    async fn run(
+        config: Rc<Config>,
+        access_list: Arc<AccessListArcSwap>,
+        request_senders: Rc<Senders<ChannelRequest>>,
+        server_start_instant: ServerStartInstant,
+        connection_id: ConnectionId,
+        tls_config: Arc<RustlsConfig>,
+        connection_slab: Rc<RefCell<Slab<ConnectionReference>>>,
+        stream: TcpStream,
+        peer_addr: CanonicalSocketAddr,
+    ) -> anyhow::Result<()> {
+        let tls_acceptor: TlsAcceptor = tls_config.into();
+        let stream = tls_acceptor.accept(stream).await?;
+
+        let mut response_buffer = [0; RESPONSE_BUFFER_SIZE];
+
+        response_buffer[..RESPONSE_HEADER.len()].copy_from_slice(&RESPONSE_HEADER);
+
+        let mut conn = Connection {
+            config: config.clone(),
+            access_list_cache: create_access_list_cache(&access_list),
+            request_senders: request_senders.clone(),
+            connection_slab,
+            server_start_instant,
+            stream,
+            peer_addr,
+            connection_id,
+            request_buffer: [0; REQUEST_BUFFER_SIZE],
+            request_buffer_position: 0,
+            response_buffer,
+        };
+
+        conn.run_request_response_loop().await?;
+
+        Ok(())
+    }
+
+    async fn run_request_response_loop(&mut self) -> anyhow::Result<()> {
+        loop {
+            let response = match self.read_request().await? {
+                Either::Left(response) => Response::Failure(response),
+                Either::Right(request) => self.handle_request(request).await?,
+            };
+
+            self.write_response(&response).await?;
+
+            if matches!(response, Response::Failure(_)) || !self.config.network.keep_alive {
+                let _ = self
+                    .stream
+                    .get_ref()
+                    .0
+                    .shutdown(std::net::Shutdown::Both)
+                    .await;
+
+                break;
+            }
+        }
+
+        Ok(())
+    }
+
+    async fn read_request(&mut self) -> anyhow::Result<Either<FailureResponse, Request>> {
+        self.request_buffer_position = 0;
+
+        loop {
+            if self.request_buffer_position == self.request_buffer.len() {
+                return Err(anyhow::anyhow!("request buffer is full"));
+            }
+
+            let bytes_read = self
+                .stream
+                .read(&mut self.request_buffer[self.request_buffer_position..])
+                .await?;
+
+            if bytes_read == 0 {
+                return Err(anyhow::anyhow!("peer closed connection"));
+            }
+
+            self.request_buffer_position += bytes_read;
+
+            match Request::from_bytes(&self.request_buffer[..self.request_buffer_position]) {
+                Ok(request) => {
+                    return Ok(Either::Right(request));
+                }
+                Err(RequestParseError::Invalid(err)) => {
+                    let response = FailureResponse {
+                        failure_reason: "Invalid request".into(),
+                    };
+
+                    ::log::debug!("Invalid request: {:#}", err);
+
+                    return Ok(Either::Left(response));
+                }
+                Err(RequestParseError::NeedMoreData) => {
+                    ::log::debug!(
+                        "need more request data. current data: {}",
+                        &self.request_buffer[..self.request_buffer_position].escape_ascii()
+                    );
+                }
+            }
+        }
+    }
+
+    /// Take a request and:
+    /// - Update connection ValidUntil
+    /// - Return error response if request is not allowed
+    /// - If it is an announce request, send it to swarm workers an await a
+    ///   response
+    /// - If it is a scrape requests, split it up, pass on the parts to
+    ///   relevant swarm workers and await a response
+    async fn handle_request(&mut self, request: Request) -> anyhow::Result<Response> {
+        if let Ok(mut slab) = self.connection_slab.try_borrow_mut() {
+            if let Some(reference) = slab.get_mut(self.connection_id.0) {
+                reference.valid_until = ValidUntil::new(
+                    self.server_start_instant,
+                    self.config.cleaning.max_connection_idle,
+                );
+            }
+        }
+
+        match request {
+            Request::Announce(request) => {
+                #[cfg(feature = "metrics")]
+                ::metrics::increment_counter!(
+                    "aquatic_requests_total",
+                    "type" => "announce",
+                    "ip_version" => peer_addr_to_ip_version_str(&self.peer_addr),
+                    "worker_index" => WORKER_INDEX.with(|index| index.get()).to_string(),
+                );
+
+                let info_hash = request.info_hash;
+
+                if self
+                    .access_list_cache
+                    .load()
+                    .allows(self.config.access_list.mode, &info_hash.0)
+                {
+                    let (response_sender, response_receiver) = shared_channel::new_bounded(1);
+
+                    let request = ChannelRequest::Announce {
+                        request,
+                        peer_addr: self.peer_addr,
+                        response_sender,
+                    };
+
+                    let consumer_index = calculate_request_consumer_index(&self.config, info_hash);
+
+                    // Only fails when receiver is closed
+                    self.request_senders
+                        .send_to(consumer_index, request)
+                        .await
+                        .unwrap();
+
+                    response_receiver
+                        .connect()
+                        .await
+                        .recv()
+                        .await
+                        .ok_or_else(|| anyhow::anyhow!("response sender closed"))
+                        .map(Response::Announce)
+                } else {
+                    let response = Response::Failure(FailureResponse {
+                        failure_reason: "Info hash not allowed".into(),
+                    });
+
+                    Ok(response)
+                }
+            }
+            Request::Scrape(ScrapeRequest { info_hashes }) => {
+                #[cfg(feature = "metrics")]
+                ::metrics::increment_counter!(
+                    "aquatic_requests_total",
+                    "type" => "scrape",
+                    "ip_version" => peer_addr_to_ip_version_str(&self.peer_addr),
+                    "worker_index" => WORKER_INDEX.with(|index| index.get()).to_string(),
+                );
+
+                let mut info_hashes_by_worker: BTreeMap<usize, Vec<InfoHash>> = BTreeMap::new();
+
+                for info_hash in info_hashes.into_iter() {
+                    let info_hashes = info_hashes_by_worker
+                        .entry(calculate_request_consumer_index(&self.config, info_hash))
+                        .or_default();
+
+                    info_hashes.push(info_hash);
+                }
+
+                let pending_worker_responses = info_hashes_by_worker.len();
+                let mut response_receivers = Vec::with_capacity(pending_worker_responses);
+
+                for (consumer_index, info_hashes) in info_hashes_by_worker {
+                    let (response_sender, response_receiver) = shared_channel::new_bounded(1);
+
+                    response_receivers.push(response_receiver);
+
+                    let request = ChannelRequest::Scrape {
+                        request: ScrapeRequest { info_hashes },
+                        peer_addr: self.peer_addr,
+                        response_sender,
+                    };
+
+                    // Only fails when receiver is closed
+                    self.request_senders
+                        .send_to(consumer_index, request)
+                        .await
+                        .unwrap();
+                }
+
+                let pending_scrape_response = PendingScrapeResponse {
+                    pending_worker_responses,
+                    stats: Default::default(),
+                };
+
+                self.wait_for_scrape_responses(response_receivers, pending_scrape_response)
+                    .await
+            }
+        }
+    }
+
+    /// Wait for partial scrape responses to arrive,
+    /// return full response
+    async fn wait_for_scrape_responses(
+        &self,
+        response_receivers: Vec<SharedReceiver<ScrapeResponse>>,
+        mut pending: PendingScrapeResponse,
+    ) -> anyhow::Result<Response> {
+        let mut responses = response_receivers
+            .into_iter()
+            .map(|receiver| async { receiver.connect().await.recv().await })
+            .collect::<FuturesUnordered<_>>();
+
+        loop {
+            let response = responses
+                .next()
+                .await
+                .ok_or_else(|| {
+                    anyhow::anyhow!("stream ended before all partial scrape responses received")
+                })?
+                .ok_or_else(|| {
+                    anyhow::anyhow!(
+                        "wait_for_scrape_response: can't receive response, sender is closed"
+                    )
+                })?;
+
+            pending.stats.extend(response.files);
+            pending.pending_worker_responses -= 1;
+
+            if pending.pending_worker_responses == 0 {
+                let response = Response::Scrape(ScrapeResponse {
+                    files: pending.stats,
+                });
+
+                break Ok(response);
+            }
+        }
+    }
+
+    async fn write_response(&mut self, response: &Response) -> anyhow::Result<()> {
+        // Write body and final newline to response buffer
+
+        let mut position = RESPONSE_HEADER.len();
+
+        let body_len = response.write(&mut &mut self.response_buffer[position..])?;
+
+        position += body_len;
+
+        if position + 2 > self.response_buffer.len() {
+            ::log::error!("Response buffer is too short for response");
+
+            return Err(anyhow::anyhow!("Response buffer is too short for response"));
+        }
+
+        (&mut self.response_buffer[position..position + 2]).copy_from_slice(b"\r\n");
+
+        position += 2;
+
+        let content_len = body_len + 2;
+
+        // Clear content-len header value
+
+        {
+            let start = RESPONSE_HEADER_A.len();
+            let end = start + RESPONSE_HEADER_B.len();
+
+            (&mut self.response_buffer[start..end]).copy_from_slice(RESPONSE_HEADER_B);
+        }
+
+        // Set content-len header value
+
+        {
+            let mut buf = ::itoa::Buffer::new();
+            let content_len_bytes = buf.format(content_len).as_bytes();
+
+            let start = RESPONSE_HEADER_A.len();
+            let end = start + content_len_bytes.len();
+
+            (&mut self.response_buffer[start..end]).copy_from_slice(content_len_bytes);
+        }
+
+        // Write buffer to stream
+
+        self.stream.write(&self.response_buffer[..position]).await?;
+        self.stream.flush().await?;
+
+        #[cfg(feature = "metrics")]
+        {
+            let response_type = match response {
+                Response::Announce(_) => "announce",
+                Response::Scrape(_) => "scrape",
+                Response::Failure(_) => "error",
+            };
+
+            ::metrics::increment_counter!(
+                "aquatic_responses_total",
+                "type" => response_type,
+                "ip_version" => peer_addr_to_ip_version_str(&self.peer_addr),
+                "worker_index" => WORKER_INDEX.with(|index| index.get()).to_string(),
+            );
+        }
+
+        Ok(())
+    }
+}
+
+fn calculate_request_consumer_index(config: &Config, info_hash: InfoHash) -> usize {
+    (info_hash.0[0] as usize) % config.swarm_workers
+}
+
+fn create_tcp_listener(
+    config: &Config,
+    priv_dropper: PrivilegeDropper,
+) -> anyhow::Result<TcpListener> {
+    let domain = if config.network.address.is_ipv4() {
+        socket2::Domain::IPV4
+    } else {
+        socket2::Domain::IPV6
+    };
+
+    let socket = socket2::Socket::new(domain, socket2::Type::STREAM, Some(socket2::Protocol::TCP))?;
+
+    if config.network.only_ipv6 {
+        socket
+            .set_only_v6(true)
+            .with_context(|| "socket: set only ipv6")?;
+    }
+
+    socket
+        .set_reuse_port(true)
+        .with_context(|| "socket: set reuse port")?;
+
+    socket
+        .bind(&config.network.address.into())
+        .with_context(|| format!("socket: bind to {}", config.network.address))?;
+
+    socket
+        .listen(config.network.tcp_backlog)
+        .with_context(|| format!("socket: listen on {}", config.network.address))?;
+
+    priv_dropper.after_socket_creation()?;
+
+    Ok(unsafe { TcpListener::from_raw_fd(socket.into_raw_fd()) })
+}
+
+#[cfg(feature = "metrics")]
+fn peer_addr_to_ip_version_str(addr: &CanonicalSocketAddr) -> &'static str {
+    if addr.is_ipv4() {
+        "4"
+    } else {
+        "6"
+    }
+}

crates/http/src/workers/swarm.rs

@@ -0,0 +1,517 @@
+use std::cell::RefCell;
+use std::collections::BTreeMap;
+use std::net::{IpAddr, Ipv4Addr, Ipv6Addr};
+use std::rc::Rc;
+use std::sync::Arc;
+use std::time::Duration;
+
+use futures_lite::{Stream, StreamExt};
+use glommio::channels::channel_mesh::{MeshBuilder, Partial, Role};
+use glommio::timer::TimerActionRepeat;
+use glommio::{enclose, prelude::*};
+use rand::prelude::SmallRng;
+use rand::Rng;
+use rand::SeedableRng;
+
+use aquatic_common::access_list::{create_access_list_cache, AccessListArcSwap, AccessListCache};
+use aquatic_common::{
+    extract_response_peers, CanonicalSocketAddr, IndexMap, PanicSentinel, SecondsSinceServerStart,
+    ServerStartInstant, ValidUntil,
+};
+use aquatic_http_protocol::common::*;
+use aquatic_http_protocol::request::*;
+use aquatic_http_protocol::response::ResponsePeer;
+use aquatic_http_protocol::response::*;
+
+use crate::common::*;
+use crate::config::Config;
+
+#[cfg(feature = "metrics")]
+thread_local! { static WORKER_INDEX: ::std::cell::Cell<usize> = Default::default() }
+
+pub trait Ip: ::std::fmt::Debug + Copy + Eq + ::std::hash::Hash {
+    #[cfg(feature = "metrics")]
+    fn ip_version_str() -> &'static str;
+}
+
+impl Ip for Ipv4Addr {
+    #[cfg(feature = "metrics")]
+    fn ip_version_str() -> &'static str {
+        "4"
+    }
+}
+impl Ip for Ipv6Addr {
+    #[cfg(feature = "metrics")]
+    fn ip_version_str() -> &'static str {
+        "6"
+    }
+}
+
+#[derive(PartialEq, Eq, Clone, Copy, Debug)]
+pub enum PeerStatus {
+    Seeding,
+    Leeching,
+    Stopped,
+}
+
+impl PeerStatus {
+    /// Determine peer status from announce event and number of bytes left.
+    ///
+    /// Likely, the last branch will be taken most of the time.
+    #[inline]
+    pub fn from_event_and_bytes_left(event: AnnounceEvent, opt_bytes_left: Option<usize>) -> Self {
+        if let AnnounceEvent::Stopped = event {
+            Self::Stopped
+        } else if let Some(0) = opt_bytes_left {
+            Self::Seeding
+        } else {
+            Self::Leeching
+        }
+    }
+}
+
+#[derive(Debug, Clone, Copy)]
+pub struct Peer<I: Ip> {
+    pub ip_address: I,
+    pub port: u16,
+    pub valid_until: ValidUntil,
+    pub seeder: bool,
+}
+
+impl<I: Ip> Peer<I> {
+    pub fn to_response_peer(&self) -> ResponsePeer<I> {
+        ResponsePeer {
+            ip_address: self.ip_address,
+            port: self.port,
+        }
+    }
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+pub struct PeerMapKey<I: Ip> {
+    pub peer_id: PeerId,
+    pub ip: I,
+}
+
+pub type PeerMap<I> = IndexMap<PeerMapKey<I>, Peer<I>>;
+
+pub struct TorrentData<I: Ip> {
+    pub peers: PeerMap<I>,
+    pub num_seeders: usize,
+}
+
+impl<I: Ip> Default for TorrentData<I> {
+    #[inline]
+    fn default() -> Self {
+        Self {
+            peers: Default::default(),
+            num_seeders: 0,
+        }
+    }
+}
+
+impl<I: Ip> TorrentData<I> {
+    fn num_leechers(&self) -> usize {
+        self.peers.len() - self.num_seeders
+    }
+}
+
+pub type TorrentMap<I> = IndexMap<InfoHash, TorrentData<I>>;
+
+#[derive(Default)]
+pub struct TorrentMaps {
+    pub ipv4: TorrentMap<Ipv4Addr>,
+    pub ipv6: TorrentMap<Ipv6Addr>,
+}
+
+impl TorrentMaps {
+    pub fn clean(
+        &mut self,
+        config: &Config,
+        access_list: &Arc<AccessListArcSwap>,
+        server_start_instant: ServerStartInstant,
+    ) {
+        let mut access_list_cache = create_access_list_cache(access_list);
+
+        let now = server_start_instant.seconds_elapsed();
+
+        Self::clean_torrent_map(config, &mut access_list_cache, &mut self.ipv4, now);
+        Self::clean_torrent_map(config, &mut access_list_cache, &mut self.ipv6, now);
+    }
+
+    fn clean_torrent_map<I: Ip>(
+        config: &Config,
+        access_list_cache: &mut AccessListCache,
+        torrent_map: &mut TorrentMap<I>,
+        now: SecondsSinceServerStart,
+    ) {
+        let mut total_num_peers = 0;
+
+        torrent_map.retain(|info_hash, torrent_data| {
+            if !access_list_cache
+                .load()
+                .allows(config.access_list.mode, &info_hash.0)
+            {
+                return false;
+            }
+
+            let num_seeders = &mut torrent_data.num_seeders;
+
+            torrent_data.peers.retain(|_, peer| {
+                let keep = peer.valid_until.valid(now);
+
+                if (!keep) & peer.seeder {
+                    *num_seeders -= 1;
+                }
+
+                keep
+            });
+
+            total_num_peers += torrent_data.peers.len() as u64;
+
+            !torrent_data.peers.is_empty()
+        });
+
+        let total_num_peers = total_num_peers as f64;
+
+        #[cfg(feature = "metrics")]
+        ::metrics::gauge!(
+            "aquatic_peers",
+            total_num_peers,
+            "ip_version" => I::ip_version_str(),
+            "worker_index" => WORKER_INDEX.with(|index| index.get()).to_string(),
+        );
+
+        torrent_map.shrink_to_fit();
+    }
+}
+
+pub async fn run_swarm_worker(
+    _sentinel: PanicSentinel,
+    config: Config,
+    state: State,
+    request_mesh_builder: MeshBuilder<ChannelRequest, Partial>,
+    server_start_instant: ServerStartInstant,
+    worker_index: usize,
+) {
+    #[cfg(feature = "metrics")]
+    WORKER_INDEX.with(|index| index.set(worker_index));
+
+    let (_, mut request_receivers) = request_mesh_builder.join(Role::Consumer).await.unwrap();
+
+    let torrents = Rc::new(RefCell::new(TorrentMaps::default()));
+    let access_list = state.access_list;
+
+    // Periodically clean torrents
+    TimerActionRepeat::repeat(enclose!((config, torrents, access_list) move || {
+        enclose!((config, torrents, access_list) move || async move {
+            torrents.borrow_mut().clean(&config, &access_list, server_start_instant);
+
+            Some(Duration::from_secs(config.cleaning.torrent_cleaning_interval))
+        })()
+    }));
+
+    let max_peer_age = config.cleaning.max_peer_age;
+    let peer_valid_until = Rc::new(RefCell::new(ValidUntil::new(
+        server_start_instant,
+        max_peer_age,
+    )));
+
+    // Periodically update peer_valid_until
+    TimerActionRepeat::repeat(enclose!((peer_valid_until) move || {
+        enclose!((peer_valid_until) move || async move {
+            *peer_valid_until.borrow_mut() = ValidUntil::new(server_start_instant, max_peer_age);
+
+            Some(Duration::from_secs(1))
+        })()
+    }));
+
+    // Periodically update torrent count metrics
+    #[cfg(feature = "metrics")]
+    TimerActionRepeat::repeat(enclose!((config, torrents) move || {
+        enclose!((config, torrents, worker_index) move || async move {
+            let torrents = torrents.borrow_mut();
+
+            ::metrics::gauge!(
+                "aquatic_torrents",
+                torrents.ipv4.len() as f64,
+                "ip_version" => "4",
+                "worker_index" => worker_index.to_string(),
+            );
+            ::metrics::gauge!(
+                "aquatic_torrents",
+                torrents.ipv6.len() as f64,
+                "ip_version" => "6",
+                "worker_index" => worker_index.to_string(),
+            );
+
+            Some(Duration::from_secs(config.metrics.torrent_count_update_interval))
+        })()
+    }));
+
+    let mut handles = Vec::new();
+
+    for (_, receiver) in request_receivers.streams() {
+        let handle = spawn_local(handle_request_stream(
+            config.clone(),
+            torrents.clone(),
+            peer_valid_until.clone(),
+            receiver,
+        ))
+        .detach();
+
+        handles.push(handle);
+    }
+
+    for handle in handles {
+        handle.await;
+    }
+}
+
+async fn handle_request_stream<S>(
+    config: Config,
+    torrents: Rc<RefCell<TorrentMaps>>,
+    peer_valid_until: Rc<RefCell<ValidUntil>>,
+    mut stream: S,
+) where
+    S: Stream<Item = ChannelRequest> + ::std::marker::Unpin,
+{
+    let mut rng = SmallRng::from_entropy();
+
+    while let Some(channel_request) = stream.next().await {
+        match channel_request {
+            ChannelRequest::Announce {
+                request,
+                peer_addr,
+                response_sender,
+            } => {
+                let response = handle_announce_request(
+                    &config,
+                    &mut rng,
+                    &mut torrents.borrow_mut(),
+                    peer_valid_until.borrow().to_owned(),
+                    peer_addr,
+                    request,
+                );
+
+                if let Err(err) = response_sender.connect().await.send(response).await {
+                    ::log::error!("swarm worker could not send announce response: {:#}", err);
+                }
+            }
+            ChannelRequest::Scrape {
+                request,
+                peer_addr,
+                response_sender,
+            } => {
+                let response =
+                    handle_scrape_request(&config, &mut torrents.borrow_mut(), peer_addr, request);
+
+                if let Err(err) = response_sender.connect().await.send(response).await {
+                    ::log::error!("swarm worker could not send scrape response: {:#}", err);
+                }
+            }
+        };
+    }
+}
+
+pub fn handle_announce_request(
+    config: &Config,
+    rng: &mut impl Rng,
+    torrent_maps: &mut TorrentMaps,
+    valid_until: ValidUntil,
+    peer_addr: CanonicalSocketAddr,
+    request: AnnounceRequest,
+) -> AnnounceResponse {
+    match peer_addr.get().ip() {
+        IpAddr::V4(peer_ip_address) => {
+            let torrent_data: &mut TorrentData<Ipv4Addr> =
+                torrent_maps.ipv4.entry(request.info_hash).or_default();
+
+            let (seeders, leechers, response_peers) = upsert_peer_and_get_response_peers(
+                config,
+                rng,
+                peer_ip_address,
+                torrent_data,
+                request,
+                valid_until,
+            );
+
+            let response = AnnounceResponse {
+                complete: seeders,
+                incomplete: leechers,
+                announce_interval: config.protocol.peer_announce_interval,
+                peers: ResponsePeerListV4(response_peers),
+                peers6: ResponsePeerListV6(vec![]),
+                warning_message: None,
+            };
+
+            response
+        }
+        IpAddr::V6(peer_ip_address) => {
+            let torrent_data: &mut TorrentData<Ipv6Addr> =
+                torrent_maps.ipv6.entry(request.info_hash).or_default();
+
+            let (seeders, leechers, response_peers) = upsert_peer_and_get_response_peers(
+                config,
+                rng,
+                peer_ip_address,
+                torrent_data,
+                request,
+                valid_until,
+            );
+
+            let response = AnnounceResponse {
+                complete: seeders,
+                incomplete: leechers,
+                announce_interval: config.protocol.peer_announce_interval,
+                peers: ResponsePeerListV4(vec![]),
+                peers6: ResponsePeerListV6(response_peers),
+                warning_message: None,
+            };
+
+            response
+        }
+    }
+}
+
+/// Insert/update peer. Return num_seeders, num_leechers and response peers
+pub fn upsert_peer_and_get_response_peers<I: Ip>(
+    config: &Config,
+    rng: &mut impl Rng,
+    peer_ip_address: I,
+    torrent_data: &mut TorrentData<I>,
+    request: AnnounceRequest,
+    valid_until: ValidUntil,
+) -> (usize, usize, Vec<ResponsePeer<I>>) {
+    // Insert/update/remove peer who sent this request
+
+    let peer_status =
+        PeerStatus::from_event_and_bytes_left(request.event, Some(request.bytes_left));
+
+    let peer_map_key = PeerMapKey {
+        peer_id: request.peer_id,
+        ip: peer_ip_address,
+    };
+
+    let opt_removed_peer = match peer_status {
+        PeerStatus::Leeching => {
+            let peer = Peer {
+                ip_address: peer_ip_address,
+                port: request.port,
+                valid_until,
+                seeder: false,
+            };
+
+            torrent_data.peers.insert(peer_map_key.clone(), peer)
+        }
+        PeerStatus::Seeding => {
+            torrent_data.num_seeders += 1;
+
+            let peer = Peer {
+                ip_address: peer_ip_address,
+                port: request.port,
+                valid_until,
+                seeder: true,
+            };
+
+            torrent_data.peers.insert(peer_map_key.clone(), peer)
+        }
+        PeerStatus::Stopped => torrent_data.peers.remove(&peer_map_key),
+    };
+
+    if let Some(&Peer { seeder: true, .. }) = opt_removed_peer.as_ref() {
+        torrent_data.num_seeders -= 1;
+    }
+
+    #[cfg(feature = "metrics")]
+    match peer_status {
+        PeerStatus::Stopped if opt_removed_peer.is_some() => {
+            ::metrics::decrement_gauge!(
+                "aquatic_peers",
+                1.0,
+                "ip_version" => I::ip_version_str(),
+                "worker_index" => WORKER_INDEX.with(|index| index.get()).to_string(),
+            );
+        }
+        PeerStatus::Leeching | PeerStatus::Seeding if opt_removed_peer.is_none() => {
+            ::metrics::increment_gauge!(
+                "aquatic_peers",
+                1.0,
+                "ip_version" => I::ip_version_str(),
+                "worker_index" => WORKER_INDEX.with(|index| index.get()).to_string(),
+            );
+        }
+        _ => {}
+    }
+
+    let response_peers = if let PeerStatus::Stopped = peer_status {
+        Vec::new()
+    } else {
+        let max_num_peers_to_take = match request.numwant {
+            Some(0) | None => config.protocol.max_peers,
+            Some(numwant) => numwant.min(config.protocol.max_peers),
+        };
+
+        extract_response_peers(
+            rng,
+            &torrent_data.peers,
+            max_num_peers_to_take,
+            peer_map_key,
+            Peer::to_response_peer,
+        )
+    };
+
+    (
+        torrent_data.num_seeders,
+        torrent_data.num_leechers(),
+        response_peers,
+    )
+}
+
+pub fn handle_scrape_request(
+    config: &Config,
+    torrent_maps: &mut TorrentMaps,
+    peer_addr: CanonicalSocketAddr,
+    request: ScrapeRequest,
+) -> ScrapeResponse {
+    let num_to_take = request
+        .info_hashes
+        .len()
+        .min(config.protocol.max_scrape_torrents);
+
+    let mut response = ScrapeResponse {
+        files: BTreeMap::new(),
+    };
+
+    let peer_ip = peer_addr.get().ip();
+
+    // If request.info_hashes is empty, don't return scrape for all
+    // torrents, even though reference server does it. It is too expensive.
+    if peer_ip.is_ipv4() {
+        for info_hash in request.info_hashes.into_iter().take(num_to_take) {
+            if let Some(torrent_data) = torrent_maps.ipv4.get(&info_hash) {
+                let stats = ScrapeStatistics {
+                    complete: torrent_data.num_seeders,
+                    downloaded: 0, // No implementation planned
+                    incomplete: torrent_data.num_leechers(),
+                };
+
+                response.files.insert(info_hash, stats);
+            }
+        }
+    } else {
+        for info_hash in request.info_hashes.into_iter().take(num_to_take) {
+            if let Some(torrent_data) = torrent_maps.ipv6.get(&info_hash) {
+                let stats = ScrapeStatistics {
+                    complete: torrent_data.num_seeders,
+                    downloaded: 0, // No implementation planned
+                    incomplete: torrent_data.num_leechers(),
+                };
+
+                response.files.insert(info_hash, stats);
+            }
+        }
+    };
+
+    response
+}