Merge pull request #26 from greatest-ape/feature-gate-cpu-pinning

Feature gate cpu pinning, improve CI

.github/workflows/cargo-build-and-test.yml

@@ -10,12 +10,30 @@ env:
   CARGO_TERM_COLOR: always
 
 jobs:
-  build:
+  build-test-linux:
     runs-on: ubuntu-latest
     timeout-minutes: 10
     steps:
     - uses: actions/checkout@v2
     - name: Build
-      run: cargo build --verbose --workspace --all-targets
+      run: |
+        cargo build --verbose -p aquatic_udp --features "cpu-pinning"
+        cargo build --verbose -p aquatic_udp --features "with-glommio cpu-pinning" --no-default-features
+
+        cargo build --verbose -p aquatic_http --features "cpu-pinning"
+
+        cargo build --verbose -p aquatic_ws --features "cpu-pinning"
+        cargo build --verbose -p aquatic_ws --features "with-glommio cpu-pinning" --no-default-features
     - name: Run tests
       run: cargo test --verbose --workspace --all-targets
+
+
+  build-macos:
+    runs-on: macos-latest
+    timeout-minutes: 10
+    steps:
+    - uses: actions/checkout@v2
+    - name: Build
+      run: |
+        cargo build --verbose -p aquatic_udp
+        cargo build --verbose -p aquatic_ws

TODO.md

@@ -1,7 +1,5 @@
 # TODO
 
-* feature-gate cpu pinning on supported OS-es
-
 * readme
   * document privilege dropping and cpu pinning
   * document access list log output levels and that it exits program
@@ -14,9 +12,6 @@
 * implement socket_recv_size and ipv6_only in glommio implementations
 
 * CI
-  * run build and check on multiple OSes
-    * Linux: all implementations
-    * FreeBSD, macOS (?): mio implementations
   * file transfer CI for all implementations
   * test access lists?
   * cargo-deny
@@ -44,6 +39,9 @@
   * consider better error type for request parsing, so that better error
     messages can be sent back (e.g., "full scrapes are not supported")
 
+* http and ws load tests
+  * add config key 'connection_open_interval_ms', default to 1000
+
 ## Less important
 
 * extract response peers: extract "one extra" to compensate for removal,

aquatic_common/Cargo.toml

@@ -10,11 +10,13 @@ repository = "https://github.com/greatest-ape/aquatic"
 [lib]
 name = "aquatic_common"
 
+[features]
+cpu-pinning = ["affinity"]
+
 [dependencies]
 ahash = "0.7"
 anyhow = "1"
 arc-swap = "1"
-affinity = "0.1"
 hashbrown = "0.11.2"
 hex = "0.4"
 indexmap-amortized = "1"
@@ -22,3 +24,5 @@ log = "0.4"
 privdrop = "0.5"
 rand = { version = "0.8", features = ["small_rng"] }
 serde = { version = "1", features = ["derive"] }
+
+affinity = { version = "0.1", optional = true }

aquatic_common/src/lib.rs

@@ -5,6 +5,7 @@ use ahash::RandomState;
 use rand::Rng;
 
 pub mod access_list;
+#[cfg(feature = "cpu-pinning")]
 pub mod cpu_pinning;
 pub mod privileges;
 

aquatic_http/Cargo.toml

@@ -15,6 +15,9 @@ path = "src/lib/lib.rs"
 name = "aquatic_http"
 path = "src/bin/main.rs"
 
+[features]
+cpu-pinning = ["aquatic_common/cpu-pinning"]
+
 [dependencies]
 anyhow = "1"
 aquatic_cli_helpers = "0.1.0"

aquatic_http/src/lib/config.rs

@@ -1,6 +1,5 @@
 use std::{net::SocketAddr, path::PathBuf};
 
-use aquatic_common::cpu_pinning::CpuPinningConfig;
 use aquatic_common::{access_list::AccessListConfig, privileges::PrivilegeConfig};
 use serde::{Deserialize, Serialize};
 
@@ -22,7 +21,8 @@ pub struct Config {
     pub cleaning: CleaningConfig,
     pub privileges: PrivilegeConfig,
     pub access_list: AccessListConfig,
-    pub cpu_pinning: CpuPinningConfig,
+    #[cfg(feature = "cpu-pinning")]
+    pub cpu_pinning: aquatic_common::cpu_pinning::CpuPinningConfig,
 }
 
 impl aquatic_cli_helpers::Config for Config {
@@ -73,6 +73,7 @@ impl Default for Config {
             cleaning: CleaningConfig::default(),
             privileges: PrivilegeConfig::default(),
             access_list: AccessListConfig::default(),
+            #[cfg(feature = "cpu-pinning")]
             cpu_pinning: Default::default(),
         }
     }

aquatic_http/src/lib/lib.rs

@@ -3,12 +3,12 @@ use std::{
     io::BufReader,
     sync::{atomic::AtomicUsize, Arc},
 };
-
 use aquatic_common::{
     access_list::update_access_list,
-    cpu_pinning::{pin_current_if_configured_to, WorkerIndex},
     privileges::drop_privileges_after_socket_binding,
 };
+#[cfg(feature = "cpu-pinning")]
+use aquatic_common::cpu_pinning::{pin_current_if_configured_to, WorkerIndex};
 use common::{State, TlsConfig};
 use glommio::{channels::channel_mesh::MeshBuilder, prelude::*};
 use signal_hook::{consts::SIGUSR1, iterator::Signals};
@@ -38,6 +38,7 @@ pub fn run(config: Config) -> ::anyhow::Result<()> {
         ::std::thread::spawn(move || run_inner(config, state));
     }
 
+    #[cfg(feature = "cpu-pinning")]
     pin_current_if_configured_to(
         &config.cpu_pinning,
         config.socket_workers,
@@ -77,6 +78,7 @@ pub fn run_inner(config: Config, state: State) -> anyhow::Result<()> {
         let num_bound_sockets = num_bound_sockets.clone();
 
         let executor = LocalExecutorBuilder::default().spawn(move || async move {
+            #[cfg(feature = "cpu-pinning")]
             pin_current_if_configured_to(
                 &config.cpu_pinning,
                 config.socket_workers,
@@ -104,6 +106,7 @@ pub fn run_inner(config: Config, state: State) -> anyhow::Result<()> {
         let response_mesh_builder = response_mesh_builder.clone();
 
         let executor = LocalExecutorBuilder::default().spawn(move || async move {
+            #[cfg(feature = "cpu-pinning")]
             pin_current_if_configured_to(
                 &config.cpu_pinning,
                 config.socket_workers,
@@ -124,6 +127,7 @@ pub fn run_inner(config: Config, state: State) -> anyhow::Result<()> {
     )
     .unwrap();
 
+    #[cfg(feature = "cpu-pinning")]
     pin_current_if_configured_to(
         &config.cpu_pinning,
         config.socket_workers,

aquatic_http_load_test/Cargo.toml

@@ -9,6 +9,9 @@ repository = "https://github.com/greatest-ape/aquatic"
 [[bin]]
 name = "aquatic_http_load_test"
 
+[features]
+cpu-pinning = ["aquatic_common/cpu-pinning"]
+
 [dependencies]
 anyhow = "1"
 aquatic_cli_helpers = "0.1.0"

aquatic_http_load_test/src/config.rs

@@ -1,7 +1,6 @@
 use std::net::SocketAddr;
 
 use aquatic_cli_helpers::LogLevel;
-use aquatic_common::cpu_pinning::CpuPinningConfig;
 use serde::{Deserialize, Serialize};
 
 #[derive(Clone, Debug, Serialize, Deserialize)]
@@ -13,7 +12,8 @@ pub struct Config {
     pub num_connections: usize,
     pub duration: usize,
     pub torrents: TorrentConfig,
-    pub cpu_pinning: CpuPinningConfig,
+    #[cfg(feature = "cpu-pinning")]
+    pub cpu_pinning: aquatic_common::cpu_pinning::CpuPinningConfig,
 }
 
 impl aquatic_cli_helpers::Config for Config {
@@ -49,7 +49,8 @@ impl Default for Config {
             num_connections: 8,
             duration: 0,
             torrents: TorrentConfig::default(),
-            cpu_pinning: CpuPinningConfig::default_for_load_test(),
+            #[cfg(feature = "cpu-pinning")]
+            cpu_pinning: aquatic_common::cpu_pinning::CpuPinningConfig::default_for_load_test(),
         }
     }
 }

aquatic_http_load_test/src/main.rs

@@ -3,6 +3,7 @@ use std::thread;
 use std::time::{Duration, Instant};
 
 use ::glommio::LocalExecutorBuilder;
+#[cfg(feature = "cpu-pinning")]
 use aquatic_common::cpu_pinning::{pin_current_if_configured_to, WorkerIndex};
 use rand::prelude::*;
 use rand_distr::Pareto;
@@ -64,6 +65,7 @@ fn run(config: Config) -> ::anyhow::Result<()> {
 
         LocalExecutorBuilder::default()
             .spawn(move || async move {
+                #[cfg(feature = "cpu-pinning")]
                 pin_current_if_configured_to(
                     &config.cpu_pinning,
                     config.num_workers,
@@ -75,6 +77,7 @@ fn run(config: Config) -> ::anyhow::Result<()> {
             .unwrap();
     }
 
+    #[cfg(feature = "cpu-pinning")]
     pin_current_if_configured_to(
         &config.cpu_pinning,
         config.num_workers as usize,

aquatic_udp/Cargo.toml

@@ -16,6 +16,7 @@ name = "aquatic_udp"
 
 [features]
 default = ["with-mio"]
+cpu-pinning = ["aquatic_common/cpu-pinning"]
 with-glommio = ["glommio", "futures-lite"]
 with-mio = ["crossbeam-channel", "histogram", "mio", "socket2"]
 

aquatic_udp/src/lib/config.rs

@@ -1,6 +1,5 @@
 use std::net::SocketAddr;
 
-use aquatic_common::cpu_pinning::CpuPinningConfig;
 use aquatic_common::{access_list::AccessListConfig, privileges::PrivilegeConfig};
 use serde::{Deserialize, Serialize};
 
@@ -26,7 +25,8 @@ pub struct Config {
     pub cleaning: CleaningConfig,
     pub privileges: PrivilegeConfig,
     pub access_list: AccessListConfig,
-    pub cpu_pinning: CpuPinningConfig,
+    #[cfg(feature = "cpu-pinning")]
+    pub cpu_pinning: aquatic_common::cpu_pinning::CpuPinningConfig,
 }
 
 impl aquatic_cli_helpers::Config for Config {
@@ -116,7 +116,8 @@ impl Default for Config {
             cleaning: CleaningConfig::default(),
             privileges: PrivilegeConfig::default(),
             access_list: AccessListConfig::default(),
-            cpu_pinning: CpuPinningConfig::default(),
+            #[cfg(feature = "cpu-pinning")]
+            cpu_pinning: Default::default(),
         }
     }
 }

aquatic_udp/src/lib/mio/mod.rs

@@ -3,6 +3,7 @@ use std::thread::Builder;
 use std::time::Duration;
 
 use anyhow::Context;
+#[cfg(feature = "cpu-pinning")]
 use aquatic_common::cpu_pinning::{pin_current_if_configured_to, WorkerIndex};
 use aquatic_common::privileges::drop_privileges_after_socket_binding;
 use crossbeam_channel::unbounded;
@@ -34,6 +35,7 @@ pub fn run(config: Config) -> ::anyhow::Result<()> {
         ::std::thread::spawn(move || run_inner(config, state));
     }
 
+    #[cfg(feature = "cpu-pinning")]
     pin_current_if_configured_to(
         &config.cpu_pinning,
         config.socket_workers,
@@ -67,6 +69,7 @@ pub fn run_inner(config: Config, state: State) -> ::anyhow::Result<()> {
         Builder::new()
             .name(format!("request-{:02}", i + 1))
             .spawn(move || {
+                #[cfg(feature = "cpu-pinning")]
                 pin_current_if_configured_to(
                     &config.cpu_pinning,
                     config.socket_workers,
@@ -88,6 +91,7 @@ pub fn run_inner(config: Config, state: State) -> ::anyhow::Result<()> {
         Builder::new()
             .name(format!("socket-{:02}", i + 1))
             .spawn(move || {
+                #[cfg(feature = "cpu-pinning")]
                 pin_current_if_configured_to(
                     &config.cpu_pinning,
                     config.socket_workers,
@@ -113,6 +117,7 @@ pub fn run_inner(config: Config, state: State) -> ::anyhow::Result<()> {
         Builder::new()
             .name("statistics-collector".to_string())
             .spawn(move || {
+                #[cfg(feature = "cpu-pinning")]
                 pin_current_if_configured_to(
                     &config.cpu_pinning,
                     config.socket_workers,
@@ -135,6 +140,7 @@ pub fn run_inner(config: Config, state: State) -> ::anyhow::Result<()> {
     )
     .unwrap();
 
+    #[cfg(feature = "cpu-pinning")]
     pin_current_if_configured_to(
         &config.cpu_pinning,
         config.socket_workers,

aquatic_udp_load_test/Cargo.toml

@@ -9,6 +9,9 @@ repository = "https://github.com/greatest-ape/aquatic"
 [[bin]]
 name = "aquatic_udp_load_test"
 
+[features]
+cpu-pinning = ["aquatic_common/cpu-pinning"]
+
 [dependencies]
 anyhow = "1"
 aquatic_cli_helpers = "0.1.0"

aquatic_udp_load_test/src/common.rs

@@ -2,6 +2,7 @@ use std::net::SocketAddr;
 use std::sync::{atomic::AtomicUsize, Arc};
 
 use aquatic_cli_helpers::LogLevel;
+#[cfg(feature = "cpu-pinning")]
 use aquatic_common::cpu_pinning::CpuPinningConfig;
 use hashbrown::HashMap;
 use parking_lot::Mutex;
@@ -30,6 +31,7 @@ pub struct Config {
     pub duration: usize,
     pub network: NetworkConfig,
     pub handler: HandlerConfig,
+    #[cfg(feature = "cpu-pinning")]
     pub cpu_pinning: CpuPinningConfig,
 }
 
@@ -110,6 +112,7 @@ impl Default for Config {
             duration: 0,
             network: NetworkConfig::default(),
             handler: HandlerConfig::default(),
+            #[cfg(feature = "cpu-pinning")]
             cpu_pinning: CpuPinningConfig::default_for_load_test(),
         }
     }

aquatic_udp_load_test/src/main.rs

@@ -3,6 +3,7 @@ use std::sync::{atomic::Ordering, Arc};
 use std::thread;
 use std::time::{Duration, Instant};
 
+#[cfg(feature = "cpu-pinning")]
 use aquatic_common::cpu_pinning::{pin_current_if_configured_to, WorkerIndex};
 use crossbeam_channel::unbounded;
 use hashbrown::HashMap;
@@ -97,6 +98,7 @@ fn run(config: Config) -> ::anyhow::Result<()> {
         let state = state.clone();
 
         thread::spawn(move || {
+            #[cfg(feature = "cpu-pinning")]
             pin_current_if_configured_to(
                 &config.cpu_pinning,
                 config.num_socket_workers as usize,
@@ -114,6 +116,7 @@ fn run(config: Config) -> ::anyhow::Result<()> {
         let response_receiver = response_receiver.clone();
 
         thread::spawn(move || {
+            #[cfg(feature = "cpu-pinning")]
             pin_current_if_configured_to(
                 &config.cpu_pinning,
                 config.num_socket_workers as usize,
@@ -132,6 +135,7 @@ fn run(config: Config) -> ::anyhow::Result<()> {
             .expect("bootstrap: add initial request to request queue");
     }
 
+    #[cfg(feature = "cpu-pinning")]
     pin_current_if_configured_to(
         &config.cpu_pinning,
         config.num_socket_workers as usize,

aquatic_ws/Cargo.toml

@@ -17,6 +17,7 @@ path = "src/bin/main.rs"
 
 [features]
 default = ["with-mio"]
+cpu-pinning = ["aquatic_common/cpu-pinning"]
 with-glommio = ["async-tungstenite", "futures-lite", "futures", "futures-rustls", "glommio", "rustls-pemfile"]
 with-mio = ["crossbeam-channel", "histogram", "mio", "native-tls", "parking_lot", "socket2"]
 

aquatic_ws/src/lib/config.rs

@@ -2,6 +2,7 @@ use std::net::SocketAddr;
 #[cfg(feature = "with-glommio")]
 use std::path::PathBuf;
 
+#[cfg(feature = "cpu-pinning")]
 use aquatic_common::cpu_pinning::CpuPinningConfig;
 use aquatic_common::{access_list::AccessListConfig, privileges::PrivilegeConfig};
 use serde::{Deserialize, Serialize};
@@ -26,6 +27,7 @@ pub struct Config {
     pub cleaning: CleaningConfig,
     pub privileges: PrivilegeConfig,
     pub access_list: AccessListConfig,
+    #[cfg(feature = "cpu-pinning")]
     pub cpu_pinning: CpuPinningConfig,
     #[cfg(feature = "with-mio")]
     pub statistics: StatisticsConfig,
@@ -118,6 +120,7 @@ impl Default for Config {
             cleaning: CleaningConfig::default(),
             privileges: PrivilegeConfig::default(),
             access_list: AccessListConfig::default(),
+            #[cfg(feature = "cpu-pinning")]
             cpu_pinning: Default::default(),
             #[cfg(feature = "with-mio")]
             statistics: Default::default(),

aquatic_ws/src/lib/glommio/mod.rs

@@ -9,10 +9,9 @@ use std::{
 };
 
 use crate::config::Config;
-use aquatic_common::{
+#[cfg(feature = "cpu-pinning")]
-    cpu_pinning::{pin_current_if_configured_to, WorkerIndex},
+use aquatic_common::cpu_pinning::{pin_current_if_configured_to, WorkerIndex};
-    privileges::drop_privileges_after_socket_binding,
+use aquatic_common::privileges::drop_privileges_after_socket_binding;
-};
 
 use self::common::*;
 
@@ -41,6 +40,7 @@ pub fn run_inner(config: Config, state: State) -> anyhow::Result<()> {
         let num_bound_sockets = num_bound_sockets.clone();
 
         let executor = LocalExecutorBuilder::default().spawn(move || async move {
+            #[cfg(feature = "cpu-pinning")]
             pin_current_if_configured_to(
                 &config.cpu_pinning,
                 config.socket_workers,
@@ -68,6 +68,7 @@ pub fn run_inner(config: Config, state: State) -> anyhow::Result<()> {
         let response_mesh_builder = response_mesh_builder.clone();
 
         let executor = LocalExecutorBuilder::default().spawn(move || async move {
+            #[cfg(feature = "cpu-pinning")]
             pin_current_if_configured_to(
                 &config.cpu_pinning,
                 config.socket_workers,
@@ -88,6 +89,7 @@ pub fn run_inner(config: Config, state: State) -> anyhow::Result<()> {
     )
     .unwrap();
 
+    #[cfg(feature = "cpu-pinning")]
     pin_current_if_configured_to(
         &config.cpu_pinning,
         config.socket_workers,

aquatic_ws/src/lib/lib.rs

@@ -1,7 +1,6 @@
-use aquatic_common::{
+#[cfg(feature = "cpu-pinning")]
-    access_list::update_access_list,
+use aquatic_common::cpu_pinning::{pin_current_if_configured_to, WorkerIndex};
-    cpu_pinning::{pin_current_if_configured_to, WorkerIndex},
+use aquatic_common::access_list::update_access_list;
-};
 use cfg_if::cfg_if;
 use signal_hook::{consts::SIGUSR1, iterator::Signals};
 
@@ -42,6 +41,7 @@ pub fn run(config: Config) -> ::anyhow::Result<()> {
         );
     }
 
+    #[cfg(feature = "cpu-pinning")]
     pin_current_if_configured_to(
         &config.cpu_pinning,
         config.socket_workers,

aquatic_ws/src/lib/mio/mod.rs

@@ -5,6 +5,7 @@ use std::thread::Builder;
 use std::time::Duration;
 
 use anyhow::Context;
+#[cfg(feature = "cpu-pinning")]
 use aquatic_common::cpu_pinning::{pin_current_if_configured_to, WorkerIndex};
 use histogram::Histogram;
 use mio::{Poll, Waker};
@@ -26,6 +27,7 @@ pub fn run(config: Config, state: State) -> anyhow::Result<()> {
 
     // TODO: privdrop here instead
 
+    #[cfg(feature = "cpu-pinning")]
     pin_current_if_configured_to(
         &config.cpu_pinning,
         config.socket_workers,
@@ -76,6 +78,7 @@ pub fn start_workers(config: Config, state: State) -> anyhow::Result<()> {
         Builder::new()
             .name(format!("socket-{:02}", i + 1))
             .spawn(move || {
+                #[cfg(feature = "cpu-pinning")]
                 pin_current_if_configured_to(
                     &config.cpu_pinning,
                     config.socket_workers,
@@ -134,6 +137,7 @@ pub fn start_workers(config: Config, state: State) -> anyhow::Result<()> {
         Builder::new()
             .name(format!("request-{:02}", i + 1))
             .spawn(move || {
+                #[cfg(feature = "cpu-pinning")]
                 pin_current_if_configured_to(
                     &config.cpu_pinning,
                     config.socket_workers,
@@ -157,6 +161,7 @@ pub fn start_workers(config: Config, state: State) -> anyhow::Result<()> {
         Builder::new()
             .name("statistics".to_string())
             .spawn(move || {
+                #[cfg(feature = "cpu-pinning")]
                 pin_current_if_configured_to(
                     &config.cpu_pinning,
                     config.socket_workers,

aquatic_ws_load_test/Cargo.toml

@@ -9,6 +9,9 @@ repository = "https://github.com/greatest-ape/aquatic"
 [[bin]]
 name = "aquatic_ws_load_test"
 
+[features]
+cpu-pinning = ["aquatic_common/cpu-pinning"]
+
 [dependencies]
 anyhow = "1"
 async-tungstenite = "0.15"

aquatic_ws_load_test/src/config.rs

@@ -1,6 +1,7 @@
 use std::net::SocketAddr;
 
 use aquatic_cli_helpers::LogLevel;
+#[cfg(feature = "cpu-pinning")]
 use aquatic_common::cpu_pinning::CpuPinningConfig;
 use serde::{Deserialize, Serialize};
 
@@ -13,6 +14,7 @@ pub struct Config {
     pub num_connections: usize,
     pub duration: usize,
     pub torrents: TorrentConfig,
+    #[cfg(feature = "cpu-pinning")]
     pub cpu_pinning: CpuPinningConfig,
 }
 
@@ -50,6 +52,7 @@ impl Default for Config {
             num_connections: 16,
             duration: 0,
             torrents: TorrentConfig::default(),
+            #[cfg(feature = "cpu-pinning")]
             cpu_pinning: CpuPinningConfig::default_for_load_test(),
         }
     }

aquatic_ws_load_test/src/main.rs

@@ -2,6 +2,7 @@ use std::sync::{atomic::Ordering, Arc};
 use std::thread;
 use std::time::{Duration, Instant};
 
+#[cfg(feature = "cpu-pinning")]
 use aquatic_common::cpu_pinning::{pin_current_if_configured_to, WorkerIndex};
 use glommio::LocalExecutorBuilder;
 use rand::prelude::*;
@@ -59,6 +60,7 @@ fn run(config: Config) -> ::anyhow::Result<()> {
 
         LocalExecutorBuilder::default()
             .spawn(move || async move {
+                #[cfg(feature = "cpu-pinning")]
                 pin_current_if_configured_to(
                     &config.cpu_pinning,
                     config.num_workers,
@@ -70,6 +72,7 @@ fn run(config: Config) -> ::anyhow::Result<()> {
             .unwrap();
     }
 
+    #[cfg(feature = "cpu-pinning")]
     pin_current_if_configured_to(
         &config.cpu_pinning,
         config.num_workers as usize,