Improve CPU pinning

aquatic_http/Cargo.toml

@@ -15,12 +15,9 @@ name = "aquatic_http"
 [[bin]]
 name = "aquatic_http"
 
-[features]
-cpu-pinning = ["aquatic_common/cpu-pinning"]
-
 [dependencies]
 aquatic_cli_helpers = "0.2.0"
-aquatic_common = "0.2.0"
+aquatic_common = { version = "0.2.0", features = ["with-glommio"] }
 aquatic_http_protocol = "0.2.0"
 aquatic_toml_config = "0.2.0"
 
@@ -31,6 +28,7 @@ futures-lite = "1"
 futures-rustls = "0.22"
 glommio = "0.7"
 itoa = "1"
+libc = "0.2"
 log = "0.4"
 mimalloc = { version = "0.1", default-features = false }
 memchr = "2"

aquatic_http/src/config.rs

@@ -23,7 +23,6 @@ pub struct Config {
     pub cleaning: CleaningConfig,
     pub privileges: PrivilegeConfig,
     pub access_list: AccessListConfig,
-    #[cfg(feature = "cpu-pinning")]
     pub cpu_pinning: aquatic_common::cpu_pinning::CpuPinningConfig,
 }
 
@@ -38,7 +37,6 @@ impl Default for Config {
             cleaning: CleaningConfig::default(),
             privileges: PrivilegeConfig::default(),
             access_list: AccessListConfig::default(),
-            #[cfg(feature = "cpu-pinning")]
             cpu_pinning: Default::default(),
         }
     }

aquatic_http/src/lib.rs

@@ -1,7 +1,10 @@
-#[cfg(feature = "cpu-pinning")]
-use aquatic_common::cpu_pinning::{pin_current_if_configured_to, WorkerIndex};
 use aquatic_common::{
-    access_list::update_access_list, privileges::drop_privileges_after_socket_binding,
+    access_list::update_access_list,
+    cpu_pinning::{
+        glommio::{get_worker_placement, set_affinity_for_util_worker},
+        WorkerIndex,
+    },
+    privileges::drop_privileges_after_socket_binding,
 };
 use common::{State, TlsConfig};
 use glommio::{channels::channel_mesh::MeshBuilder, prelude::*};
@@ -37,12 +40,13 @@ pub fn run(config: Config) -> ::anyhow::Result<()> {
         ::std::thread::spawn(move || run_inner(config, state));
     }
 
-    #[cfg(feature = "cpu-pinning")]
-    pin_current_if_configured_to(
-        &config.cpu_pinning,
-        config.socket_workers,
-        WorkerIndex::Other,
-    );
+    if config.cpu_pinning.active {
+        set_affinity_for_util_worker(
+            &config.cpu_pinning,
+            config.socket_workers,
+            config.request_workers,
+        )?;
+    }
 
     for signal in &mut signals {
         match signal {
@@ -76,16 +80,15 @@ pub fn run_inner(config: Config, state: State) -> anyhow::Result<()> {
         let response_mesh_builder = response_mesh_builder.clone();
         let num_bound_sockets = num_bound_sockets.clone();
 
-        let builder = LocalExecutorBuilder::default().name("socket");
+        let placement = get_worker_placement(
+            &config.cpu_pinning,
+            config.socket_workers,
+            config.request_workers,
+            WorkerIndex::SocketWorker(i),
+        )?;
+        let builder = LocalExecutorBuilder::new(placement).name("socket");
 
         let executor = builder.spawn(move || async move {
-            #[cfg(feature = "cpu-pinning")]
-            pin_current_if_configured_to(
-                &config.cpu_pinning,
-                config.socket_workers,
-                WorkerIndex::SocketWorker(i),
-            );
-
             workers::socket::run_socket_worker(
                 config,
                 state,
@@ -106,16 +109,15 @@ pub fn run_inner(config: Config, state: State) -> anyhow::Result<()> {
         let request_mesh_builder = request_mesh_builder.clone();
         let response_mesh_builder = response_mesh_builder.clone();
 
-        let builder = LocalExecutorBuilder::default().name("request");
+        let placement = get_worker_placement(
+            &config.cpu_pinning,
+            config.socket_workers,
+            config.request_workers,
+            WorkerIndex::RequestWorker(i),
+        )?;
+        let builder = LocalExecutorBuilder::new(placement).name("request");
 
         let executor = builder.spawn(move || async move {
-            #[cfg(feature = "cpu-pinning")]
-            pin_current_if_configured_to(
-                &config.cpu_pinning,
-                config.socket_workers,
-                WorkerIndex::RequestWorker(i),
-            );
-
             workers::request::run_request_worker(
                 config,
                 state,
@@ -135,12 +137,13 @@ pub fn run_inner(config: Config, state: State) -> anyhow::Result<()> {
     )
     .unwrap();
 
-    #[cfg(feature = "cpu-pinning")]
-    pin_current_if_configured_to(
-        &config.cpu_pinning,
-        config.socket_workers,
-        WorkerIndex::Other,
-    );
+    if config.cpu_pinning.active {
+        set_affinity_for_util_worker(
+            &config.cpu_pinning,
+            config.socket_workers,
+            config.request_workers,
+        )?;
+    }
 
     for executor in executors {
         executor