udp: add optional resending buffer for responses that failed to send

aquatic_udp/src/workers/socket/mod.rs

@@ -23,6 +23,8 @@ use requests::read_requests;
 use responses::send_responses;
 use storage::PendingScrapeResponseSlab;
 
+use self::responses::send_responses_with_resends;
+
 pub fn run_socket_worker(
     _sentinel: PanicSentinel,
     state: State,
@@ -50,6 +52,7 @@ pub fn run_socket_worker(
     let mut access_list_cache = create_access_list_cache(&state.access_list);
 
     let mut local_responses: Vec<(Response, CanonicalSocketAddr)> = Vec::new();
+    let mut resend_buffer: Vec<(Response, CanonicalSocketAddr)> = Vec::new();
 
     let poll_timeout = Duration::from_millis(config.network.poll_timeout_ms);
 
@@ -60,6 +63,7 @@ pub fn run_socket_worker(
     let mut last_pending_scrape_cleaning = Instant::now();
 
     let mut iter_counter = 0usize;
+    let response_resending_active = config.network.resend_buffer_max_len > 0;
 
     loop {
         poll.poll(&mut events, Some(poll_timeout))
@@ -84,15 +88,28 @@ pub fn run_socket_worker(
             }
         }
 
-        send_responses(
-            &state,
-            &config,
-            &mut socket,
-            &mut buffer,
-            &response_receiver,
-            &mut pending_scrape_responses,
-            local_responses.drain(..),
-        );
+        if response_resending_active {
+            send_responses_with_resends(
+                &state,
+                &config,
+                &mut socket,
+                &mut buffer,
+                &response_receiver,
+                &mut pending_scrape_responses,
+                local_responses.drain(..),
+                &mut resend_buffer,
+            );
+        } else {
+            send_responses(
+                &state,
+                &config,
+                &mut socket,
+                &mut buffer,
+                &response_receiver,
+                &mut pending_scrape_responses,
+                local_responses.drain(..),
+            );
+        }
 
         // Run periodic ValidUntil updates and state cleaning
         if iter_counter % 256 == 0 {

aquatic_udp/src/workers/socket/responses.rs

@@ -1,8 +1,9 @@
-use std::io::Cursor;
+use std::io::{Cursor, ErrorKind};
 use std::sync::atomic::Ordering;
 use std::vec::Drain;
 
 use crossbeam_channel::Receiver;
+use libc::ENOBUFS;
 use mio::net::UdpSocket;
 
 use aquatic_common::CanonicalSocketAddr;
@@ -23,7 +24,7 @@ pub fn send_responses(
     local_responses: Drain<(Response, CanonicalSocketAddr)>,
 ) {
     for (response, addr) in local_responses {
-        send_response(state, config, socket, buffer, response, addr);
+        let _ = send_response(state, config, socket, buffer, &response, addr);
     }
 
     for (response, addr) in response_receiver.try_iter() {
@@ -36,19 +37,76 @@ pub fn send_responses(
         };
 
         if let Some(response) = opt_response {
-            send_response(state, config, socket, buffer, response, addr);
+            let _ = send_response(state, config, socket, buffer, &response, addr);
         }
     }
 }
 
+pub fn send_responses_with_resends(
+    state: &State,
+    config: &Config,
+    socket: &mut UdpSocket,
+    buffer: &mut [u8],
+    response_receiver: &Receiver<(ConnectedResponse, CanonicalSocketAddr)>,
+    pending_scrape_responses: &mut PendingScrapeResponseSlab,
+    local_responses: Drain<(Response, CanonicalSocketAddr)>,
+    resend_buffer: &mut Vec<(Response, CanonicalSocketAddr)>,
+) {
+    let resend_buffer_max_len = config.network.resend_buffer_max_len;
+
+    for (response, addr) in resend_buffer.drain(..) {
+        let _ = send_response(state, config, socket, buffer, &response, addr);
+    }
+
+    for (response, addr) in local_responses {
+        match send_response(state, config, socket, buffer, &response, addr) {
+            Err(err) if error_should_cause_resend(&err) => {
+                if resend_buffer.len() < resend_buffer_max_len {
+                    resend_buffer.push((response, addr));
+                } else {
+                    ::log::warn!("response resend buffer full, dropping response");
+                }
+            }
+            _ => (),
+        }
+    }
+
+    for (response, addr) in response_receiver.try_iter() {
+        let opt_response = match response {
+            ConnectedResponse::Scrape(r) => pending_scrape_responses
+                .add_and_get_finished(r)
+                .map(Response::Scrape),
+            ConnectedResponse::AnnounceIpv4(r) => Some(Response::AnnounceIpv4(r)),
+            ConnectedResponse::AnnounceIpv6(r) => Some(Response::AnnounceIpv6(r)),
+        };
+
+        if let Some(response) = opt_response {
+            match send_response(state, config, socket, buffer, &response, addr) {
+                Err(err) if error_should_cause_resend(&err) => {
+                    if resend_buffer.len() < resend_buffer_max_len {
+                        resend_buffer.push((response, addr));
+                    } else {
+                        ::log::warn!("response resend buffer full, dropping response");
+                    }
+                }
+                _ => (),
+            }
+        }
+    }
+}
+
+fn error_should_cause_resend(err: &::std::io::Error) -> bool {
+    (err.raw_os_error() == Some(ENOBUFS)) | (err.kind() == ErrorKind::WouldBlock)
+}
+
 fn send_response(
     state: &State,
     config: &Config,
     socket: &mut UdpSocket,
     buffer: &mut [u8],
-    response: Response,
+    response: &Response,
     addr: CanonicalSocketAddr,
-) {
+) -> std::io::Result<()> {
     let mut cursor = Cursor::new(buffer);
 
     let canonical_addr_is_ipv4 = addr.is_ipv4();
@@ -90,15 +148,21 @@ fn send_response(
                             stats.responses_sent_error.fetch_add(1, Ordering::Relaxed);
                         }
                     }
+
+                    Ok(())
                 }
-                Ok(_) => {}
+                Ok(_) => Ok(()),
                 Err(err) => {
                     ::log::warn!("send_to error: {:#}", err);
+
+                    Err(err)
                 }
             }
         }
         Err(err) => {
             ::log::error!("Response::write error: {:?}", err);
+
+            Err(err)
         }
     }
 }