aquatic_ws_protocol: refactor, moving code into submodules

aquatic_ws_protocol/src/common.rs

@@ -1,6 +1,81 @@
-use serde::{de::Visitor, Deserializer, Serializer};
+use serde::{de::Visitor, Deserialize, Deserializer, Serialize, Serializer};
 
-use super::{AnnounceAction, ScrapeAction};
+#[derive(Debug, Clone, Copy, Hash, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(transparent)]
+pub struct PeerId(
+    #[serde(
+        deserialize_with = "deserialize_20_bytes",
+        serialize_with = "serialize_20_bytes"
+    )]
+    pub [u8; 20],
+);
+
+#[derive(Debug, Clone, Copy, Hash, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(transparent)]
+pub struct InfoHash(
+    #[serde(
+        deserialize_with = "deserialize_20_bytes",
+        serialize_with = "serialize_20_bytes"
+    )]
+    pub [u8; 20],
+);
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(transparent)]
+pub struct OfferId(
+    #[serde(
+        deserialize_with = "deserialize_20_bytes",
+        serialize_with = "serialize_20_bytes"
+    )]
+    pub [u8; 20],
+);
+
+/// Some kind of nested structure from https://www.npmjs.com/package/simple-peer
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(transparent)]
+pub struct JsonValue(pub ::serde_json::Value);
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct AnnounceAction;
+
+impl Serialize for AnnounceAction {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        serializer.serialize_str("announce")
+    }
+}
+
+impl<'de> Deserialize<'de> for AnnounceAction {
+    fn deserialize<D>(deserializer: D) -> Result<AnnounceAction, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        deserializer.deserialize_str(AnnounceActionVisitor)
+    }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct ScrapeAction;
+
+impl Serialize for ScrapeAction {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        serializer.serialize_str("scrape")
+    }
+}
+
+impl<'de> Deserialize<'de> for ScrapeAction {
+    fn deserialize<D>(deserializer: D) -> Result<ScrapeAction, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        deserializer.deserialize_str(ScrapeActionVisitor)
+    }
+}
 
 pub struct AnnounceActionVisitor;
 
@@ -44,7 +119,7 @@ impl<'de> Visitor<'de> for ScrapeActionVisitor {
     }
 }
 
-pub fn serialize_20_bytes<S>(data: &[u8; 20], serializer: S) -> Result<S::Ok, S::Error>
+fn serialize_20_bytes<S>(data: &[u8; 20], serializer: S) -> Result<S::Ok, S::Error>
 where
     S: Serializer,
 {
@@ -102,7 +177,7 @@ impl<'de> Visitor<'de> for TwentyByteVisitor {
 }
 
 #[inline]
-pub fn deserialize_20_bytes<'de, D>(deserializer: D) -> Result<[u8; 20], D::Error>
+fn deserialize_20_bytes<'de, D>(deserializer: D) -> Result<[u8; 20], D::Error>
 where
     D: Deserializer<'de>,
 {
@@ -113,7 +188,7 @@ where
 mod tests {
     use quickcheck_macros::quickcheck;
 
-    use crate::InfoHash;
+    use crate::common::InfoHash;
 
     fn info_hash_from_bytes(bytes: &[u8]) -> InfoHash {
         let mut arr = [0u8; 20];

aquatic_ws_protocol/src/lib.rs

@@ -1,303 +1,10 @@
-use std::borrow::Cow;
+pub mod common;
+pub mod request;
+pub mod response;
 
-use anyhow::Context;
+pub use common::*;
-use hashbrown::HashMap;
+pub use request::*;
-use serde::{Deserialize, Deserializer, Serialize, Serializer};
+pub use response::*;
-
-mod serde_helpers;
-
-use serde_helpers::*;
-
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-pub struct AnnounceAction;
-
-impl Serialize for AnnounceAction {
-    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
-    where
-        S: Serializer,
-    {
-        serializer.serialize_str("announce")
-    }
-}
-
-impl<'de> Deserialize<'de> for AnnounceAction {
-    fn deserialize<D>(deserializer: D) -> Result<AnnounceAction, D::Error>
-    where
-        D: Deserializer<'de>,
-    {
-        deserializer.deserialize_str(AnnounceActionVisitor)
-    }
-}
-
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-pub struct ScrapeAction;
-
-impl Serialize for ScrapeAction {
-    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
-    where
-        S: Serializer,
-    {
-        serializer.serialize_str("scrape")
-    }
-}
-
-impl<'de> Deserialize<'de> for ScrapeAction {
-    fn deserialize<D>(deserializer: D) -> Result<ScrapeAction, D::Error>
-    where
-        D: Deserializer<'de>,
-    {
-        deserializer.deserialize_str(ScrapeActionVisitor)
-    }
-}
-
-#[derive(Debug, Clone, Copy, Hash, PartialEq, Eq, Serialize, Deserialize)]
-#[serde(transparent)]
-pub struct PeerId(
-    #[serde(
-        deserialize_with = "deserialize_20_bytes",
-        serialize_with = "serialize_20_bytes"
-    )]
-    pub [u8; 20],
-);
-
-#[derive(Debug, Clone, Copy, Hash, PartialEq, Eq, Serialize, Deserialize)]
-#[serde(transparent)]
-pub struct InfoHash(
-    #[serde(
-        deserialize_with = "deserialize_20_bytes",
-        serialize_with = "serialize_20_bytes"
-    )]
-    pub [u8; 20],
-);
-
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
-#[serde(transparent)]
-pub struct OfferId(
-    #[serde(
-        deserialize_with = "deserialize_20_bytes",
-        serialize_with = "serialize_20_bytes"
-    )]
-    pub [u8; 20],
-);
-
-/// Some kind of nested structure from https://www.npmjs.com/package/simple-peer
-#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
-#[serde(transparent)]
-pub struct JsonValue(pub ::serde_json::Value);
-
-#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
-#[serde(rename_all = "lowercase")]
-pub enum AnnounceEvent {
-    Started,
-    Stopped,
-    Completed,
-    Update,
-}
-
-impl Default for AnnounceEvent {
-    fn default() -> Self {
-        Self::Update
-    }
-}
-
-/// Apparently, these are sent to a number of peers when they are set
-/// in an AnnounceRequest
-/// action = "announce"
-#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
-pub struct MiddlemanOfferToPeer {
-    pub action: AnnounceAction,
-    /// Peer id of peer sending offer
-    /// Note: if equal to client peer_id, client ignores offer
-    pub peer_id: PeerId,
-    pub info_hash: InfoHash,
-    /// Gets copied from AnnounceRequestOffer
-    pub offer: JsonValue,
-    /// Gets copied from AnnounceRequestOffer
-    pub offer_id: OfferId,
-}
-
-/// If announce request has answer = true, send this to peer with
-/// peer id == "to_peer_id" field
-/// Action field should be 'announce'
-#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
-pub struct MiddlemanAnswerToPeer {
-    pub action: AnnounceAction,
-    /// Note: if equal to client peer_id, client ignores answer
-    pub peer_id: PeerId,
-    pub info_hash: InfoHash,
-    pub answer: JsonValue,
-    pub offer_id: OfferId,
-}
-
-/// Element of AnnounceRequest.offers
-#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
-pub struct AnnounceRequestOffer {
-    pub offer: JsonValue,
-    pub offer_id: OfferId,
-}
-
-#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
-pub struct AnnounceRequest {
-    pub action: AnnounceAction,
-    pub info_hash: InfoHash,
-    pub peer_id: PeerId,
-    /// Just called "left" in protocol. Is set to None in some cases, such as
-    /// when opening a magnet link
-    #[serde(rename = "left")]
-    pub bytes_left: Option<usize>,
-    /// Can be empty. Then, default is "update"
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub event: Option<AnnounceEvent>,
-
-    /// Only when this is an array offers are sent to random peers
-    /// Length of this is number of peers wanted?
-    /// Max length of this is 10 in reference client code
-    /// Not sent when announce event is stopped or completed
-    pub offers: Option<Vec<AnnounceRequestOffer>>,
-    /// Seems to only get sent by client when sending offers, and is also same
-    /// as length of offers vector (or at least never less)
-    /// Max length of this is 10 in reference client code
-    /// Could probably be ignored, `offers.len()` should provide needed info
-    pub numwant: Option<usize>,
-
-    /// If empty, send response before sending offers (or possibly "skip sending update back"?)
-    /// Else, send MiddlemanAnswerToPeer to peer with "to_peer_id" as peer_id.
-    /// I think using Option is good, it seems like this isn't always set
-    /// (same as `offers`)
-    pub answer: Option<JsonValue>,
-    /// Likely undefined if !(answer == true)
-    pub to_peer_id: Option<PeerId>,
-    /// Sent if answer is set
-    pub offer_id: Option<OfferId>,
-}
-
-#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
-pub struct AnnounceResponse {
-    pub action: AnnounceAction,
-    pub info_hash: InfoHash,
-    /// Client checks if this is null, not clear why
-    pub complete: usize,
-    pub incomplete: usize,
-    #[serde(rename = "interval")]
-    pub announce_interval: usize, // Default 2 min probably
-}
-
-#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
-#[serde(untagged)]
-pub enum ScrapeRequestInfoHashes {
-    Single(InfoHash),
-    Multiple(Vec<InfoHash>),
-}
-
-impl ScrapeRequestInfoHashes {
-    pub fn as_vec(self) -> Vec<InfoHash> {
-        match self {
-            Self::Single(info_hash) => vec![info_hash],
-            Self::Multiple(info_hashes) => info_hashes,
-        }
-    }
-}
-
-#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
-pub struct ScrapeRequest {
-    pub action: ScrapeAction,
-    // If omitted, scrape for all torrents, apparently
-    // There is some kind of parsing here too which accepts a single info hash
-    // and puts it into a vector
-    #[serde(rename = "info_hash")]
-    pub info_hashes: Option<ScrapeRequestInfoHashes>,
-}
-
-#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
-pub struct ScrapeStatistics {
-    pub complete: usize,
-    pub incomplete: usize,
-    pub downloaded: usize,
-}
-
-#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
-pub struct ScrapeResponse {
-    pub action: ScrapeAction,
-    pub files: HashMap<InfoHash, ScrapeStatistics>,
-    // Looks like `flags` field is ignored in reference client
-    // pub flags: HashMap<String, usize>,
-}
-
-#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
-#[serde(rename_all = "lowercase")]
-pub enum ErrorResponseAction {
-    Announce,
-    Scrape,
-}
-
-#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
-pub struct ErrorResponse {
-    #[serde(rename = "failure reason")]
-    pub failure_reason: Cow<'static, str>,
-    /// Action of original request
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub action: Option<ErrorResponseAction>,
-    // Should not be renamed
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub info_hash: Option<InfoHash>,
-}
-
-#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
-#[serde(untagged)]
-pub enum InMessage {
-    AnnounceRequest(AnnounceRequest),
-    ScrapeRequest(ScrapeRequest),
-}
-
-impl InMessage {
-    #[inline]
-    pub fn to_ws_message(&self) -> ::tungstenite::Message {
-        ::tungstenite::Message::from(::serde_json::to_string(&self).unwrap())
-    }
-
-    #[inline]
-    pub fn from_ws_message(ws_message: tungstenite::Message) -> ::anyhow::Result<Self> {
-        use tungstenite::Message::Text;
-
-        let mut text = if let Text(text) = ws_message {
-            text
-        } else {
-            return Err(anyhow::anyhow!("Message is not text"));
-        };
-
-        return ::simd_json::serde::from_str(&mut text).context("deserialize with serde");
-    }
-}
-
-#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
-#[serde(untagged)]
-pub enum OutMessage {
-    Offer(MiddlemanOfferToPeer),
-    Answer(MiddlemanAnswerToPeer),
-    AnnounceResponse(AnnounceResponse),
-    ScrapeResponse(ScrapeResponse),
-    ErrorResponse(ErrorResponse),
-}
-
-impl OutMessage {
-    #[inline]
-    pub fn to_ws_message(&self) -> tungstenite::Message {
-        ::tungstenite::Message::from(::serde_json::to_string(&self).unwrap())
-    }
-
-    #[inline]
-    pub fn from_ws_message(message: ::tungstenite::Message) -> ::anyhow::Result<Self> {
-        use tungstenite::Message::{Binary, Text};
-
-        let mut text = match message {
-            Text(text) => text,
-            Binary(bytes) => String::from_utf8(bytes)?,
-            _ => return Err(anyhow::anyhow!("Message is neither text nor bytes")),
-        };
-
-        Ok(::simd_json::serde::from_str(&mut text)?)
-    }
-}
 
 #[cfg(test)]
 mod tests {

aquatic_ws_protocol/src/request/announce.rs

@@ -0,0 +1,60 @@
+use serde::{Deserialize, Serialize};
+
+use crate::common::*;
+
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "lowercase")]
+pub enum AnnounceEvent {
+    Started,
+    Stopped,
+    Completed,
+    Update,
+}
+
+impl Default for AnnounceEvent {
+    fn default() -> Self {
+        Self::Update
+    }
+}
+
+/// Element of AnnounceRequest.offers
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub struct AnnounceRequestOffer {
+    pub offer: JsonValue,
+    pub offer_id: OfferId,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub struct AnnounceRequest {
+    pub action: AnnounceAction,
+    pub info_hash: InfoHash,
+    pub peer_id: PeerId,
+    /// Just called "left" in protocol. Is set to None in some cases, such as
+    /// when opening a magnet link
+    #[serde(rename = "left")]
+    pub bytes_left: Option<usize>,
+    /// Can be empty. Then, default is "update"
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub event: Option<AnnounceEvent>,
+
+    /// Only when this is an array offers are sent to random peers
+    /// Length of this is number of peers wanted?
+    /// Max length of this is 10 in reference client code
+    /// Not sent when announce event is stopped or completed
+    pub offers: Option<Vec<AnnounceRequestOffer>>,
+    /// Seems to only get sent by client when sending offers, and is also same
+    /// as length of offers vector (or at least never less)
+    /// Max length of this is 10 in reference client code
+    /// Could probably be ignored, `offers.len()` should provide needed info
+    pub numwant: Option<usize>,
+
+    /// If empty, send response before sending offers (or possibly "skip sending update back"?)
+    /// Else, send MiddlemanAnswerToPeer to peer with "to_peer_id" as peer_id.
+    /// I think using Option is good, it seems like this isn't always set
+    /// (same as `offers`)
+    pub answer: Option<JsonValue>,
+    /// Likely undefined if !(answer == true)
+    pub to_peer_id: Option<PeerId>,
+    /// Sent if answer is set
+    pub offer_id: Option<OfferId>,
+}

aquatic_ws_protocol/src/request/mod.rs

@@ -0,0 +1,35 @@
+use anyhow::Context;
+use serde::{Deserialize, Serialize};
+
+pub mod announce;
+pub mod scrape;
+
+pub use announce::*;
+pub use scrape::*;
+
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(untagged)]
+pub enum InMessage {
+    AnnounceRequest(AnnounceRequest),
+    ScrapeRequest(ScrapeRequest),
+}
+
+impl InMessage {
+    #[inline]
+    pub fn to_ws_message(&self) -> ::tungstenite::Message {
+        ::tungstenite::Message::from(::serde_json::to_string(&self).unwrap())
+    }
+
+    #[inline]
+    pub fn from_ws_message(ws_message: tungstenite::Message) -> ::anyhow::Result<Self> {
+        use tungstenite::Message::Text;
+
+        let mut text = if let Text(text) = ws_message {
+            text
+        } else {
+            return Err(anyhow::anyhow!("Message is not text"));
+        };
+
+        return ::simd_json::serde::from_str(&mut text).context("deserialize with serde");
+    }
+}

aquatic_ws_protocol/src/request/scrape.rs

@@ -0,0 +1,29 @@
+use serde::{Deserialize, Serialize};
+
+use crate::common::*;
+
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(untagged)]
+pub enum ScrapeRequestInfoHashes {
+    Single(InfoHash),
+    Multiple(Vec<InfoHash>),
+}
+
+impl ScrapeRequestInfoHashes {
+    pub fn as_vec(self) -> Vec<InfoHash> {
+        match self {
+            Self::Single(info_hash) => vec![info_hash],
+            Self::Multiple(info_hashes) => info_hashes,
+        }
+    }
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub struct ScrapeRequest {
+    pub action: ScrapeAction,
+    // If omitted, scrape for all torrents, apparently
+    // There is some kind of parsing here too which accepts a single info hash
+    // and puts it into a vector
+    #[serde(rename = "info_hash")]
+    pub info_hashes: Option<ScrapeRequestInfoHashes>,
+}

aquatic_ws_protocol/src/response/announce.rs

@@ -0,0 +1,14 @@
+use serde::{Deserialize, Serialize};
+
+use crate::common::*;
+
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub struct AnnounceResponse {
+    pub action: AnnounceAction,
+    pub info_hash: InfoHash,
+    /// Client checks if this is null, not clear why
+    pub complete: usize,
+    pub incomplete: usize,
+    #[serde(rename = "interval")]
+    pub announce_interval: usize, // Default 2 min probably
+}

aquatic_ws_protocol/src/response/answer.rs

@@ -0,0 +1,16 @@
+use serde::{Deserialize, Serialize};
+
+use crate::common::*;
+
+/// If announce request has answer = true, send this to peer with
+/// peer id == "to_peer_id" field
+/// Action field should be 'announce'
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub struct MiddlemanAnswerToPeer {
+    pub action: AnnounceAction,
+    /// Note: if equal to client peer_id, client ignores answer
+    pub peer_id: PeerId,
+    pub info_hash: InfoHash,
+    pub answer: JsonValue,
+    pub offer_id: OfferId,
+}

aquatic_ws_protocol/src/response/error.rs

@@ -0,0 +1,24 @@
+use std::borrow::Cow;
+
+use serde::{Deserialize, Serialize};
+
+use crate::common::*;
+
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "lowercase")]
+pub enum ErrorResponseAction {
+    Announce,
+    Scrape,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub struct ErrorResponse {
+    #[serde(rename = "failure reason")]
+    pub failure_reason: Cow<'static, str>,
+    /// Action of original request
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub action: Option<ErrorResponseAction>,
+    // Should not be renamed
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub info_hash: Option<InfoHash>,
+}

aquatic_ws_protocol/src/response/mod.rs

@@ -0,0 +1,43 @@
+use serde::{Deserialize, Serialize};
+
+pub mod announce;
+pub mod answer;
+pub mod error;
+pub mod offer;
+pub mod scrape;
+
+pub use announce::*;
+pub use answer::*;
+pub use error::*;
+pub use offer::*;
+pub use scrape::*;
+
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(untagged)]
+pub enum OutMessage {
+    Offer(offer::MiddlemanOfferToPeer),
+    Answer(answer::MiddlemanAnswerToPeer),
+    AnnounceResponse(announce::AnnounceResponse),
+    ScrapeResponse(scrape::ScrapeResponse),
+    ErrorResponse(error::ErrorResponse),
+}
+
+impl OutMessage {
+    #[inline]
+    pub fn to_ws_message(&self) -> tungstenite::Message {
+        ::tungstenite::Message::from(::serde_json::to_string(&self).unwrap())
+    }
+
+    #[inline]
+    pub fn from_ws_message(message: ::tungstenite::Message) -> ::anyhow::Result<Self> {
+        use tungstenite::Message::{Binary, Text};
+
+        let mut text = match message {
+            Text(text) => text,
+            Binary(bytes) => String::from_utf8(bytes)?,
+            _ => return Err(anyhow::anyhow!("Message is neither text nor bytes")),
+        };
+
+        Ok(::simd_json::serde::from_str(&mut text)?)
+    }
+}

aquatic_ws_protocol/src/response/offer.rs

@@ -0,0 +1,19 @@
+use serde::{Deserialize, Serialize};
+
+use crate::common::*;
+
+/// Apparently, these are sent to a number of peers when they are set
+/// in an AnnounceRequest
+/// action = "announce"
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub struct MiddlemanOfferToPeer {
+    pub action: AnnounceAction,
+    /// Peer id of peer sending offer
+    /// Note: if equal to client peer_id, client ignores offer
+    pub peer_id: PeerId,
+    pub info_hash: InfoHash,
+    /// Gets copied from AnnounceRequestOffer
+    pub offer: JsonValue,
+    /// Gets copied from AnnounceRequestOffer
+    pub offer_id: OfferId,
+}

aquatic_ws_protocol/src/response/scrape.rs

@@ -0,0 +1,19 @@
+use hashbrown::HashMap;
+use serde::{Deserialize, Serialize};
+
+use crate::common::*;
+
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub struct ScrapeResponse {
+    pub action: ScrapeAction,
+    pub files: HashMap<InfoHash, ScrapeStatistics>,
+    // Looks like `flags` field is ignored in reference client
+    // pub flags: HashMap<String, usize>,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub struct ScrapeStatistics {
+    pub complete: usize,
+    pub incomplete: usize,
+    pub downloaded: usize,
+}