Move all crates to new crates dir

crates/http_protocol/src/common.rs

@@ -0,0 +1,102 @@
+use std::str::FromStr;
+
+use serde::{Deserialize, Serialize};
+
+use super::utils::*;
+
+#[derive(Debug, Clone, Copy, Hash, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(transparent)]
+pub struct PeerId(
+    #[serde(
+        serialize_with = "serialize_20_bytes",
+        deserialize_with = "deserialize_20_bytes"
+    )]
+    pub [u8; 20],
+);
+
+#[derive(Debug, Clone, Copy, Hash, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
+#[serde(transparent)]
+pub struct InfoHash(
+    #[serde(
+        serialize_with = "serialize_20_bytes",
+        deserialize_with = "deserialize_20_bytes"
+    )]
+    pub [u8; 20],
+);
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum AnnounceEvent {
+    Started,
+    Stopped,
+    Completed,
+    Empty,
+}
+
+impl Default for AnnounceEvent {
+    fn default() -> Self {
+        Self::Empty
+    }
+}
+
+impl FromStr for AnnounceEvent {
+    type Err = String;
+
+    fn from_str(value: &str) -> std::result::Result<Self, String> {
+        match value {
+            "started" => Ok(Self::Started),
+            "stopped" => Ok(Self::Stopped),
+            "completed" => Ok(Self::Completed),
+            "empty" => Ok(Self::Empty),
+            value => Err(format!("Unknown value: {}", value)),
+        }
+    }
+}
+
+impl AnnounceEvent {
+    pub fn as_str(&self) -> Option<&str> {
+        match self {
+            Self::Started => Some("started"),
+            Self::Stopped => Some("stopped"),
+            Self::Completed => Some("completed"),
+            Self::Empty => None,
+        }
+    }
+}
+
+#[cfg(test)]
+impl quickcheck::Arbitrary for InfoHash {
+    fn arbitrary(g: &mut quickcheck::Gen) -> Self {
+        let mut arr = [b'0'; 20];
+
+        for byte in arr.iter_mut() {
+            *byte = u8::arbitrary(g);
+        }
+
+        Self(arr)
+    }
+}
+
+#[cfg(test)]
+impl quickcheck::Arbitrary for PeerId {
+    fn arbitrary(g: &mut quickcheck::Gen) -> Self {
+        let mut arr = [b'0'; 20];
+
+        for byte in arr.iter_mut() {
+            *byte = u8::arbitrary(g);
+        }
+
+        Self(arr)
+    }
+}
+
+#[cfg(test)]
+impl quickcheck::Arbitrary for AnnounceEvent {
+    fn arbitrary(g: &mut quickcheck::Gen) -> Self {
+        match (bool::arbitrary(g), bool::arbitrary(g)) {
+            (false, false) => Self::Started,
+            (true, false) => Self::Started,
+            (false, true) => Self::Completed,
+            (true, true) => Self::Empty,
+        }
+    }
+}

crates/http_protocol/src/lib.rs

@@ -0,0 +1,4 @@
+pub mod common;
+pub mod request;
+pub mod response;
+mod utils;

crates/http_protocol/src/request.rs

@@ -0,0 +1,467 @@
+use std::io::Write;
+
+use anyhow::Context;
+use compact_str::CompactString;
+
+use super::common::*;
+use super::utils::*;
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct AnnounceRequest {
+    pub info_hash: InfoHash,
+    pub peer_id: PeerId,
+    pub port: u16,
+    pub bytes_uploaded: usize,
+    pub bytes_downloaded: usize,
+    pub bytes_left: usize,
+    pub event: AnnounceEvent,
+    /// Number of response peers wanted
+    pub numwant: Option<usize>,
+    pub key: Option<CompactString>,
+}
+
+impl AnnounceRequest {
+    fn write<W: Write>(&self, output: &mut W, url_suffix: &[u8]) -> ::std::io::Result<()> {
+        output.write_all(b"GET /announce")?;
+        output.write_all(url_suffix)?;
+        output.write_all(b"?info_hash=")?;
+        urlencode_20_bytes(self.info_hash.0, output)?;
+
+        output.write_all(b"&peer_id=")?;
+        urlencode_20_bytes(self.peer_id.0, output)?;
+
+        output.write_all(b"&port=")?;
+        output.write_all(itoa::Buffer::new().format(self.port).as_bytes())?;
+
+        output.write_all(b"&uploaded=")?;
+        output.write_all(itoa::Buffer::new().format(self.bytes_uploaded).as_bytes())?;
+
+        output.write_all(b"&downloaded=")?;
+        output.write_all(itoa::Buffer::new().format(self.bytes_downloaded).as_bytes())?;
+
+        output.write_all(b"&left=")?;
+        output.write_all(itoa::Buffer::new().format(self.bytes_left).as_bytes())?;
+
+        match self.event {
+            AnnounceEvent::Started => output.write_all(b"&event=started")?,
+            AnnounceEvent::Stopped => output.write_all(b"&event=stopped")?,
+            AnnounceEvent::Completed => output.write_all(b"&event=completed")?,
+            AnnounceEvent::Empty => (),
+        };
+
+        if let Some(numwant) = self.numwant {
+            output.write_all(b"&numwant=")?;
+            output.write_all(itoa::Buffer::new().format(numwant).as_bytes())?;
+        }
+
+        if let Some(ref key) = self.key {
+            output.write_all(b"&key=")?;
+            output.write_all(::urlencoding::encode(key.as_str()).as_bytes())?;
+        }
+
+        // Always ask for compact responses to ease load testing of non-aquatic trackers
+        output.write_all(b"&compact=1")?;
+
+        output.write_all(b" HTTP/1.1\r\nHost: localhost\r\n\r\n")?;
+
+        Ok(())
+    }
+
+    pub fn from_query_string(query_string: &str) -> anyhow::Result<Self> {
+        // -- Parse key-value pairs
+
+        let mut opt_info_hash = None;
+        let mut opt_peer_id = None;
+        let mut opt_port = None;
+        let mut opt_bytes_left = None;
+        let mut opt_bytes_uploaded = None;
+        let mut opt_bytes_downloaded = None;
+        let mut event = AnnounceEvent::default();
+        let mut opt_numwant = None;
+        let mut opt_key = None;
+
+        let query_string_bytes = query_string.as_bytes();
+
+        let mut ampersand_iter = ::memchr::memchr_iter(b'&', query_string_bytes);
+        let mut position = 0usize;
+
+        for equal_sign_index in ::memchr::memchr_iter(b'=', query_string_bytes) {
+            let segment_end = ampersand_iter.next().unwrap_or_else(|| query_string.len());
+
+            let key = query_string
+                .get(position..equal_sign_index)
+                .with_context(|| format!("no key at {}..{}", position, equal_sign_index))?;
+            let value = query_string
+                .get(equal_sign_index + 1..segment_end)
+                .with_context(|| {
+                    format!("no value at {}..{}", equal_sign_index + 1, segment_end)
+                })?;
+
+            match key {
+                "info_hash" => {
+                    let value = urldecode_20_bytes(value)?;
+
+                    opt_info_hash = Some(InfoHash(value));
+                }
+                "peer_id" => {
+                    let value = urldecode_20_bytes(value)?;
+
+                    opt_peer_id = Some(PeerId(value));
+                }
+                "port" => {
+                    opt_port = Some(value.parse::<u16>().with_context(|| "parse port")?);
+                }
+                "left" => {
+                    opt_bytes_left = Some(value.parse::<usize>().with_context(|| "parse left")?);
+                }
+                "uploaded" => {
+                    opt_bytes_uploaded =
+                        Some(value.parse::<usize>().with_context(|| "parse uploaded")?);
+                }
+                "downloaded" => {
+                    opt_bytes_downloaded =
+                        Some(value.parse::<usize>().with_context(|| "parse downloaded")?);
+                }
+                "event" => {
+                    event = value
+                        .parse::<AnnounceEvent>()
+                        .map_err(|err| anyhow::anyhow!("invalid event: {}", err))?;
+                }
+                "compact" => {
+                    if value != "1" {
+                        return Err(anyhow::anyhow!("compact set, but not to 1"));
+                    }
+                }
+                "numwant" => {
+                    opt_numwant = Some(value.parse::<usize>().with_context(|| "parse numwant")?);
+                }
+                "key" => {
+                    if value.len() > 100 {
+                        return Err(anyhow::anyhow!("'key' is too long"));
+                    }
+                    opt_key = Some(::urlencoding::decode(value)?.into());
+                }
+                k => {
+                    ::log::debug!("ignored unrecognized key: {}", k)
+                }
+            }
+
+            if segment_end == query_string.len() {
+                break;
+            } else {
+                position = segment_end + 1;
+            }
+        }
+
+        Ok(AnnounceRequest {
+            info_hash: opt_info_hash.with_context(|| "no info_hash")?,
+            peer_id: opt_peer_id.with_context(|| "no peer_id")?,
+            port: opt_port.with_context(|| "no port")?,
+            bytes_uploaded: opt_bytes_uploaded.with_context(|| "no uploaded")?,
+            bytes_downloaded: opt_bytes_downloaded.with_context(|| "no downloaded")?,
+            bytes_left: opt_bytes_left.with_context(|| "no left")?,
+            event,
+            numwant: opt_numwant,
+            key: opt_key,
+        })
+    }
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct ScrapeRequest {
+    pub info_hashes: Vec<InfoHash>,
+}
+
+impl ScrapeRequest {
+    fn write<W: Write>(&self, output: &mut W, url_suffix: &[u8]) -> ::std::io::Result<()> {
+        output.write_all(b"GET /scrape")?;
+        output.write_all(url_suffix)?;
+        output.write_all(b"?")?;
+
+        let mut first = true;
+
+        for info_hash in self.info_hashes.iter() {
+            if !first {
+                output.write_all(b"&")?;
+            }
+
+            output.write_all(b"info_hash=")?;
+            urlencode_20_bytes(info_hash.0, output)?;
+
+            first = false;
+        }
+
+        output.write_all(b" HTTP/1.1\r\nHost: localhost\r\n\r\n")?;
+
+        Ok(())
+    }
+
+    pub fn from_query_string(query_string: &str) -> anyhow::Result<Self> {
+        // -- Parse key-value pairs
+
+        let mut info_hashes = Vec::new();
+
+        let query_string_bytes = query_string.as_bytes();
+
+        let mut ampersand_iter = ::memchr::memchr_iter(b'&', query_string_bytes);
+        let mut position = 0usize;
+
+        for equal_sign_index in ::memchr::memchr_iter(b'=', query_string_bytes) {
+            let segment_end = ampersand_iter.next().unwrap_or_else(|| query_string.len());
+
+            let key = query_string
+                .get(position..equal_sign_index)
+                .with_context(|| format!("no key at {}..{}", position, equal_sign_index))?;
+            let value = query_string
+                .get(equal_sign_index + 1..segment_end)
+                .with_context(|| {
+                    format!("no value at {}..{}", equal_sign_index + 1, segment_end)
+                })?;
+
+            match key {
+                "info_hash" => {
+                    let value = urldecode_20_bytes(value)?;
+
+                    info_hashes.push(InfoHash(value));
+                }
+                k => {
+                    ::log::debug!("ignored unrecognized key: {}", k)
+                }
+            }
+
+            if segment_end == query_string.len() {
+                break;
+            } else {
+                position = segment_end + 1;
+            }
+        }
+
+        if info_hashes.is_empty() {
+            return Err(anyhow::anyhow!("No info hashes sent"));
+        }
+
+        Ok(ScrapeRequest { info_hashes })
+    }
+}
+
+#[derive(Debug)]
+pub enum RequestParseError {
+    NeedMoreData,
+    Invalid(anyhow::Error),
+}
+
+impl ::std::fmt::Display for RequestParseError {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            Self::NeedMoreData => write!(f, "Incomplete request, more data needed"),
+            Self::Invalid(err) => write!(f, "Invalid request: {:#}", err),
+        }
+    }
+}
+
+impl ::std::error::Error for RequestParseError {}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum Request {
+    Announce(AnnounceRequest),
+    Scrape(ScrapeRequest),
+}
+
+impl Request {
+    /// Parse Request from HTTP request bytes
+    pub fn from_bytes(bytes: &[u8]) -> Result<Self, RequestParseError> {
+        let mut headers = [httparse::EMPTY_HEADER; 16];
+        let mut http_request = httparse::Request::new(&mut headers);
+
+        match http_request.parse(bytes) {
+            Ok(httparse::Status::Complete(_)) => {
+                if let Some(path) = http_request.path {
+                    Self::from_http_get_path(path).map_err(RequestParseError::Invalid)
+                } else {
+                    Err(RequestParseError::Invalid(anyhow::anyhow!("no http path")))
+                }
+            }
+            Ok(httparse::Status::Partial) => Err(RequestParseError::NeedMoreData),
+            Err(err) => Err(RequestParseError::Invalid(anyhow::Error::from(err))),
+        }
+    }
+
+    /// Parse Request from http GET path (`/announce?info_hash=...`)
+    ///
+    /// Existing serde-url decode crates were insufficient, so the decision was
+    /// made to create a custom parser. serde_urlencoded doesn't support multiple
+    /// values with same key, and serde_qs pulls in lots of dependencies. Both
+    /// would need preprocessing for the binary format used for info_hash and
+    /// peer_id.
+    ///
+    /// The info hashes and peer id's that are received are url-encoded byte
+    /// by byte, e.g., %fa for byte 0xfa. However, they need to be parsed as
+    /// UTF-8 string, meaning that non-ascii bytes are invalid characters.
+    /// Therefore, these bytes must be converted to their equivalent multi-byte
+    /// UTF-8 encodings.
+    pub fn from_http_get_path(path: &str) -> anyhow::Result<Self> {
+        ::log::debug!("request GET path: {}", path);
+
+        let mut split_parts = path.splitn(2, '?');
+
+        let location = split_parts.next().with_context(|| "no location")?;
+        let query_string = split_parts.next().with_context(|| "no query string")?;
+
+        if location == "/announce" {
+            Ok(Request::Announce(AnnounceRequest::from_query_string(
+                query_string,
+            )?))
+        } else if location == "/scrape" {
+            Ok(Request::Scrape(ScrapeRequest::from_query_string(
+                query_string,
+            )?))
+        } else {
+            Err(anyhow::anyhow!("Path must be /announce or /scrape"))
+        }
+    }
+
+    pub fn write<W: Write>(&self, output: &mut W, url_suffix: &[u8]) -> ::std::io::Result<()> {
+        match self {
+            Self::Announce(r) => r.write(output, url_suffix),
+            Self::Scrape(r) => r.write(output, url_suffix),
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use quickcheck::{quickcheck, Arbitrary, Gen, TestResult};
+
+    use super::*;
+
+    static ANNOUNCE_REQUEST_PATH: &str = "/announce?info_hash=%04%0bkV%3f%5cr%14%a6%b7%98%adC%c3%c9.%40%24%00%b9&peer_id=-ABC940-5ert69muw5t8&port=12345&uploaded=1&downloaded=2&left=3&numwant=0&key=4ab4b877&compact=1&supportcrypto=1&event=started";
+    static SCRAPE_REQUEST_PATH: &str =
+        "/scrape?info_hash=%04%0bkV%3f%5cr%14%a6%b7%98%adC%c3%c9.%40%24%00%b9";
+    static REFERENCE_INFO_HASH: [u8; 20] = [
+        0x04, 0x0b, b'k', b'V', 0x3f, 0x5c, b'r', 0x14, 0xa6, 0xb7, 0x98, 0xad, b'C', 0xc3, 0xc9,
+        b'.', 0x40, 0x24, 0x00, 0xb9,
+    ];
+    static REFERENCE_PEER_ID: [u8; 20] = [
+        b'-', b'A', b'B', b'C', b'9', b'4', b'0', b'-', b'5', b'e', b'r', b't', b'6', b'9', b'm',
+        b'u', b'w', b'5', b't', b'8',
+    ];
+
+    fn get_reference_announce_request() -> Request {
+        Request::Announce(AnnounceRequest {
+            info_hash: InfoHash(REFERENCE_INFO_HASH),
+            peer_id: PeerId(REFERENCE_PEER_ID),
+            port: 12345,
+            bytes_uploaded: 1,
+            bytes_downloaded: 2,
+            bytes_left: 3,
+            event: AnnounceEvent::Started,
+            numwant: Some(0),
+            key: Some("4ab4b877".into()),
+        })
+    }
+
+    #[test]
+    fn test_announce_request_from_bytes() {
+        let mut bytes = Vec::new();
+
+        bytes.extend_from_slice(b"GET ");
+        bytes.extend_from_slice(&ANNOUNCE_REQUEST_PATH.as_bytes());
+        bytes.extend_from_slice(b" HTTP/1.1\r\n\r\n");
+
+        let parsed_request = Request::from_bytes(&bytes[..]).unwrap();
+        let reference_request = get_reference_announce_request();
+
+        assert_eq!(parsed_request, reference_request);
+    }
+
+    #[test]
+    fn test_scrape_request_from_bytes() {
+        let mut bytes = Vec::new();
+
+        bytes.extend_from_slice(b"GET ");
+        bytes.extend_from_slice(&SCRAPE_REQUEST_PATH.as_bytes());
+        bytes.extend_from_slice(b" HTTP/1.1\r\n\r\n");
+
+        let parsed_request = Request::from_bytes(&bytes[..]).unwrap();
+        let reference_request = Request::Scrape(ScrapeRequest {
+            info_hashes: vec![InfoHash(REFERENCE_INFO_HASH)],
+        });
+
+        assert_eq!(parsed_request, reference_request);
+    }
+
+    impl Arbitrary for AnnounceRequest {
+        fn arbitrary(g: &mut Gen) -> Self {
+            let key: Option<String> = Arbitrary::arbitrary(g);
+
+            AnnounceRequest {
+                info_hash: Arbitrary::arbitrary(g),
+                peer_id: Arbitrary::arbitrary(g),
+                port: Arbitrary::arbitrary(g),
+                bytes_uploaded: Arbitrary::arbitrary(g),
+                bytes_downloaded: Arbitrary::arbitrary(g),
+                bytes_left: Arbitrary::arbitrary(g),
+                event: Arbitrary::arbitrary(g),
+                numwant: Arbitrary::arbitrary(g),
+                key: key.map(|key| key.into()),
+            }
+        }
+    }
+
+    impl Arbitrary for ScrapeRequest {
+        fn arbitrary(g: &mut Gen) -> Self {
+            ScrapeRequest {
+                info_hashes: Arbitrary::arbitrary(g),
+            }
+        }
+    }
+
+    impl Arbitrary for Request {
+        fn arbitrary(g: &mut Gen) -> Self {
+            if Arbitrary::arbitrary(g) {
+                Self::Announce(Arbitrary::arbitrary(g))
+            } else {
+                Self::Scrape(Arbitrary::arbitrary(g))
+            }
+        }
+    }
+
+    #[test]
+    fn quickcheck_serde_identity_request() {
+        fn prop(request: Request) -> TestResult {
+            match request {
+                Request::Announce(AnnounceRequest {
+                    key: Some(ref key), ..
+                }) => {
+                    if key.len() > 30 {
+                        return TestResult::discard();
+                    }
+                }
+                Request::Scrape(ScrapeRequest { ref info_hashes }) => {
+                    if info_hashes.is_empty() {
+                        return TestResult::discard();
+                    }
+                }
+                _ => {}
+            }
+
+            let mut bytes = Vec::new();
+
+            request.write(&mut bytes, &[]).unwrap();
+
+            let parsed_request = Request::from_bytes(&bytes[..]).unwrap();
+
+            let success = request == parsed_request;
+
+            if !success {
+                println!("request:        {:?}", request);
+                println!("parsed request: {:?}", parsed_request);
+                println!("bytes as str:   {}", String::from_utf8_lossy(&bytes));
+            }
+
+            TestResult::from_bool(success)
+        }
+
+        quickcheck(prop as fn(Request) -> TestResult);
+    }
+}

crates/http_protocol/src/response.rs

@@ -0,0 +1,335 @@
+use std::borrow::Cow;
+use std::io::Write;
+use std::net::{Ipv4Addr, Ipv6Addr};
+
+use serde::{Deserialize, Serialize};
+use std::collections::BTreeMap;
+
+use super::common::*;
+use super::utils::*;
+
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub struct ResponsePeer<I: Eq> {
+    pub ip_address: I,
+    pub port: u16,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize, Default)]
+#[serde(transparent)]
+pub struct ResponsePeerListV4(
+    #[serde(
+        serialize_with = "serialize_response_peers_ipv4",
+        deserialize_with = "deserialize_response_peers_ipv4"
+    )]
+    pub Vec<ResponsePeer<Ipv4Addr>>,
+);
+
+#[derive(Debug, Clone, Serialize, Deserialize, Default)]
+#[serde(transparent)]
+pub struct ResponsePeerListV6(
+    #[serde(
+        serialize_with = "serialize_response_peers_ipv6",
+        deserialize_with = "deserialize_response_peers_ipv6"
+    )]
+    pub Vec<ResponsePeer<Ipv6Addr>>,
+);
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct ScrapeStatistics {
+    pub complete: usize,
+    pub incomplete: usize,
+    pub downloaded: usize,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct AnnounceResponse {
+    #[serde(rename = "interval")]
+    pub announce_interval: usize,
+    pub complete: usize,
+    pub incomplete: usize,
+    #[serde(default)]
+    pub peers: ResponsePeerListV4,
+    #[serde(default)]
+    pub peers6: ResponsePeerListV6,
+    // Serialize as string if Some, otherwise skip
+    #[serde(
+        rename = "warning message",
+        skip_serializing_if = "Option::is_none",
+        serialize_with = "serialize_optional_string"
+    )]
+    pub warning_message: Option<String>,
+}
+
+impl AnnounceResponse {
+    pub fn write<W: Write>(&self, output: &mut W) -> ::std::io::Result<usize> {
+        let mut bytes_written = 0usize;
+
+        bytes_written += output.write(b"d8:completei")?;
+        bytes_written += output.write(itoa::Buffer::new().format(self.complete).as_bytes())?;
+
+        bytes_written += output.write(b"e10:incompletei")?;
+        bytes_written += output.write(itoa::Buffer::new().format(self.incomplete).as_bytes())?;
+
+        bytes_written += output.write(b"e8:intervali")?;
+        bytes_written += output.write(
+            itoa::Buffer::new()
+                .format(self.announce_interval)
+                .as_bytes(),
+        )?;
+
+        bytes_written += output.write(b"e5:peers")?;
+        bytes_written += output.write(
+            itoa::Buffer::new()
+                .format(self.peers.0.len() * 6)
+                .as_bytes(),
+        )?;
+        bytes_written += output.write(b":")?;
+        for peer in self.peers.0.iter() {
+            bytes_written += output.write(&u32::from(peer.ip_address).to_be_bytes())?;
+            bytes_written += output.write(&peer.port.to_be_bytes())?;
+        }
+
+        bytes_written += output.write(b"6:peers6")?;
+        bytes_written += output.write(
+            itoa::Buffer::new()
+                .format(self.peers6.0.len() * 18)
+                .as_bytes(),
+        )?;
+        bytes_written += output.write(b":")?;
+        for peer in self.peers6.0.iter() {
+            bytes_written += output.write(&u128::from(peer.ip_address).to_be_bytes())?;
+            bytes_written += output.write(&peer.port.to_be_bytes())?;
+        }
+
+        if let Some(ref warning_message) = self.warning_message {
+            let message_bytes = warning_message.as_bytes();
+
+            bytes_written += output.write(b"15:warning message")?;
+            bytes_written +=
+                output.write(itoa::Buffer::new().format(message_bytes.len()).as_bytes())?;
+            bytes_written += output.write(b":")?;
+            bytes_written += output.write(message_bytes)?;
+        }
+
+        bytes_written += output.write(b"e")?;
+
+        Ok(bytes_written)
+    }
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct ScrapeResponse {
+    /// BTreeMap instead of HashMap since keys need to be serialized in order
+    pub files: BTreeMap<InfoHash, ScrapeStatistics>,
+}
+
+impl ScrapeResponse {
+    pub fn write<W: Write>(&self, output: &mut W) -> ::std::io::Result<usize> {
+        let mut bytes_written = 0usize;
+
+        bytes_written += output.write(b"d5:filesd")?;
+
+        for (info_hash, statistics) in self.files.iter() {
+            bytes_written += output.write(b"20:")?;
+            bytes_written += output.write(&info_hash.0)?;
+            bytes_written += output.write(b"d8:completei")?;
+            bytes_written +=
+                output.write(itoa::Buffer::new().format(statistics.complete).as_bytes())?;
+            bytes_written += output.write(b"e10:downloadedi0e10:incompletei")?;
+            bytes_written +=
+                output.write(itoa::Buffer::new().format(statistics.incomplete).as_bytes())?;
+            bytes_written += output.write(b"ee")?;
+        }
+
+        bytes_written += output.write(b"ee")?;
+
+        Ok(bytes_written)
+    }
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct FailureResponse {
+    #[serde(rename = "failure reason")]
+    pub failure_reason: Cow<'static, str>,
+}
+
+impl FailureResponse {
+    pub fn new<S: Into<Cow<'static, str>>>(reason: S) -> Self {
+        Self {
+            failure_reason: reason.into(),
+        }
+    }
+
+    pub fn write<W: Write>(&self, output: &mut W) -> ::std::io::Result<usize> {
+        let mut bytes_written = 0usize;
+
+        let reason_bytes = self.failure_reason.as_bytes();
+
+        bytes_written += output.write(b"d14:failure reason")?;
+        bytes_written += output.write(itoa::Buffer::new().format(reason_bytes.len()).as_bytes())?;
+        bytes_written += output.write(b":")?;
+        bytes_written += output.write(reason_bytes)?;
+        bytes_written += output.write(b"e")?;
+
+        Ok(bytes_written)
+    }
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(untagged)]
+pub enum Response {
+    Announce(AnnounceResponse),
+    Scrape(ScrapeResponse),
+    Failure(FailureResponse),
+}
+
+impl Response {
+    pub fn write<W: Write>(&self, output: &mut W) -> ::std::io::Result<usize> {
+        match self {
+            Response::Announce(r) => r.write(output),
+            Response::Failure(r) => r.write(output),
+            Response::Scrape(r) => r.write(output),
+        }
+    }
+    pub fn from_bytes(bytes: &[u8]) -> Result<Self, ::serde_bencode::Error> {
+        ::serde_bencode::from_bytes(bytes)
+    }
+}
+
+#[cfg(test)]
+impl quickcheck::Arbitrary for ResponsePeer<Ipv4Addr> {
+    fn arbitrary(g: &mut quickcheck::Gen) -> Self {
+        Self {
+            ip_address: Ipv4Addr::arbitrary(g),
+            port: u16::arbitrary(g),
+        }
+    }
+}
+
+#[cfg(test)]
+impl quickcheck::Arbitrary for ResponsePeer<Ipv6Addr> {
+    fn arbitrary(g: &mut quickcheck::Gen) -> Self {
+        Self {
+            ip_address: Ipv6Addr::arbitrary(g),
+            port: u16::arbitrary(g),
+        }
+    }
+}
+
+#[cfg(test)]
+impl quickcheck::Arbitrary for ResponsePeerListV4 {
+    fn arbitrary(g: &mut quickcheck::Gen) -> Self {
+        Self(Vec::arbitrary(g))
+    }
+}
+
+#[cfg(test)]
+impl quickcheck::Arbitrary for ResponsePeerListV6 {
+    fn arbitrary(g: &mut quickcheck::Gen) -> Self {
+        Self(Vec::arbitrary(g))
+    }
+}
+
+#[cfg(test)]
+impl quickcheck::Arbitrary for ScrapeStatistics {
+    fn arbitrary(g: &mut quickcheck::Gen) -> Self {
+        Self {
+            complete: usize::arbitrary(g),
+            incomplete: usize::arbitrary(g),
+            downloaded: 0,
+        }
+    }
+}
+
+#[cfg(test)]
+impl quickcheck::Arbitrary for AnnounceResponse {
+    fn arbitrary(g: &mut quickcheck::Gen) -> Self {
+        Self {
+            announce_interval: usize::arbitrary(g),
+            complete: usize::arbitrary(g),
+            incomplete: usize::arbitrary(g),
+            peers: ResponsePeerListV4::arbitrary(g),
+            peers6: ResponsePeerListV6::arbitrary(g),
+            warning_message: quickcheck::Arbitrary::arbitrary(g),
+        }
+    }
+}
+
+#[cfg(test)]
+impl quickcheck::Arbitrary for ScrapeResponse {
+    fn arbitrary(g: &mut quickcheck::Gen) -> Self {
+        Self {
+            files: BTreeMap::arbitrary(g),
+        }
+    }
+}
+
+#[cfg(test)]
+impl quickcheck::Arbitrary for FailureResponse {
+    fn arbitrary(g: &mut quickcheck::Gen) -> Self {
+        Self {
+            failure_reason: String::arbitrary(g).into(),
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use quickcheck_macros::*;
+
+    use super::*;
+
+    #[quickcheck]
+    fn test_announce_response_to_bytes(response: AnnounceResponse) -> bool {
+        let reference = bendy::serde::to_bytes(&Response::Announce(response.clone())).unwrap();
+
+        let mut hand_written = Vec::new();
+
+        response.write(&mut hand_written).unwrap();
+
+        let success = hand_written == reference;
+
+        if !success {
+            println!("reference:    {}", String::from_utf8_lossy(&reference));
+            println!("hand_written: {}", String::from_utf8_lossy(&hand_written));
+        }
+
+        success
+    }
+
+    #[quickcheck]
+    fn test_scrape_response_to_bytes(response: ScrapeResponse) -> bool {
+        let reference = bendy::serde::to_bytes(&Response::Scrape(response.clone())).unwrap();
+
+        let mut hand_written = Vec::new();
+
+        response.write(&mut hand_written).unwrap();
+
+        let success = hand_written == reference;
+
+        if !success {
+            println!("reference: {}", String::from_utf8_lossy(&reference));
+            println!("hand_written: {}", String::from_utf8_lossy(&hand_written));
+        }
+
+        success
+    }
+
+    #[quickcheck]
+    fn test_failure_response_to_bytes(response: FailureResponse) -> bool {
+        let reference = bendy::serde::to_bytes(&Response::Failure(response.clone())).unwrap();
+
+        let mut hand_written = Vec::new();
+
+        response.write(&mut hand_written).unwrap();
+
+        let success = hand_written == reference;
+
+        if !success {
+            println!("reference:    {}", String::from_utf8_lossy(&reference));
+            println!("hand_written: {}", String::from_utf8_lossy(&hand_written));
+        }
+
+        success
+    }
+}

crates/http_protocol/src/utils.rs

@@ -0,0 +1,334 @@
+use std::io::Write;
+use std::net::{Ipv4Addr, Ipv6Addr};
+
+use anyhow::Context;
+use serde::{de::Visitor, Deserializer, Serializer};
+
+use super::response::ResponsePeer;
+
+pub fn urlencode_20_bytes(input: [u8; 20], output: &mut impl Write) -> ::std::io::Result<()> {
+    let mut tmp = [b'%'; 60];
+
+    for i in 0..input.len() {
+        hex::encode_to_slice(&input[i..i + 1], &mut tmp[i * 3 + 1..i * 3 + 3]).unwrap();
+    }
+
+    output.write_all(&tmp)?;
+
+    Ok(())
+}
+
+pub fn urldecode_20_bytes(value: &str) -> anyhow::Result<[u8; 20]> {
+    let mut out_arr = [0u8; 20];
+
+    let mut chars = value.chars();
+
+    for i in 0..20 {
+        let c = chars.next().with_context(|| "less than 20 chars")?;
+
+        if c as u32 > 255 {
+            return Err(anyhow::anyhow!(
+                "character not in single byte range: {:#?}",
+                c
+            ));
+        }
+
+        if c == '%' {
+            let first = chars
+                .next()
+                .with_context(|| "missing first urldecode char in pair")?;
+            let second = chars
+                .next()
+                .with_context(|| "missing second urldecode char in pair")?;
+
+            let hex = [first as u8, second as u8];
+
+            hex::decode_to_slice(&hex, &mut out_arr[i..i + 1])
+                .map_err(|err| anyhow::anyhow!("hex decode error: {:?}", err))?;
+        } else {
+            out_arr[i] = c as u8;
+        }
+    }
+
+    if chars.next().is_some() {
+        return Err(anyhow::anyhow!("more than 20 chars"));
+    }
+
+    Ok(out_arr)
+}
+
+#[inline]
+pub fn serialize_optional_string<S>(v: &Option<String>, serializer: S) -> Result<S::Ok, S::Error>
+where
+    S: Serializer,
+{
+    match v {
+        Some(s) => serializer.serialize_str(s.as_str()),
+        None => Err(serde::ser::Error::custom("use skip_serializing_if")),
+    }
+}
+
+#[inline]
+pub fn serialize_20_bytes<S>(bytes: &[u8; 20], serializer: S) -> Result<S::Ok, S::Error>
+where
+    S: Serializer,
+{
+    serializer.serialize_bytes(bytes)
+}
+
+struct TwentyByteVisitor;
+
+impl<'de> Visitor<'de> for TwentyByteVisitor {
+    type Value = [u8; 20];
+
+    fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {
+        formatter.write_str("20 bytes")
+    }
+
+    #[inline]
+    fn visit_bytes<E>(self, value: &[u8]) -> Result<Self::Value, E>
+    where
+        E: ::serde::de::Error,
+    {
+        if value.len() != 20 {
+            return Err(::serde::de::Error::custom("not 20 bytes"));
+        }
+
+        let mut arr = [0u8; 20];
+
+        arr.copy_from_slice(value);
+
+        Ok(arr)
+    }
+}
+
+#[inline]
+pub fn deserialize_20_bytes<'de, D>(deserializer: D) -> Result<[u8; 20], D::Error>
+where
+    D: Deserializer<'de>,
+{
+    deserializer.deserialize_any(TwentyByteVisitor)
+}
+
+pub fn serialize_response_peers_ipv4<S>(
+    response_peers: &[ResponsePeer<Ipv4Addr>],
+    serializer: S,
+) -> Result<S::Ok, S::Error>
+where
+    S: Serializer,
+{
+    let mut bytes = Vec::with_capacity(response_peers.len() * 6);
+
+    for peer in response_peers {
+        bytes.extend_from_slice(&u32::from(peer.ip_address).to_be_bytes());
+        bytes.extend_from_slice(&peer.port.to_be_bytes())
+    }
+
+    serializer.serialize_bytes(&bytes)
+}
+
+pub fn serialize_response_peers_ipv6<S>(
+    response_peers: &[ResponsePeer<Ipv6Addr>],
+    serializer: S,
+) -> Result<S::Ok, S::Error>
+where
+    S: Serializer,
+{
+    let mut bytes = Vec::with_capacity(response_peers.len() * 6);
+
+    for peer in response_peers {
+        bytes.extend_from_slice(&u128::from(peer.ip_address).to_be_bytes());
+        bytes.extend_from_slice(&peer.port.to_be_bytes())
+    }
+
+    serializer.serialize_bytes(&bytes)
+}
+
+struct ResponsePeersIpv4Visitor;
+
+impl<'de> Visitor<'de> for ResponsePeersIpv4Visitor {
+    type Value = Vec<ResponsePeer<Ipv4Addr>>;
+
+    fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {
+        formatter.write_str("byte-encoded ipv4 address-port pairs")
+    }
+
+    #[inline]
+    fn visit_bytes<E>(self, value: &[u8]) -> Result<Self::Value, E>
+    where
+        E: ::serde::de::Error,
+    {
+        let chunks = value.chunks_exact(6);
+
+        if !chunks.remainder().is_empty() {
+            return Err(::serde::de::Error::custom("trailing bytes"));
+        }
+
+        let mut ip_bytes = [0u8; 4];
+        let mut port_bytes = [0u8; 2];
+
+        let peers = chunks
+            .into_iter()
+            .map(|chunk| {
+                ip_bytes.copy_from_slice(&chunk[0..4]);
+                port_bytes.copy_from_slice(&chunk[4..6]);
+
+                ResponsePeer {
+                    ip_address: Ipv4Addr::from(u32::from_be_bytes(ip_bytes)),
+                    port: u16::from_be_bytes(port_bytes),
+                }
+            })
+            .collect();
+
+        Ok(peers)
+    }
+}
+
+#[inline]
+pub fn deserialize_response_peers_ipv4<'de, D>(
+    deserializer: D,
+) -> Result<Vec<ResponsePeer<Ipv4Addr>>, D::Error>
+where
+    D: Deserializer<'de>,
+{
+    deserializer.deserialize_any(ResponsePeersIpv4Visitor)
+}
+
+struct ResponsePeersIpv6Visitor;
+
+impl<'de> Visitor<'de> for ResponsePeersIpv6Visitor {
+    type Value = Vec<ResponsePeer<Ipv6Addr>>;
+
+    fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {
+        formatter.write_str("byte-encoded ipv6 address-port pairs")
+    }
+
+    #[inline]
+    fn visit_bytes<E>(self, value: &[u8]) -> Result<Self::Value, E>
+    where
+        E: ::serde::de::Error,
+    {
+        let chunks = value.chunks_exact(18);
+
+        if !chunks.remainder().is_empty() {
+            return Err(::serde::de::Error::custom("trailing bytes"));
+        }
+
+        let mut ip_bytes = [0u8; 16];
+        let mut port_bytes = [0u8; 2];
+
+        let peers = chunks
+            .into_iter()
+            .map(|chunk| {
+                ip_bytes.copy_from_slice(&chunk[0..16]);
+                port_bytes.copy_from_slice(&chunk[16..18]);
+
+                ResponsePeer {
+                    ip_address: Ipv6Addr::from(u128::from_be_bytes(ip_bytes)),
+                    port: u16::from_be_bytes(port_bytes),
+                }
+            })
+            .collect();
+
+        Ok(peers)
+    }
+}
+
+#[inline]
+pub fn deserialize_response_peers_ipv6<'de, D>(
+    deserializer: D,
+) -> Result<Vec<ResponsePeer<Ipv6Addr>>, D::Error>
+where
+    D: Deserializer<'de>,
+{
+    deserializer.deserialize_any(ResponsePeersIpv6Visitor)
+}
+
+#[cfg(test)]
+mod tests {
+    use quickcheck_macros::*;
+
+    use crate::common::InfoHash;
+
+    use super::*;
+
+    #[test]
+    fn test_urlencode_20_bytes() {
+        let mut input = [0u8; 20];
+
+        for (i, b) in input.iter_mut().enumerate() {
+            *b = i as u8 % 10;
+        }
+
+        let mut output = Vec::new();
+
+        urlencode_20_bytes(input, &mut output).unwrap();
+
+        assert_eq!(output.len(), 60);
+
+        for (i, chunk) in output.chunks_exact(3).enumerate() {
+            // Not perfect but should do the job
+            let reference = [b'%', b'0', input[i] + 48];
+
+            let success = chunk == reference;
+
+            if !success {
+                println!("failing index: {}", i);
+            }
+
+            assert_eq!(chunk, reference);
+        }
+    }
+
+    #[quickcheck]
+    fn test_urlencode_urldecode_20_bytes(
+        a: u8,
+        b: u8,
+        c: u8,
+        d: u8,
+        e: u8,
+        f: u8,
+        g: u8,
+        h: u8,
+    ) -> bool {
+        let input: [u8; 20] = [a, b, c, d, e, f, g, h, b, c, d, a, e, f, g, h, a, b, d, c];
+
+        let mut output = Vec::new();
+
+        urlencode_20_bytes(input, &mut output).unwrap();
+
+        let s = ::std::str::from_utf8(&output).unwrap();
+
+        let decoded = urldecode_20_bytes(s).unwrap();
+
+        assert_eq!(input, decoded);
+
+        input == decoded
+    }
+
+    #[quickcheck]
+    fn test_serde_response_peers_ipv4(peers: Vec<ResponsePeer<Ipv4Addr>>) -> bool {
+        let serialized = bendy::serde::to_bytes(&peers).unwrap();
+        let deserialized: Vec<ResponsePeer<Ipv4Addr>> =
+            ::bendy::serde::from_bytes(&serialized).unwrap();
+
+        peers == deserialized
+    }
+
+    #[quickcheck]
+    fn test_serde_response_peers_ipv6(peers: Vec<ResponsePeer<Ipv6Addr>>) -> bool {
+        let serialized = bendy::serde::to_bytes(&peers).unwrap();
+        let deserialized: Vec<ResponsePeer<Ipv6Addr>> =
+            ::bendy::serde::from_bytes(&serialized).unwrap();
+
+        peers == deserialized
+    }
+
+    #[quickcheck]
+    fn test_serde_info_hash(info_hash: InfoHash) -> bool {
+        let serialized = bendy::serde::to_bytes(&info_hash).unwrap();
+        let deserialized: InfoHash = ::bendy::serde::from_bytes(&serialized).unwrap();
+
+        info_hash == deserialized
+    }
+}