begin the rocket framework implementation, based on the aquatic-crawler fs

Cargo.toml

@@ -4,24 +4,16 @@ version = "0.1.0"
 edition = "2024"
 license = "MIT"
 readme = "README.md"
-description = "Crawler daemon for the yggtracker-redb index, based on the librqbit API"
+description = "BitTorrent aggregation web-server, based on the Rocket framework and aquatic-crawler FS"
-keywords = ["aquatic", "librqbit", "yggtracker", "crawler", "bittorrent"]
+keywords = ["yggtracker", "bittorrent", "server", "aggregator", "catalog"]
 categories = ["network-programming"]
 repository = "https://github.com/YGGverse/yggtrackerd"
 # homepage = "https://yggverse.github.io"
 
 [dependencies]
-anyhow = "1.0"
-chrono = "0.4.41"
 clap = { version = "4.5", features = ["derive"] }
-librqbit = {version = "9.0.0-beta.1", features = ["disable-upload"] }
+rocket = "0.5"
-tokio = { version = "1.45", features = ["full"] }
+librqbit-core = "5.0"
-tracing-subscriber = "0.3"
+chrono = "0.4.41"
 url = "2.5"
 urlencoding = "2.1"
-libyggtracker-redb = "0.1"
-
-[patch.crates-io]
-librqbit = { git = "https://github.com/ikatson/rqbit.git", rev="b580a9610ae7c6eaacd305a3905f7e2d3202ca69" }
-libyggtracker-redb = { git = "https://github.com/YGGverse/libyggtracker-redb.git", rev="e567777ec172a8bf011483c1f49bf1d444543753" }
-# libyggtracker-redb = { path = "../libyggtracker-redb" }

README.md

@@ -4,7 +4,7 @@
 [![Dependencies](https://deps.rs/repo/github/YGGverse/yggtrackerd/status.svg)](https://deps.rs/repo/github/YGGverse/yggtrackerd)
 [![crates.io](https://img.shields.io/crates/v/yggtrackerd.svg)](https://crates.io/crates/yggtrackerd)
 
-Crawler daemon for the yggtracker-redb index, based on the librqbit API
+BitTorrent aggregation web-server, based on the [Rocket](https://rocket.rs) framework and [aquatic-crawler](https://github.com/YGGverse/aquatic-crawler) FS
 
 ## Install
 
@@ -22,6 +22,7 @@ yggtrackerd --infohash /path/to/info-hash-ipv6.bin\
             --database /path/to/index.redb\
             --preload  /path/to/directory
 ```
+* append `RUST_LOG=debug` for detailed information output
 
 ### Options
 

src/api.rs

@@ -1,53 +0,0 @@
-mod info_hash;
-use info_hash::InfoHash;
-
-/// Parse infohash from the source filepath,
-/// decode hash bytes to `InfoHash` array on success.
-///
-/// * return `None` if the `path` is not reachable
-pub fn get(path: &str, capacity: usize) -> Option<Vec<InfoHash>> {
-    use std::io::Read;
-    if !path.ends_with(".bin") {
-        todo!("Only sources in the `.bin` format are supported!")
-    }
-    if path.contains("://") {
-        todo!("URL source format is not supported!")
-    }
-    const L: usize = 20; // v1 only
-    let mut r = Vec::with_capacity(capacity);
-    let mut f = std::fs::File::open(path).ok()?;
-    loop {
-        let mut b = [0; L];
-        if f.read(&mut b).ok()? != L {
-            break;
-        }
-        r.push(InfoHash::V1(b))
-    }
-    Some(r)
-}
-
-#[test]
-fn test() {
-    use std::fs;
-
-    #[cfg(not(any(target_os = "linux", target_os = "macos",)))]
-    {
-        todo!()
-    }
-
-    const C: usize = 2;
-
-    const P0: &str = "/tmp/yggtrackerd-api-test-0.bin";
-    const P1: &str = "/tmp/yggtrackerd-api-test-1.bin";
-    const P2: &str = "/tmp/yggtrackerd-api-test-2.bin";
-
-    fs::write(P0, vec![]).unwrap();
-    fs::write(P1, vec![1; 40]).unwrap(); // 20 + 20 bytes
-
-    assert!(get(P0, C).is_some_and(|b| b.is_empty()));
-    assert!(get(P1, C).is_some_and(|b| b.len() == 2));
-    assert!(get(P2, C).is_none());
-
-    fs::remove_file(P0).unwrap();
-    fs::remove_file(P1).unwrap();
-}

src/api/info_hash.rs

@@ -1,15 +0,0 @@
-pub enum InfoHash {
-    V1([u8; 20]),
-}
-
-impl std::fmt::Display for InfoHash {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        match self {
-            Self::V1(i) => write!(
-                f,
-                "{}",
-                i.iter().map(|b| format!("{b:02x}")).collect::<String>()
-            ),
-        }
-    }
-}

src/config.rs

@@ -1,86 +1,35 @@
 use clap::Parser;
 use std::path::PathBuf;
+use url::Url;
 
 #[derive(Parser, Debug)]
 #[command(version, about, long_about = None)]
 pub struct Config {
-    /// Path to the permanent [redb](https://www.redb.org) database
+    /// Path to the [aquatic-crawler](https://github.com/YGGverse/aquatic-crawler) file storage
     #[arg(long, short)]
-    pub database: PathBuf,
+    pub storage: PathBuf,
 
-    /// Print debug output
+    /// Default listing limit
-    #[arg(long, default_value_t = false)]
+    #[arg(long, default_value_t = 50)]
-    pub debug: bool,
+    pub limit: usize,
 
-    /// Absolute path(s) or URL(s) to import infohashes from the Aquatic tracker binary API
+    /// Default capacity (estimated torrents in `storage`)
-    ///
-    /// * PR#233 feature ([Wiki](https://github.com/YGGverse/aquatic-crawler/wiki/Aquatic))
-    #[arg(long, short)]
-    pub infohash: Vec<String>,
-
-    /// Define custom tracker(s) to preload the `.torrent` files info
-    #[arg(long, short)]
-    pub tracker: Vec<String>,
-
-    /// Define initial peer(s) to preload the `.torrent` files info
-    #[arg(long)]
-    pub initial_peer: Vec<String>,
-
-    /// Appends `--tracker` value to magnets and torrents
-    #[arg(long, default_value_t = false)]
-    pub export_trackers: bool,
-
-    /// Enable DHT resolver
-    #[arg(long, default_value_t = false)]
-    pub enable_dht: bool,
-
-    /// Bind resolver session on specified device name (`tun0`, `mycelium`, etc.)
-    #[arg(long)]
-    pub bind: Option<String>,
-
-    /// Directory path to store temporary preload data
-    #[arg(long, short)]
-    pub preload: PathBuf,
-
-    /// Max size sum of preloaded files per torrent (match `preload_regex`)
-    #[arg(long)]
-    pub preload_max_filesize: Option<u64>,
-
-    /// Max count of preloaded files per torrent (match `preload_regex`)
-    #[arg(long)]
-    pub preload_max_filecount: Option<usize>,
-
-    /// Use `socks5://[username:password@]host:port`
-    #[arg(long)]
-    pub proxy_url: Option<String>,
-
-    // Peer options
-    #[arg(long)]
-    pub peer_connect_timeout: Option<u64>,
-
-    #[arg(long)]
-    pub peer_read_write_timeout: Option<u64>,
-
-    #[arg(long)]
-    pub peer_keep_alive_interval: Option<u64>,
-
-    /// Estimated info-hash index capacity
     #[arg(long, default_value_t = 1000)]
-    pub index_capacity: usize,
+    pub capacity: usize,
 
-    /// Max time to handle each torrent
+    /// Server name
-    #[arg(long, default_value_t = 10)]
+    #[arg(long, default_value_t = String::from("YGGtracker"))]
-    pub add_torrent_timeout: u64,
+    pub title: String,
 
-    /// Crawl loop delay in seconds
+    /// Server description
-    #[arg(long, default_value_t = 300)]
-    pub sleep: u64,
-
-    /// Limit upload speed (b/s)
     #[arg(long)]
-    pub upload_limit: Option<u32>,
+    pub description: Option<String>,
 
-    /// Limit download speed (b/s)
+    /// Canonical URL
     #[arg(long)]
-    pub download_limit: Option<u32>,
+    pub link: Option<Url>,
+
+    /// Appends following tracker(s) to the magnet links
+    #[arg(long)]
+    pub tracker: Option<Vec<Url>>,
 }

src/feed.rs

@@ -0,0 +1,140 @@
+use crate::format;
+use std::collections::HashSet;
+use url::Url;
+
+/// Export crawl index to the RSS file
+pub struct Feed {
+    description: Option<String>,
+    link: Option<String>,
+    title: String,
+    /// Valid, parsed from Url, ready-to-use address string donor
+    trackers: Option<HashSet<String>>,
+}
+
+impl Feed {
+    pub fn init(
+        title: String,
+        description: Option<String>,
+        link: Option<Url>,
+        trackers: Option<HashSet<Url>>,
+    ) -> Self {
+        Self {
+            description: description.map(escape),
+            link: link.map(|s| escape(s.to_string())),
+            title: escape(title),
+            trackers: trackers.map(|v| v.into_iter().map(|u| u.to_string()).collect()),
+        }
+    }
+
+    pub fn transaction(&self, capacity: usize) -> String {
+        let t = chrono::Utc::now().to_rfc2822();
+        let mut b = String::with_capacity(capacity);
+
+        b.push_str("<?xml version=\"1.0\" encoding=\"UTF-8\"?><rss version=\"2.0\"><channel>");
+
+        b.push_str("<pubDate>");
+        b.push_str(&t);
+        b.push_str("</pubDate>");
+
+        b.push_str("<lastBuildDate>");
+        b.push_str(&t);
+        b.push_str("</lastBuildDate>");
+
+        b.push_str("<title>");
+        b.push_str(&self.title);
+        b.push_str("</title>");
+
+        if let Some(ref description) = self.description {
+            b.push_str("<description>");
+            b.push_str(description);
+            b.push_str("</description>")
+        }
+
+        if let Some(ref link) = self.link {
+            b.push_str("<link>");
+            b.push_str(link);
+            b.push_str("</link>")
+        }
+        b
+    }
+
+    /// Append `item` to the feed `channel`
+    pub fn push(&self, buffer: &mut String, torrent: crate::storage::Torrent) {
+        buffer.push_str(&format!(
+            "<item><guid>{}</guid><title>{}</title><link>{}</link>",
+            &torrent.info_hash,
+            escape(
+                torrent
+                    .name
+                    .as_ref()
+                    .map(|b| b.to_string())
+                    .unwrap_or("?".into()) // @TODO
+            ),
+            escape(self.magnet(&torrent.info_hash))
+        ));
+
+        if let Some(d) = item_description(torrent.length, torrent.files) {
+            buffer.push_str("<description>");
+            buffer.push_str(&escape(d));
+            buffer.push_str("</description>")
+        }
+
+        buffer.push_str("<pubDate>");
+        buffer.push_str(&torrent.time.to_rfc2822());
+        buffer.push_str("</pubDate>");
+
+        buffer.push_str("</item>")
+    }
+
+    /// Write final bytes
+    pub fn commit(&self, mut buffer: String) -> String {
+        buffer.push_str("</channel></rss>");
+        buffer
+    }
+
+    // Tools
+
+    fn magnet(&self, info_hash: &str) -> String {
+        let mut b = if info_hash.len() == 40 {
+            format!("magnet:?xt=urn:btih:{info_hash}")
+        } else {
+            todo!("info-hash v2 is not supported by librqbit")
+        };
+        if let Some(ref trackers) = self.trackers {
+            for tracker in trackers {
+                b.push_str("&tr=");
+                b.push_str(&urlencoding::encode(tracker))
+            }
+        }
+        b
+    }
+}
+
+fn escape(subject: String) -> String {
+    subject
+        .replace('&', "&amp;")
+        .replace('<', "&lt;")
+        .replace('>', "&gt;")
+        .replace('"', "&quot;")
+        .replace("'", "&apos;")
+}
+
+fn item_description(size: Option<u64>, list: Option<Vec<crate::storage::File>>) -> Option<String> {
+    if size.is_none() && list.is_none() {
+        return None;
+    }
+    let mut b = Vec::with_capacity(list.as_ref().map(|l| l.len()).unwrap_or_default() + 1);
+    if let Some(s) = size {
+        b.push(format::bytes(s))
+    }
+    if let Some(files) = list {
+        for file in files {
+            b.push(format!(
+                "{} ({})",
+                file.name.as_deref().unwrap_or("?"), // @TODO invalid encoding
+                format::bytes(file.length)
+            ))
+        }
+    }
+    Some(b.join("\n"))
+}

src/format.rs

@@ -0,0 +1,17 @@
+pub fn bytes(value: u64) -> String {
+    const KB: f32 = 1024.0;
+    const MB: f32 = KB * KB;
+    const GB: f32 = MB * KB;
+
+    let f = value as f32;
+
+    if f < KB {
+        format!("{value} B")
+    } else if f < MB {
+        format!("{:.2} KB", f / KB)
+    } else if f < GB {
+        format!("{:.2} MB", f / MB)
+    } else {
+        format!("{:.2} GB", f / GB)
+    }
+}

src/main.rs

@@ -1,283 +1,53 @@
-mod api;
+#[macro_use]
+extern crate rocket;
+
 mod config;
-mod peers;
+mod feed;
-mod preload;
+mod format;
-mod trackers;
+mod storage;
 
-use anyhow::Result;
 use config::Config;
-use librqbit::{
+use feed::Feed;
-    AddTorrent, AddTorrentOptions, AddTorrentResponse, ConnectionOptions, PeerConnectionOptions,
+use rocket::{
-    SessionOptions,
+    State,
+    http::Status,
+    response::{content::RawXml, status::Custom},
 };
-use libyggtracker_redb::{
+use storage::{Order, Sort, Storage};
-    Database,
-    torrent::{Image, Torrent, image},
-};
-use peers::Peers;
-use preload::Preload;
-use std::{collections::HashSet, num::NonZero, os::unix::ffi::OsStrExt, time::Duration};
-use trackers::Trackers;
-use url::Url;
 
-#[tokio::main]
+#[get("/")]
-async fn main() -> Result<()> {
+pub fn index() -> &'static str {
-    use chrono::Local;
+    "Catalog in development, use /rss"
+}
+
+#[get("/rss")]
+pub fn rss(feed: &State<Feed>, storage: &State<Storage>) -> Result<RawXml<String>, Custom<String>> {
+    let mut b = feed.transaction(1024); // @TODO
+    for torrent in storage
+        .torrents(
+            Some((Sort::Modified, Order::Asc)),
+            Some(storage.default_limit),
+        )
+        .map_err(|e| Custom(Status::InternalServerError, e.to_string()))?
+    {
+        feed.push(&mut b, torrent)
+    }
+    Ok(RawXml(feed.commit(b)))
+}
+
+#[launch]
+fn rocket() -> _ {
     use clap::Parser;
-    use tokio::time;
-
-    // init components
-    let time_init = Local::now();
     let config = Config::parse();
-    if std::env::var("RUST_LOG").is_ok() {
+    let feed = Feed::init(
-        tracing_subscriber::fmt::init()
+        config.title,
-    } // librqbit impl dependency
+        config.description,
-    let database = Database::init(&config.database)?;
+        config.link,
-    let peers = Peers::init(&config.initial_peer)?;
+        config.tracker.map(|u| u.into_iter().collect()), // make sure it's unique
-    let preload = Preload::init(
+    );
-        config.preload,
+    let storage = Storage::init(config.storage, config.limit, config.capacity).unwrap(); // @TODO handle
-        config.preload_max_filecount,
+    rocket::build()
-        config.preload_max_filesize,
+        .manage(feed)
-    )?;
+        .manage(storage)
-    let trackers = Trackers::init(&config.tracker)?;
+        .mount("/", routes![index, rss])
-    let session = librqbit::Session::new_with_opts(
-        preload.root().clone(),
-        SessionOptions {
-            bind_device_name: config.bind,
-            listen: None,
-            connect: Some(ConnectionOptions {
-                enable_tcp: true,
-                proxy_url: config.proxy_url,
-                peer_opts: Some(PeerConnectionOptions {
-                    connect_timeout: config.peer_connect_timeout.map(Duration::from_secs),
-                    read_write_timeout: config.peer_read_write_timeout.map(Duration::from_secs),
-                    keep_alive_interval: config.peer_keep_alive_interval.map(Duration::from_secs),
-                }),
-            }),
-            disable_upload: false,
-            disable_dht: !config.enable_dht,
-            disable_dht_persistence: true,
-            persistence: None,
-            ratelimits: librqbit::limits::LimitsConfig {
-                upload_bps: config.upload_limit.and_then(NonZero::new),
-                download_bps: config.download_limit.and_then(NonZero::new),
-            },
-            trackers: trackers.list().clone(),
-            ..SessionOptions::default()
-        },
-    )
-    .await?;
-
-    // begin
-    println!("Crawler started on {time_init}");
-    loop {
-        let time_queue = Local::now();
-        if config.debug {
-            println!("\tQueue crawl begin on {time_queue}...")
-        }
-        for source in &config.infohash {
-            if config.debug {
-                println!("\tIndex source `{source}`...")
-            }
-            // grab latest info-hashes from this source
-            // * aquatic server may update the stats at this moment, handle result manually
-            for i in match api::get(source, config.index_capacity) {
-                Some(i) => i,
-                None => {
-                    // skip without panic
-                    if config.debug {
-                        eprintln!(
-                            "The feed `{source}` has an incomplete format (or is still updating); skip."
-                        )
-                    }
-                    continue;
-                }
-            } {
-                // convert to string once
-                let i = i.to_string();
-                // already indexed?
-                if database.has_torrent(&i)? {
-                    continue;
-                }
-                if config.debug {
-                    println!("\t\tIndex `{i}`...")
-                }
-                // run the crawler in single thread for performance reasons,
-                // use `timeout` argument option to skip the dead connections.
-                match time::timeout(
-                    Duration::from_secs(config.add_torrent_timeout),
-                    session.add_torrent(
-                        AddTorrent::from_url(magnet(
-                            &i,
-                            if config.export_trackers && !trackers.is_empty() {
-                                Some(trackers.list())
-                            } else {
-                                None
-                            },
-                        )),
-                        Some(AddTorrentOptions {
-                            paused: true, // continue after `only_files` init
-                            overwrite: true,
-                            disable_trackers: trackers.is_empty(),
-                            initial_peers: peers.initial_peers(),
-                            list_only: false, // we want to grab the images
-                            // it is important to blacklist all files preload until initiation
-                            only_files: Some(Vec::with_capacity(
-                                config.preload_max_filecount.unwrap_or_default(),
-                            )),
-                            // the folder to preload temporary files (e.g. images for the audio albums)
-                            output_folder: Some(
-                                preload.output_folder(&i)?.to_string_lossy().to_string(),
-                            ),
-                            ..Default::default()
-                        }),
-                    ),
-                )
-                .await
-                {
-                    Ok(r) => match r {
-                        Ok(AddTorrentResponse::Added(id, mt)) => {
-                            let mut only_files = HashSet::with_capacity(
-                                config
-                                    .preload_max_filecount
-                                    .unwrap_or(config.index_capacity),
-                            );
-                            let mut images = Vec::with_capacity(
-                                config
-                                    .preload_max_filecount
-                                    .unwrap_or(config.index_capacity),
-                            );
-                            mt.wait_until_initialized().await?;
-                            let bytes = mt.with_metadata(|m| {
-                                for info in &m.file_infos {
-                                    if preload.max_filecount.is_some_and(|limit| only_files.len() + 1 > limit) {
-                                        if config.debug {
-                                            println!(
-                                                "\t\t\ttotal files count limit ({}) for `{i}` reached!",
-                                                preload.max_filecount.unwrap()
-                                            )
-                                        }
-                                        break;
-                                    }
-                                    if info.relative_filename.extension().is_none_or(|e|
-                                        !matches!(e.as_bytes(), b"png" | b"jpeg" | b"jpg" | b"gif" | b"webp")) {
-                                        continue;
-                                    }
-                                    if preload.max_filesize.is_some_and(|limit| info.len > limit) {
-                                        if config.debug {
-                                            println!(
-                                                "\t\t\ttotal files size limit `{i}` reached!"
-                                            )
-                                        }
-                                        continue;
-                                    }
-                                    assert!(only_files.insert(id));
-                                    images.push(info.relative_filename.clone());
-                                }
-                                m.info_bytes.to_vec()
-                            })?;
-                            session.update_only_files(&mt, &only_files).await?;
-                            session.unpause(&mt).await?;
-                            mt.wait_until_completed().await?;
-
-                            // persist torrent data resolved
-                            database.set_torrent(
-                                &i,
-                                Torrent {
-                                    bytes,
-                                    images: if images.is_empty() {
-                                        None
-                                    } else {
-                                        Some(
-                                            images
-                                                .into_iter()
-                                                .filter_map(|p| {
-                                                    extension(&p).map(|extension| Image {
-                                                        alt: p.to_str().map(|s| s.to_string()),
-                                                        bytes: preload.bytes(&p).unwrap(),
-                                                        extension,
-                                                    })
-                                                })
-                                                .collect(),
-                                        )
-                                    },
-                                    time: chrono::Utc::now(),
-                                },
-                            )?;
-
-                            // remove torrent from session as indexed
-                            session
-                                .delete(librqbit::api::TorrentIdOrHash::Id(id), false)
-                                .await?;
-
-                            // cleanup `output_folder` only if the torrent is resolved
-                            // to prevent extra write operations on the next iteration
-                            preload.clear_output_folder(&i)?;
-
-                            if config.debug {
-                                println!("\t\t\ttorrent data successfully resolved.")
-                            }
-                        }
-                        Ok(_) => panic!(),
-                        Err(e) => eprintln!("Failed to resolve `{i}`: `{e}`."),
-                    },
-                    Err(e) => {
-                        if config.debug {
-                            println!("\t\t\tfailed to resolve `{i}`: `{e}`")
-                        }
-                    }
-                }
-            }
-        }
-        if config.debug {
-            println!(
-                "Queue completed on {time_queue}\n\ttime: {} s\n\tuptime: {} s\n\tawait {} seconds to continue...",
-                Local::now()
-                    .signed_duration_since(time_queue)
-                    .as_seconds_f32(),
-                Local::now()
-                    .signed_duration_since(time_init)
-                    .as_seconds_f32(),
-                config.sleep,
-            )
-        }
-        std::thread::sleep(Duration::from_secs(config.sleep))
-    }
-}
-
-/// Build magnet URI
-fn magnet(infohash: &str, trackers: Option<&HashSet<Url>>) -> String {
-    let mut m = if infohash.len() == 40 {
-        format!("magnet:?xt=urn:btih:{infohash}")
-    } else {
-        todo!("infohash v2 is not supported by librqbit")
-    };
-    if let Some(t) = trackers {
-        for tracker in t {
-            m.push_str("&tr=");
-            m.push_str(&urlencoding::encode(tracker.as_str()))
-        }
-    }
-    m
-}
-
-use image::Extension;
-fn extension(path: &std::path::Path) -> Option<Extension> {
-    match path.extension() {
-        Some(p) => {
-            let e = p.to_string_lossy().to_lowercase();
-            if e == "png" {
-                Some(Extension::Png)
-            } else if e == "jpeg" || e == "jpg" {
-                Some(Extension::Jpeg)
-            } else if e == "webp" {
-                Some(Extension::Webp)
-            } else if e == "gif" {
-                Some(Extension::Gif)
-            } else {
-                return None;
-            }
-        }
-        None => None,
-    }
 }

src/peers.rs

@@ -1,21 +0,0 @@
-use std::{net::SocketAddr, str::FromStr};
-
-pub struct Peers(Vec<SocketAddr>);
-
-impl Peers {
-    pub fn init(peers: &Vec<String>) -> anyhow::Result<Self> {
-        let mut p = Vec::with_capacity(peers.len());
-        for peer in peers {
-            p.push(SocketAddr::from_str(peer)?);
-        }
-        Ok(Self(p))
-    }
-
-    pub fn initial_peers(&self) -> Option<Vec<SocketAddr>> {
-        if self.0.is_empty() {
-            None
-        } else {
-            Some(self.0.clone())
-        }
-    }
-}

src/preload.rs

@@ -1,76 +0,0 @@
-use anyhow::{Result, bail};
-use std::{fs, path::PathBuf};
-
-/// Temporary file storage for `librqbit` preload data
-pub struct Preload {
-    root: PathBuf,
-    pub max_filecount: Option<usize>,
-    pub max_filesize: Option<u64>,
-}
-
-impl Preload {
-    pub fn init(
-        directory: PathBuf,
-        max_filecount: Option<usize>,
-        max_filesize: Option<u64>,
-    ) -> Result<Self> {
-        if !directory.is_dir() {
-            bail!("Preload location is not directory!");
-        }
-        Ok(Self {
-            max_filecount,
-            max_filesize,
-            root: directory.canonicalize()?,
-        })
-    }
-
-    pub fn clear_output_folder(&self, info_hash: &str) -> Result<()> {
-        if !is_info_hash(info_hash) {
-            bail!("Invalid info-hash `{info_hash}`")
-        }
-        let mut p = PathBuf::from(&self.root);
-        p.push(info_hash);
-        if !p.is_dir() {
-            bail!(
-                "Requested target `{}` is not directory!",
-                p.to_string_lossy()
-            )
-        }
-        Ok(fs::remove_dir_all(&p)?)
-    }
-
-    /// * create new directory if not exists
-    pub fn output_folder(&self, info_hash: &str) -> Result<PathBuf> {
-        if !is_info_hash(info_hash) {
-            bail!("Invalid info-hash `{info_hash}`")
-        }
-        let mut p = PathBuf::from(&self.root);
-        p.push(info_hash);
-        if !p.exists() {
-            fs::create_dir(&p)?
-        }
-        Ok(p)
-    }
-
-    pub fn root(&self) -> &PathBuf {
-        &self.root
-    }
-
-    pub fn bytes(&self, relative: &PathBuf) -> Result<Vec<u8>> {
-        let mut p = PathBuf::from(&self.root);
-        p.push(relative);
-        // make sure that given relative path
-        // does not contain relative navigation entities
-        if !p.canonicalize()?.starts_with(&self.root) {
-            bail!(
-                "Unexpected absolute path resolved for `{}`!",
-                p.to_string_lossy()
-            )
-        }
-        Ok(std::fs::read(p)?)
-    }
-}
-
-fn is_info_hash(value: &str) -> bool {
-    value.len() == 40 && value.chars().all(|c| c.is_ascii_hexdigit())
-}

src/storage.rs

@@ -0,0 +1,180 @@
+use chrono::{DateTime, Utc};
+use librqbit_core::{torrent_metainfo, torrent_metainfo::TorrentMetaV1Owned};
+use std::{
+    fs::{self, DirEntry},
+    path::PathBuf,
+};
+
+#[derive(Clone, Debug, Default)]
+pub enum Sort {
+    #[default]
+    Modified,
+}
+
+#[derive(Clone, Debug, Default)]
+pub enum Order {
+    #[default]
+    Asc,
+    Desc,
+}
+
+pub struct File {
+    pub name: Option<String>,
+    pub length: u64,
+}
+
+pub struct Torrent {
+    pub announce: Option<String>,
+    pub comment: Option<String>,
+    pub created_by: Option<String>,
+    pub creation_date: Option<DateTime<Utc>>,
+    pub files: Option<Vec<File>>,
+    pub info_hash: String,
+    pub is_private: bool,
+    pub length: Option<u64>,
+    pub name: Option<String>,
+    pub publisher_url: Option<String>,
+    pub publisher: Option<String>,
+    /// File (modified)
+    pub time: DateTime<Utc>,
+}
+
+pub struct Storage {
+    pub default_limit: usize,
+    default_capacity: usize,
+    root: PathBuf,
+}
+
+impl Storage {
+    // Constructors
+
+    pub fn init(
+        root: PathBuf,
+        default_limit: usize,
+        default_capacity: usize,
+    ) -> Result<Self, String> {
+        if !root.is_dir() {
+            return Err("Storage root is not directory".into());
+        }
+        Ok(Self {
+            default_limit,
+            default_capacity,
+            root: root.canonicalize().map_err(|e| e.to_string())?,
+        })
+    }
+
+    // Getters
+
+    pub fn torrents(
+        &self,
+        sort_order: Option<(Sort, Order)>,
+        limit: Option<usize>,
+    ) -> Result<Vec<Torrent>, String> {
+        let f = self.files(sort_order)?;
+        let l = limit.unwrap_or(f.len());
+        let mut b = Vec::with_capacity(l);
+        for file in f.into_iter().take(l) {
+            if file
+                .path()
+                .extension()
+                .is_none_or(|e| e.is_empty() || e.to_string_lossy() != "torrent")
+            {
+                return Err("Unexpected file extension".into());
+            }
+            let i: TorrentMetaV1Owned = torrent_metainfo::torrent_from_bytes(
+                &fs::read(file.path()).map_err(|e| e.to_string())?,
+            )
+            .map_err(|e| e.to_string())?;
+            b.push(Torrent {
+                info_hash: i.info_hash.as_string(),
+                announce: i.announce.map(|a| a.to_string()),
+                comment: i.comment.map(|c| c.to_string()),
+                created_by: i.created_by.map(|c| c.to_string()),
+                creation_date: i
+                    .creation_date
+                    .map(|t| DateTime::from_timestamp_nanos(t as i64)),
+                files: i.info.files.map(|files| {
+                    let limit = 1000; // @TODO
+                    let mut b = Vec::with_capacity(files.len());
+                    let mut i = files.iter();
+                    let mut t = 0;
+                    for f in i.by_ref() {
+                        if t < limit {
+                            t += 1;
+                            b.push(File {
+                                name: String::from_utf8(
+                                    f.path
+                                        .iter()
+                                        .enumerate()
+                                        .flat_map(|(n, b)| {
+                                            if n == 0 {
+                                                b.0.to_vec()
+                                            } else {
+                                                let mut p = vec![b'/'];
+                                                p.extend(b.0.to_vec());
+                                                p
+                                            }
+                                        })
+                                        .collect(),
+                                )
+                                .ok(),
+                                length: f.length,
+                            });
+                            continue;
+                        }
+                        // limit reached: count sizes left and use placeholder as the last item name
+                        let mut l = 0;
+                        for f in i.by_ref() {
+                            l += f.length
+                        }
+                        b.push(File {
+                            name: Some("...".to_string()),
+                            length: l,
+                        });
+                        break;
+                    }
+                    b[..t].sort_by(|a, b| a.name.cmp(&b.name)); // @TODO optional
+                    b
+                }),
+                publisher_url: i.publisher_url.map(|u| u.to_string()),
+                publisher: i.publisher.map(|p| p.to_string()),
+                is_private: i.info.private,
+                length: i.info.length,
+                name: i.info.name.map(|e| e.to_string()),
+                time: file
+                    .metadata()
+                    .map_err(|e| e.to_string())?
+                    .modified()
+                    .map_err(|e| e.to_string())?
+                    .into(),
+            })
+        }
+        Ok(b)
+    }
+
+    // Helpers
+
+    fn files(&self, sort_order: Option<(Sort, Order)>) -> Result<Vec<DirEntry>, String> {
+        let mut b = Vec::with_capacity(self.default_capacity);
+        for entry in fs::read_dir(&self.root).map_err(|e| e.to_string())? {
+            let e = entry.map_err(|e| e.to_string())?;
+            match e.file_type() {
+                Ok(t) => {
+                    if t.is_file() {
+                        b.push((e.metadata().unwrap().modified().unwrap(), e))
+                    }
+                }
+                Err(e) => warn!("{}", e.to_string()),
+            }
+        }
+        if let Some((sort, order)) = sort_order {
+            match sort {
+                Sort::Modified => match order {
+                    Order::Asc => b.sort_by(|a, b| a.0.cmp(&b.0)),
+                    Order::Desc => b.sort_by(|a, b| b.0.cmp(&a.0)),
+                },
+            }
+        }
+        Ok(b.into_iter().map(|e| e.1).collect())
+    }
+}

src/trackers.rs

@@ -1,20 +0,0 @@
-use std::{collections::HashSet, str::FromStr};
-use url::Url;
-
-pub struct Trackers(HashSet<Url>);
-
-impl Trackers {
-    pub fn init(trackers: &Vec<String>) -> anyhow::Result<Self> {
-        let mut t = HashSet::with_capacity(trackers.len());
-        for tracker in trackers {
-            t.insert(Url::from_str(tracker)?);
-        }
-        Ok(Self(t))
-    }
-    pub fn is_empty(&self) -> bool {
-        self.0.is_empty()
-    }
-    pub fn list(&self) -> &HashSet<Url> {
-        &self.0
-    }
-}