rename infohash_file to infohash, storage to optional preload, torrent_tracker to tracker

README.md

@@ -38,12 +38,12 @@ Crawler for [Aquatic](https://github.com/greatest-ape/aquatic) BitTorrent tracke
 ## Usage
 
 ``` bash
-aquatic-crawler --infohash-file   /path/to/info-hash-ipv4.json\
+aquatic-crawler --infohash /path/to/info-hash-ipv4.json\
-                --infohash-file   /path/to/info-hash-ipv6.json\
+                --infohash /path/to/info-hash-ipv6.json\
-                --infohash-file   /path/to/another-source.json\
+                --infohash /path/to/another-source.json\
-                --torrent-tracker udp://host1:port\
+                --tracker  udp://host1:port\
-                --torrent-tracker udp://host2:port\
+                --tracker  udp://host2:port\
-                --storage         /path/to/storage\
+                --preload  /path/to/directory\
                 --enable-tcp
 ```
 
@@ -62,15 +62,12 @@ aquatic-crawler --infohash-file   /path/to/info-hash-ipv4.json\
       --clear
           Clear previous index collected on crawl session start
 
-      --infohash-file <INFOHASH_FILE>
+      --infohash <INFOHASH>
-          Absolute filename(s) to the Aquatic tracker info-hash JSON/API
+          Absolute path(s) or URL(s) to import infohashes from the Aquatic tracker JSON/API
 
           * PR#233 feature
 
-      --storage <STORAGE>
+      --tracker <TRACKER>
-          Directory path to store preloaded data (e.g. `.torrent` files)
-
-      --torrent-tracker <TORRENT_TRACKER>
           Define custom tracker(s) to preload the `.torrent` files info
 
       --initial-peer <INITIAL_PEER>
@@ -105,6 +102,9 @@ aquatic-crawler --infohash-file   /path/to/info-hash-ipv4.json\
       --enable-upload
           Enable upload
 
+      --preload <PRELOAD>
+          Directory path to store preloaded data (e.g. `.torrent` files)
+
       --preload-regex <PRELOAD_REGEX>
           Preload only files match regex pattern (list only without preload by default)
           * see also `preload_max_filesize`, `preload_max_filecount` options

src/config.rs

@@ -15,19 +15,15 @@ pub struct Config {
     #[arg(long, default_value_t = false)]
     pub clear: bool,
 
-    /// Absolute filename(s) to the Aquatic tracker info-hash JSON/API
+    /// Absolute path(s) or URL(s) to import infohashes from the Aquatic tracker JSON/API
     ///
     /// * PR#233 feature
     #[arg(long)]
-    pub infohash_file: Vec<String>,
+    pub infohash: Vec<String>,
-
-    /// Directory path to store preloaded data (e.g. `.torrent` files)
-    #[arg(long)]
-    pub storage: String,
 
     /// Define custom tracker(s) to preload the `.torrent` files info
     #[arg(long)]
-    pub torrent_tracker: Vec<String>,
+    pub tracker: Vec<String>,
 
     /// Define initial peer(s) to preload the `.torrent` files info
     #[arg(long)]
@@ -69,6 +65,10 @@ pub struct Config {
     #[arg(long, default_value_t = false)]
     pub enable_upload: bool,
 
+    /// Directory path to store preloaded data (e.g. `.torrent` files)
+    #[arg(long)]
+    pub preload: Option<String>,
+
     /// Preload only files match regex pattern (list only without preload by default)
     /// * see also `preload_max_filesize`, `preload_max_filecount` options
     ///

src/main.rs

@@ -3,17 +3,17 @@ mod config;
 mod debug;
 mod index;
 mod peers;
+mod preload;
 mod rss;
-mod storage;
 mod torrent;
 mod trackers;
 
 use anyhow::Result;
 use debug::Debug;
 use index::Index;
+use preload::Preload;
 use rss::Rss;
-use std::{collections::HashSet, num::NonZero, time::Duration};
+use std::{collections::HashSet, num::NonZero, path::PathBuf, time::Duration};
-use storage::Storage;
 use torrent::Torrent;
 use url::Url;
 
@@ -30,14 +30,19 @@ async fn main() -> Result<()> {
     let config = config::Config::parse();
     let debug = Debug::init(&config.debug)?;
     let peers = peers::Peers::init(&config.initial_peer)?;
-    let storage = Storage::init(&config.storage, config.clear)?;
+    let preload = config
-    let trackers = trackers::Trackers::init(&config.torrent_tracker)?;
+        .preload
+        .map(|ref p| Preload::init(p, config.clear).unwrap());
+    let trackers = trackers::Trackers::init(&config.tracker)?;
     let torrent = config.export_torrents.map(|p| Torrent::init(&p).unwrap());
     let preload_regex = config
         .preload_regex
         .map(|ref r| regex::Regex::new(r).unwrap());
     let session = librqbit::Session::new_with_opts(
-        storage.path(),
+        match preload {
+            Some(ref p) => p.path(),
+            None => PathBuf::new(),
+        },
         SessionOptions {
             connect: Some(ConnectionOptions {
                 enable_tcp: config.enable_tcp,
@@ -68,7 +73,10 @@ async fn main() -> Result<()> {
     loop {
         debug.info("Index queue begin...");
         index.refresh();
-        for source in &config.infohash_file {
+        for source in &config.infohash {
+            if source.contains("://") {
+                todo!("URL sources yet not supported")
+            }
             debug.info(&format!("Index source `{source}`..."));
             // grab latest info-hashes from this source
             // * aquatic server may update the stats at this moment, handle result manually
@@ -98,7 +106,9 @@ async fn main() -> Result<()> {
                                     )),
                                     // the destination folder to preload files match `only_files_regex`
                                     // * e.g. images for audio albums
-                                    output_folder: storage.output_folder(&i, true).ok(),
+                                    output_folder: preload
+                                        .as_ref()
+                                        .map(|p| p.output_folder(&i, true).unwrap()),
                                     ..Default::default()
                                 }),
                             ),
@@ -140,7 +150,10 @@ async fn main() -> Result<()> {
                                                         break;
                                                     }
                                                     only_files_size += info.len;
-                                                    only_files_keep.push(storage.absolute(&i, &info.relative_filename));
+                                                    if let Some(ref p) = preload {
+                                                        only_files_keep.push(
+                                                            p.absolute(&i, &info.relative_filename))
+                                                    }
                                                     only_files.insert(id);
                                                 }
                                             }
@@ -159,7 +172,9 @@ async fn main() -> Result<()> {
                                         .delete(librqbit::api::TorrentIdOrHash::Id(id), false)
                                         .await?;
                                     // cleanup irrelevant files (see rqbit#408)
-                                    storage.cleanup(&i, Some(only_files_keep))?;
+                                    if let Some(p) = &preload {
+                                        p.cleanup(&i, Some(only_files_keep))?
+                                    }
 
                                     index.insert(i, only_files_size, name)
                                 }

src/preload.rs

@@ -1,11 +1,11 @@
 use anyhow::{Result, bail};
 use std::{fs, path::PathBuf, str::FromStr};
 
-pub struct Storage(PathBuf);
+pub struct Preload(PathBuf);
 
-impl Storage {
+impl Preload {
-    pub fn init(storage: &str, clear: bool) -> Result<Self> {
+    pub fn init(directory: &str, clear: bool) -> Result<Self> {
-        let p = PathBuf::from_str(storage)?;
+        let p = PathBuf::from_str(directory)?;
         if let Ok(t) = fs::metadata(&p) {
             if t.is_file() {
                 bail!("Storage destination is not directory!");