diff --git a/README.md b/README.md index 8b3684f..6f37e76 100644 --- a/README.md +++ b/README.md @@ -38,12 +38,12 @@ Crawler for [Aquatic](https://github.com/greatest-ape/aquatic) BitTorrent tracke ## Usage ``` bash -aquatic-crawler --infohash-file /path/to/info-hash-ipv4.json\ - --infohash-file /path/to/info-hash-ipv6.json\ - --infohash-file /path/to/another-source.json\ - --torrent-tracker udp://host1:port\ - --torrent-tracker udp://host2:port\ - --storage /path/to/storage\ +aquatic-crawler --infohash /path/to/info-hash-ipv4.json\ + --infohash /path/to/info-hash-ipv6.json\ + --infohash /path/to/another-source.json\ + --tracker udp://host1:port\ + --tracker udp://host2:port\ + --preload /path/to/directory\ --enable-tcp ``` @@ -62,15 +62,12 @@ aquatic-crawler --infohash-file /path/to/info-hash-ipv4.json\ --clear Clear previous index collected on crawl session start - --infohash-file - Absolute filename(s) to the Aquatic tracker info-hash JSON/API + --infohash + Absolute path(s) or URL(s) to import infohashes from the Aquatic tracker JSON/API * PR#233 feature - --storage - Directory path to store preloaded data (e.g. `.torrent` files) - - --torrent-tracker + --tracker Define custom tracker(s) to preload the `.torrent` files info --initial-peer @@ -105,6 +102,9 @@ aquatic-crawler --infohash-file /path/to/info-hash-ipv4.json\ --enable-upload Enable upload + --preload + Directory path to store preloaded data (e.g. `.torrent` files) + --preload-regex Preload only files match regex pattern (list only without preload by default) * see also `preload_max_filesize`, `preload_max_filecount` options diff --git a/src/config.rs b/src/config.rs index 2154da7..04b1a05 100644 --- a/src/config.rs +++ b/src/config.rs @@ -15,19 +15,15 @@ pub struct Config { #[arg(long, default_value_t = false)] pub clear: bool, - /// Absolute filename(s) to the Aquatic tracker info-hash JSON/API + /// Absolute path(s) or URL(s) to import infohashes from the Aquatic tracker JSON/API /// /// * PR#233 feature #[arg(long)] - pub infohash_file: Vec, - - /// Directory path to store preloaded data (e.g. `.torrent` files) - #[arg(long)] - pub storage: String, + pub infohash: Vec, /// Define custom tracker(s) to preload the `.torrent` files info #[arg(long)] - pub torrent_tracker: Vec, + pub tracker: Vec, /// Define initial peer(s) to preload the `.torrent` files info #[arg(long)] @@ -69,6 +65,10 @@ pub struct Config { #[arg(long, default_value_t = false)] pub enable_upload: bool, + /// Directory path to store preloaded data (e.g. `.torrent` files) + #[arg(long)] + pub preload: Option, + /// Preload only files match regex pattern (list only without preload by default) /// * see also `preload_max_filesize`, `preload_max_filecount` options /// diff --git a/src/main.rs b/src/main.rs index 4e840a9..446cf50 100644 --- a/src/main.rs +++ b/src/main.rs @@ -3,17 +3,17 @@ mod config; mod debug; mod index; mod peers; +mod preload; mod rss; -mod storage; mod torrent; mod trackers; use anyhow::Result; use debug::Debug; use index::Index; +use preload::Preload; use rss::Rss; -use std::{collections::HashSet, num::NonZero, time::Duration}; -use storage::Storage; +use std::{collections::HashSet, num::NonZero, path::PathBuf, time::Duration}; use torrent::Torrent; use url::Url; @@ -30,14 +30,19 @@ async fn main() -> Result<()> { let config = config::Config::parse(); let debug = Debug::init(&config.debug)?; let peers = peers::Peers::init(&config.initial_peer)?; - let storage = Storage::init(&config.storage, config.clear)?; - let trackers = trackers::Trackers::init(&config.torrent_tracker)?; + let preload = config + .preload + .map(|ref p| Preload::init(p, config.clear).unwrap()); + let trackers = trackers::Trackers::init(&config.tracker)?; let torrent = config.export_torrents.map(|p| Torrent::init(&p).unwrap()); let preload_regex = config .preload_regex .map(|ref r| regex::Regex::new(r).unwrap()); let session = librqbit::Session::new_with_opts( - storage.path(), + match preload { + Some(ref p) => p.path(), + None => PathBuf::new(), + }, SessionOptions { connect: Some(ConnectionOptions { enable_tcp: config.enable_tcp, @@ -68,7 +73,10 @@ async fn main() -> Result<()> { loop { debug.info("Index queue begin..."); index.refresh(); - for source in &config.infohash_file { + for source in &config.infohash { + if source.contains("://") { + todo!("URL sources yet not supported") + } debug.info(&format!("Index source `{source}`...")); // grab latest info-hashes from this source // * aquatic server may update the stats at this moment, handle result manually @@ -98,7 +106,9 @@ async fn main() -> Result<()> { )), // the destination folder to preload files match `only_files_regex` // * e.g. images for audio albums - output_folder: storage.output_folder(&i, true).ok(), + output_folder: preload + .as_ref() + .map(|p| p.output_folder(&i, true).unwrap()), ..Default::default() }), ), @@ -140,7 +150,10 @@ async fn main() -> Result<()> { break; } only_files_size += info.len; - only_files_keep.push(storage.absolute(&i, &info.relative_filename)); + if let Some(ref p) = preload { + only_files_keep.push( + p.absolute(&i, &info.relative_filename)) + } only_files.insert(id); } } @@ -159,7 +172,9 @@ async fn main() -> Result<()> { .delete(librqbit::api::TorrentIdOrHash::Id(id), false) .await?; // cleanup irrelevant files (see rqbit#408) - storage.cleanup(&i, Some(only_files_keep))?; + if let Some(p) = &preload { + p.cleanup(&i, Some(only_files_keep))? + } index.insert(i, only_files_size, name) } diff --git a/src/storage.rs b/src/preload.rs similarity index 92% rename from src/storage.rs rename to src/preload.rs index 7834a2e..a2943ca 100644 --- a/src/storage.rs +++ b/src/preload.rs @@ -1,11 +1,11 @@ use anyhow::{Result, bail}; use std::{fs, path::PathBuf, str::FromStr}; -pub struct Storage(PathBuf); +pub struct Preload(PathBuf); -impl Storage { - pub fn init(storage: &str, clear: bool) -> Result { - let p = PathBuf::from_str(storage)?; +impl Preload { + pub fn init(directory: &str, clear: bool) -> Result { + let p = PathBuf::from_str(directory)?; if let Ok(t) = fs::metadata(&p) { if t.is_file() { bail!("Storage destination is not directory!");