YGGverse/aquatic-crawler
1
0
Fork 0
mirror of https://github.com/YGGverse/aquatic-crawler.git synced 2026-04-02 10:05:30 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

rename argument to config

Browse source
This commit is contained in:
yggverse 2025-07-07 01:44:55 +03:00
parent b30be0e9f8
commit 39e717ec04
2 changed files with 35 additions and 33 deletions
Download patch file Download diff file Expand all files Collapse all files

2
src/argument.rs → src/config.rs
View file

@ -2,7 +2,7 @@ use clap::Parser;
#[derive(Parser, Debug)] #[derive(Parser, Debug)]
#[command(version, about, long_about = None)] #[command(version, about, long_about = None)]
pub struct Argument { pub struct Config {
/// Debug level /// Debug level
/// ///
/// * `e` - error /// * `e` - error

66
src/main.rs
View file

@ -1,5 +1,5 @@
mod api; mod api;
mod argument; mod config;
mod debug; mod debug;
mod index; mod index;
mod peers; mod peers;
@ -27,32 +27,34 @@ async fn main() -> Result<()> {
use tokio::time; use tokio::time;
// init components // init components
let arg = argument::Argument::parse(); let config = config::Config::parse();
let debug = Debug::init(&arg.debug)?; let debug = Debug::init(&config.debug)?;
let peers = peers::Peers::init(&arg.initial_peer)?; let peers = peers::Peers::init(&config.initial_peer)?;
let storage = Storage::init(&arg.storage, arg.clear)?; let storage = Storage::init(&config.storage, config.clear)?;
let trackers = trackers::Trackers::init(&arg.torrent_tracker)?; let trackers = trackers::Trackers::init(&config.torrent_tracker)?;
let torrent = arg.export_torrents.map(|p| Torrent::init(&p).unwrap()); let torrent = config.export_torrents.map(|p| Torrent::init(&p).unwrap());
let preload_regex = arg.preload_regex.map(|ref r| regex::Regex::new(r).unwrap()); let preload_regex = config
.preload_regex
.map(|ref r| regex::Regex::new(r).unwrap());
let session = librqbit::Session::new_with_opts( let session = librqbit::Session::new_with_opts(
storage.path(), storage.path(),
SessionOptions { SessionOptions {
connect: Some(ConnectionOptions { connect: Some(ConnectionOptions {
enable_tcp: arg.enable_tcp, enable_tcp: config.enable_tcp,
proxy_url: arg.proxy_url, proxy_url: config.proxy_url,
peer_opts: Some(PeerConnectionOptions { peer_opts: Some(PeerConnectionOptions {
connect_timeout: arg.peer_connect_timeout.map(Duration::from_secs), connect_timeout: config.peer_connect_timeout.map(Duration::from_secs),
read_write_timeout: arg.peer_read_write_timeout.map(Duration::from_secs), read_write_timeout: config.peer_read_write_timeout.map(Duration::from_secs),
keep_alive_interval: arg.peer_keep_alive_interval.map(Duration::from_secs), keep_alive_interval: config.peer_keep_alive_interval.map(Duration::from_secs),
}), }),
}), }),
disable_upload: !arg.enable_upload, disable_upload: !config.enable_upload,
disable_dht: !arg.enable_dht, disable_dht: !config.enable_dht,
disable_dht_persistence: true, disable_dht_persistence: true,
persistence: None, persistence: None,
ratelimits: librqbit::limits::LimitsConfig { ratelimits: librqbit::limits::LimitsConfig {
upload_bps: arg.upload_limit.and_then(NonZero::new), upload_bps: config.upload_limit.and_then(NonZero::new),
download_bps: arg.download_limit.and_then(NonZero::new), download_bps: config.download_limit.and_then(NonZero::new),
}, },
trackers: trackers.clone(), trackers: trackers.clone(),
..SessionOptions::default() ..SessionOptions::default()
@ -62,11 +64,11 @@ async fn main() -> Result<()> {
// begin // begin
debug.info("Crawler started"); debug.info("Crawler started");
let mut index = Index::init(arg.index_capacity); let mut index = Index::init(config.index_capacity);
loop { loop {
debug.info("Index queue begin..."); debug.info("Index queue begin...");
index.refresh(); index.refresh();
for source in &arg.infohash_file { for source in &config.infohash_file {
debug.info(&format!("Index source `{source}`...")); debug.info(&format!("Index source `{source}`..."));
// grab latest info-hashes from this source // grab latest info-hashes from this source
// * aquatic server may update the stats at this moment, handle result manually // * aquatic server may update the stats at this moment, handle result manually
@ -81,7 +83,7 @@ async fn main() -> Result<()> {
// run the crawler in single thread for performance reasons, // run the crawler in single thread for performance reasons,
// use `timeout` argument option to skip the dead connections. // use `timeout` argument option to skip the dead connections.
match time::timeout( match time::timeout(
Duration::from_secs(arg.add_torrent_timeout), Duration::from_secs(config.add_torrent_timeout),
session.add_torrent( session.add_torrent(
AddTorrent::from_url(magnet(&i, None)), AddTorrent::from_url(magnet(&i, None)),
Some(AddTorrentOptions { Some(AddTorrentOptions {
@ -92,7 +94,7 @@ async fn main() -> Result<()> {
list_only: preload_regex.is_none(), list_only: preload_regex.is_none(),
// it is important to blacklist all files preload until initiation // it is important to blacklist all files preload until initiation
only_files: Some(Vec::with_capacity( only_files: Some(Vec::with_capacity(
arg.preload_max_filecount.unwrap_or_default(), config.preload_max_filecount.unwrap_or_default(),
)), )),
// the destination folder to preload files match `only_files_regex` // the destination folder to preload files match `only_files_regex`
// * e.g. images for audio albums // * e.g. images for audio albums
@ -108,10 +110,10 @@ async fn main() -> Result<()> {
Ok(AddTorrentResponse::Added(id, mt)) => { Ok(AddTorrentResponse::Added(id, mt)) => {
let mut only_files_size = 0; let mut only_files_size = 0;
let mut only_files_keep = Vec::with_capacity( let mut only_files_keep = Vec::with_capacity(
arg.preload_max_filecount.unwrap_or_default(), config.preload_max_filecount.unwrap_or_default(),
); );
let mut only_files = HashSet::with_capacity( let mut only_files = HashSet::with_capacity(
arg.preload_max_filecount.unwrap_or_default(), config.preload_max_filecount.unwrap_or_default(),
); );
mt.wait_until_initialized().await?; mt.wait_until_initialized().await?;
let name = mt.with_metadata(|m| { let name = mt.with_metadata(|m| {
@ -121,7 +123,7 @@ async fn main() -> Result<()> {
if regex.is_match( if regex.is_match(
info.relative_filename.to_str().unwrap(), info.relative_filename.to_str().unwrap(),
) { ) {
if arg.preload_max_filesize.is_some_and( if config.preload_max_filesize.is_some_and(
|limit| only_files_size + info.len > limit, |limit| only_files_size + info.len > limit,
) { ) {
debug.info(&format!( debug.info(&format!(
@ -129,7 +131,7 @@ async fn main() -> Result<()> {
)); ));
break; break;
} }
if arg.preload_max_filecount.is_some_and( if config.preload_max_filecount.is_some_and(
|limit| only_files.len() + 1 > limit, |limit| only_files.len() + 1 > limit,
) { ) {
debug.info(&format!( debug.info(&format!(
@ -183,14 +185,14 @@ async fn main() -> Result<()> {
Err(e) => debug.error(&format!("API issue for `{source}`: `{e}`")), Err(e) => debug.error(&format!("API issue for `{source}`: `{e}`")),
} }
} }
if let Some(ref export_rss) = arg.export_rss if let Some(ref export_rss) = config.export_rss
&& index.is_changed() && index.is_changed()
{ {
let mut rss = Rss::new( let mut rss = Rss::new(
export_rss, export_rss,
&arg.export_rss_title, &config.export_rss_title,
&arg.export_rss_link, &config.export_rss_link,
&arg.export_rss_description, &config.export_rss_description,
Some(trackers.clone()), Some(trackers.clone()),
)?; )?;
for (k, v) in index.list() { for (k, v) in index.list() {
@ -203,15 +205,15 @@ async fn main() -> Result<()> {
} }
rss.commit()? rss.commit()?
} }
if arg.preload_total_size.is_some_and(|s| index.nodes() > s) { if config.preload_total_size.is_some_and(|s| index.nodes() > s) {
panic!("Preload content size {} bytes reached!", 0) panic!("Preload content size {} bytes reached!", 0)
} }
debug.info(&format!( debug.info(&format!(
"Index completed, {} total, await {} seconds to continue...", "Index completed, {} total, await {} seconds to continue...",
index.len(), index.len(),
arg.sleep, config.sleep,
)); ));
std::thread::sleep(Duration::from_secs(arg.sleep)); std::thread::sleep(Duration::from_secs(config.sleep));
} }
} }