YGGverse/aquatic-crawler
1
0
Fork 0
mirror of https://github.com/YGGverse/aquatic-crawler.git synced 2026-03-31 17:15:35 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

rename argument to config

Browse source
This commit is contained in:
yggverse 2025-07-07 01:44:55 +03:00
parent b30be0e9f8
commit 39e717ec04
2 changed files with 35 additions and 33 deletions
Download patch file Download diff file Expand all files Collapse all files

66
src/main.rs
View file

@ -1,5 +1,5 @@
mod api;
mod argument;
mod config;
mod debug;
mod index;
mod peers;
@ -27,32 +27,34 @@ async fn main() -> Result<()> {
use tokio::time;
// init components
let arg = argument::Argument::parse();
let debug = Debug::init(&arg.debug)?;
let peers = peers::Peers::init(&arg.initial_peer)?;
let storage = Storage::init(&arg.storage, arg.clear)?;
let trackers = trackers::Trackers::init(&arg.torrent_tracker)?;
let torrent = arg.export_torrents.map(|p| Torrent::init(&p).unwrap());
let preload_regex = arg.preload_regex.map(|ref r| regex::Regex::new(r).unwrap());
let config = config::Config::parse();
let debug = Debug::init(&config.debug)?;
let peers = peers::Peers::init(&config.initial_peer)?;
let storage = Storage::init(&config.storage, config.clear)?;
let trackers = trackers::Trackers::init(&config.torrent_tracker)?;
let torrent = config.export_torrents.map(|p| Torrent::init(&p).unwrap());
let preload_regex = config
.preload_regex
.map(|ref r| regex::Regex::new(r).unwrap());
let session = librqbit::Session::new_with_opts(
storage.path(),
SessionOptions {
connect: Some(ConnectionOptions {
enable_tcp: arg.enable_tcp,
proxy_url: arg.proxy_url,
enable_tcp: config.enable_tcp,
proxy_url: config.proxy_url,
peer_opts: Some(PeerConnectionOptions {
connect_timeout: arg.peer_connect_timeout.map(Duration::from_secs),
read_write_timeout: arg.peer_read_write_timeout.map(Duration::from_secs),
keep_alive_interval: arg.peer_keep_alive_interval.map(Duration::from_secs),
connect_timeout: config.peer_connect_timeout.map(Duration::from_secs),
read_write_timeout: config.peer_read_write_timeout.map(Duration::from_secs),
keep_alive_interval: config.peer_keep_alive_interval.map(Duration::from_secs),
}),
}),
disable_upload: !arg.enable_upload,
disable_dht: !arg.enable_dht,
disable_upload: !config.enable_upload,
disable_dht: !config.enable_dht,
disable_dht_persistence: true,
persistence: None,
ratelimits: librqbit::limits::LimitsConfig {
upload_bps: arg.upload_limit.and_then(NonZero::new),
download_bps: arg.download_limit.and_then(NonZero::new),
upload_bps: config.upload_limit.and_then(NonZero::new),
download_bps: config.download_limit.and_then(NonZero::new),
},
trackers: trackers.clone(),
..SessionOptions::default()
@ -62,11 +64,11 @@ async fn main() -> Result<()> {
// begin
debug.info("Crawler started");
let mut index = Index::init(arg.index_capacity);
let mut index = Index::init(config.index_capacity);
loop {
debug.info("Index queue begin...");
index.refresh();
for source in &arg.infohash_file {
for source in &config.infohash_file {
debug.info(&format!("Index source `{source}`..."));
// grab latest info-hashes from this source
// * aquatic server may update the stats at this moment, handle result manually
@ -81,7 +83,7 @@ async fn main() -> Result<()> {
// run the crawler in single thread for performance reasons,
// use `timeout` argument option to skip the dead connections.
match time::timeout(
Duration::from_secs(arg.add_torrent_timeout),
Duration::from_secs(config.add_torrent_timeout),
session.add_torrent(
AddTorrent::from_url(magnet(&i, None)),
Some(AddTorrentOptions {
@ -92,7 +94,7 @@ async fn main() -> Result<()> {
list_only: preload_regex.is_none(),
// it is important to blacklist all files preload until initiation
only_files: Some(Vec::with_capacity(
arg.preload_max_filecount.unwrap_or_default(),
config.preload_max_filecount.unwrap_or_default(),
)),
// the destination folder to preload files match `only_files_regex`
// * e.g. images for audio albums
@ -108,10 +110,10 @@ async fn main() -> Result<()> {
Ok(AddTorrentResponse::Added(id, mt)) => {
let mut only_files_size = 0;
let mut only_files_keep = Vec::with_capacity(
arg.preload_max_filecount.unwrap_or_default(),
config.preload_max_filecount.unwrap_or_default(),
);
let mut only_files = HashSet::with_capacity(
arg.preload_max_filecount.unwrap_or_default(),
config.preload_max_filecount.unwrap_or_default(),
);
mt.wait_until_initialized().await?;
let name = mt.with_metadata(|m| {
@ -121,7 +123,7 @@ async fn main() -> Result<()> {
if regex.is_match(
info.relative_filename.to_str().unwrap(),
) {
if arg.preload_max_filesize.is_some_and(
if config.preload_max_filesize.is_some_and(
|limit| only_files_size + info.len > limit,
) {
debug.info(&format!(
@ -129,7 +131,7 @@ async fn main() -> Result<()> {
));
break;
}
if arg.preload_max_filecount.is_some_and(
if config.preload_max_filecount.is_some_and(
|limit| only_files.len() + 1 > limit,
) {
debug.info(&format!(
@ -183,14 +185,14 @@ async fn main() -> Result<()> {
Err(e) => debug.error(&format!("API issue for `{source}`: `{e}`")),
}
}
if let Some(ref export_rss) = arg.export_rss
if let Some(ref export_rss) = config.export_rss
&& index.is_changed()
{
let mut rss = Rss::new(
export_rss,
&arg.export_rss_title,
&arg.export_rss_link,
&arg.export_rss_description,
&config.export_rss_title,
&config.export_rss_link,
&config.export_rss_description,
Some(trackers.clone()),
)?;
for (k, v) in index.list() {
@ -203,15 +205,15 @@ async fn main() -> Result<()> {
}
rss.commit()?
}
if arg.preload_total_size.is_some_and(|s| index.nodes() > s) {
if config.preload_total_size.is_some_and(|s| index.nodes() > s) {
panic!("Preload content size {} bytes reached!", 0)
}
debug.info(&format!(
"Index completed, {} total, await {} seconds to continue...",
index.len(),
arg.sleep,
config.sleep,
));
std::thread::sleep(Duration::from_secs(arg.sleep));
std::thread::sleep(Duration::from_secs(config.sleep));
}
}