YGGverse/aquatic-crawler
1
0
Fork 0
mirror of https://github.com/YGGverse/aquatic-crawler.git synced 2026-03-31 09:05:33 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

separate ban timeout config option, update ban handlers

Browse source
This commit is contained in:
yggverse 2025-08-13 16:53:09 +03:00
parent 7568a9e77d
commit 9a2f184a19
2 changed files with 16 additions and 14 deletions
Download patch file Download diff file Expand all files Collapse all files

14
src/config.rs
View file

@ -94,18 +94,22 @@ pub struct Config {
#[arg(long, default_value_t = 1000)]
pub index_capacity: usize,
/// Max time to handle each torrent
/// Max time in seconds to add new torrent
#[arg(long, default_value_t = 60)]
pub add_torrent_timeout: u64,
/// Ban time in seconds on torrent add failure
#[arg(long, default_value_t = 3600)]
pub add_torrent_ban: u64,
/// Ban time in seconds on torrent resolve failure
#[arg(long, default_value_t = 3600)]
pub resolve_torrent_ban: u64,
/// Crawl loop delay in seconds
#[arg(long, default_value_t = 60)]
pub sleep: u64,
/// Ban unresolvable info-hashes for `n` seconds
#[arg(long, default_value_t = 3600)]
pub ban: u64,
/// Limit download speed (b/s)
#[arg(long)]
pub download_limit: Option<u32>, // * reminder: upload feature is not planed by the crawler impl

16
src/main.rs
View file

@ -71,7 +71,7 @@ async fn main() -> Result<()> {
let time_queue = Local::now();
log::debug!("queue crawl begin at {time_queue}...");
ban.retain(|i, &mut t| {
let is_expired = t > time_queue - Duration::from_secs(config.ban);
let is_expired = t >= time_queue;
if is_expired {
log::debug!(
"remove ban for `{}` by the timeout expiration ({t})",
@ -221,19 +221,17 @@ async fn main() -> Result<()> {
}
Ok(_) => panic!(),
Err(e) => {
log::debug!(
"failed to resolve torrent `{h}`: `{e}`, ban for {} seconds.",
config.ban
);
assert!(ban.insert(i, Local::now()).is_none());
let t = Local::now() + Duration::from_secs(config.add_torrent_ban);
log::debug!("failed to resolve torrent `{h}`: `{e}`, ban until {t}.");
assert!(ban.insert(i, t).is_none());
}
},
Err(e) => {
let t = Local::now() + Duration::from_secs(config.add_torrent_ban);
log::debug!(
"skip awaiting the completion of adding torrent `{h}` data (`{e}`), ban for {} seconds.",
config.ban
"skip awaiting the completion of adding torrent `{h}` data (`{e}`), ban until {t}."
);
assert!(ban.insert(i, Local::now()).is_none());
assert!(ban.insert(i, t).is_none());
}
}
}