YGGverse/aquatic-crawler
1
0
Fork 0
mirror of https://github.com/YGGverse/aquatic-crawler.git synced 2026-03-31 09:05:33 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

separate ban timeout config option, update ban handlers

Browse source
This commit is contained in:
yggverse 2025-08-13 16:53:09 +03:00
parent 7568a9e77d
commit 9a2f184a19
2 changed files with 16 additions and 14 deletions
Download patch file Download diff file Expand all files Collapse all files

14
src/config.rs
View file

@ -94,18 +94,22 @@ pub struct Config {
#[arg(long, default_value_t = 1000)] #[arg(long, default_value_t = 1000)]
pub index_capacity: usize, pub index_capacity: usize,
/// Max time to handle each torrent /// Max time in seconds to add new torrent
#[arg(long, default_value_t = 60)] #[arg(long, default_value_t = 60)]
pub add_torrent_timeout: u64, pub add_torrent_timeout: u64,
/// Ban time in seconds on torrent add failure
#[arg(long, default_value_t = 3600)]
pub add_torrent_ban: u64,
/// Ban time in seconds on torrent resolve failure
#[arg(long, default_value_t = 3600)]
pub resolve_torrent_ban: u64,
/// Crawl loop delay in seconds /// Crawl loop delay in seconds
#[arg(long, default_value_t = 60)] #[arg(long, default_value_t = 60)]
pub sleep: u64, pub sleep: u64,
/// Ban unresolvable info-hashes for `n` seconds
#[arg(long, default_value_t = 3600)]
pub ban: u64,
/// Limit download speed (b/s) /// Limit download speed (b/s)
#[arg(long)] #[arg(long)]
pub download_limit: Option<u32>, // * reminder: upload feature is not planed by the crawler impl pub download_limit: Option<u32>, // * reminder: upload feature is not planed by the crawler impl

16
src/main.rs
View file

@ -71,7 +71,7 @@ async fn main() -> Result<()> {
let time_queue = Local::now(); let time_queue = Local::now();
log::debug!("queue crawl begin at {time_queue}..."); log::debug!("queue crawl begin at {time_queue}...");
ban.retain(|i, &mut t| { ban.retain(|i, &mut t| {
let is_expired = t > time_queue - Duration::from_secs(config.ban); let is_expired = t >= time_queue;
if is_expired { if is_expired {
log::debug!( log::debug!(
"remove ban for `{}` by the timeout expiration ({t})", "remove ban for `{}` by the timeout expiration ({t})",
@ -221,19 +221,17 @@ async fn main() -> Result<()> {
} }
Ok(_) => panic!(), Ok(_) => panic!(),
Err(e) => { Err(e) => {
log::debug!( let t = Local::now() + Duration::from_secs(config.add_torrent_ban);
"failed to resolve torrent `{h}`: `{e}`, ban for {} seconds.", log::debug!("failed to resolve torrent `{h}`: `{e}`, ban until {t}.");
config.ban assert!(ban.insert(i, t).is_none());
);
assert!(ban.insert(i, Local::now()).is_none());
} }
}, },
Err(e) => { Err(e) => {
let t = Local::now() + Duration::from_secs(config.add_torrent_ban);
log::debug!( log::debug!(
"skip awaiting the completion of adding torrent `{h}` data (`{e}`), ban for {} seconds.", "skip awaiting the completion of adding torrent `{h}` data (`{e}`), ban until {t}."
config.ban
); );
assert!(ban.insert(i, Local::now()).is_none()); assert!(ban.insert(i, t).is_none());
} }
} }
} }