YGGverse/aquatic-crawler
1
0
Fork 0
mirror of https://github.com/YGGverse/aquatic-crawler.git synced 2026-03-31 09:05:33 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

use Id20 as the ban key

Browse source
This commit is contained in:
yggverse 2025-08-13 15:41:55 +03:00
parent c44acbb50c
commit b2a0a3e5ad
1 changed files with 10 additions and 6 deletions
Download patch file Download diff file Expand all files Collapse all files

16
src/main.rs
View file

@ -64,7 +64,8 @@ async fn main() -> Result<()> {
}, },
) )
.await?; .await?;
let mut ban: HashMap<String, DateTime<Utc>> = HashMap::with_capacity(config.index_capacity); let mut ban: HashMap<librqbit::dht::Id20, DateTime<Utc>> =
HashMap::with_capacity(config.index_capacity);
log::info!("crawler started at {time_init}"); log::info!("crawler started at {time_init}");
loop { loop {
let time_queue = Local::now(); let time_queue = Local::now();
@ -72,7 +73,10 @@ async fn main() -> Result<()> {
ban.retain(|k, &mut v| { ban.retain(|k, &mut v| {
let is_expired = v > Utc::now() - Duration::from_secs(config.ban); let is_expired = v > Utc::now() - Duration::from_secs(config.ban);
if is_expired { if is_expired {
log::debug!("remove `{k}` from the ban list (timeout expired)") log::debug!(
"remove `{}` from the ban list (timeout expired)",
k.as_string()
)
} }
is_expired is_expired
}); });
@ -99,7 +103,7 @@ async fn main() -> Result<()> {
log::debug!("torrent `{h}` exists and already indexed, skip."); log::debug!("torrent `{h}` exists and already indexed, skip.");
continue; continue;
} }
if ban.contains_key(&h) { if ban.contains_key(&i) {
log::debug!("torrent `{h}` is temporary banned, skip for this queue."); log::debug!("torrent `{h}` is temporary banned, skip for this queue.");
continue; continue;
} }
@ -197,7 +201,7 @@ async fn main() -> Result<()> {
log::debug!( log::debug!(
"skip awaiting the completion of preload `{h}` data (`{e}`)" "skip awaiting the completion of preload `{h}` data (`{e}`)"
); );
ban.insert(h, Utc::now()); ban.insert(i, Utc::now());
session session
.delete(librqbit::api::TorrentIdOrHash::Id(id), false) .delete(librqbit::api::TorrentIdOrHash::Id(id), false)
.await?; // * do not collect billions of slow torrents in the session pool .await?; // * do not collect billions of slow torrents in the session pool
@ -219,14 +223,14 @@ async fn main() -> Result<()> {
Ok(_) => panic!(), Ok(_) => panic!(),
Err(e) => { Err(e) => {
log::debug!("failed to resolve torrent `{h}`: `{e}`."); log::debug!("failed to resolve torrent `{h}`: `{e}`.");
assert!(ban.insert(h, Utc::now()).is_none()); assert!(ban.insert(i, Utc::now()).is_none());
} }
}, },
Err(e) => { Err(e) => {
log::debug!( log::debug!(
"skip awaiting the completion of adding torrent `{h}` data (`{e}`)" "skip awaiting the completion of adding torrent `{h}` data (`{e}`)"
); );
assert!(ban.insert(h, Utc::now()).is_none()); assert!(ban.insert(i, Utc::now()).is_none());
} }
} }
} }