mirror of
https://github.com/YGGverse/aquatic-crawler.git
synced 2026-03-31 17:15:35 +00:00
implement ban Item, handle update results outside
This commit is contained in:
parent
ea0d2b4c67
commit
31087f7936
2 changed files with 21 additions and 7 deletions
20
src/ban.rs
20
src/ban.rs
|
|
@ -2,6 +2,11 @@ use chrono::{DateTime, Local};
|
||||||
use librqbit::dht::Id20;
|
use librqbit::dht::Id20;
|
||||||
use std::{collections::HashMap, time::Duration};
|
use std::{collections::HashMap, time::Duration};
|
||||||
|
|
||||||
|
pub struct Item {
|
||||||
|
pub expires: DateTime<Local>,
|
||||||
|
pub info_hash: String,
|
||||||
|
}
|
||||||
|
|
||||||
pub struct Ban {
|
pub struct Ban {
|
||||||
index: HashMap<Id20, DateTime<Local>>,
|
index: HashMap<Id20, DateTime<Local>>,
|
||||||
timeout: u64,
|
timeout: u64,
|
||||||
|
|
@ -23,20 +28,23 @@ impl Ban {
|
||||||
self.index.len()
|
self.index.len()
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn update(&mut self, time: DateTime<Local>) {
|
pub fn update(&mut self, time: DateTime<Local>) -> Vec<Item> {
|
||||||
|
let mut b = Vec::with_capacity(self.index.len());
|
||||||
self.index.retain(|i, &mut expires| {
|
self.index.retain(|i, &mut expires| {
|
||||||
if time > expires {
|
if time > expires {
|
||||||
log::debug!(
|
b.push(Item {
|
||||||
"remove ban for `{}` by the timeout expiration ({expires})",
|
expires,
|
||||||
i.as_string()
|
info_hash: i.as_string(),
|
||||||
);
|
});
|
||||||
false
|
false
|
||||||
} else {
|
} else {
|
||||||
true
|
true
|
||||||
}
|
}
|
||||||
})
|
});
|
||||||
|
b
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// * returns expiration time
|
||||||
pub fn add(&mut self, key: Id20) -> DateTime<Local> {
|
pub fn add(&mut self, key: Id20) -> DateTime<Local> {
|
||||||
let t = Local::now() + Duration::from_secs(self.index.len() as u64 * self.timeout);
|
let t = Local::now() + Duration::from_secs(self.index.len() as u64 * self.timeout);
|
||||||
assert!(self.index.insert(key, t).is_none());
|
assert!(self.index.insert(key, t).is_none());
|
||||||
|
|
|
||||||
|
|
@ -61,7 +61,13 @@ async fn main() -> Result<()> {
|
||||||
loop {
|
loop {
|
||||||
let time_queue = Local::now();
|
let time_queue = Local::now();
|
||||||
log::debug!("queue crawl begin at {time_queue}...");
|
log::debug!("queue crawl begin at {time_queue}...");
|
||||||
ban.update(time_queue);
|
for r in ban.update(time_queue) {
|
||||||
|
log::debug!(
|
||||||
|
"remove the ban for `{}` as it has expired on {}",
|
||||||
|
r.info_hash,
|
||||||
|
r.expires
|
||||||
|
)
|
||||||
|
}
|
||||||
for source in &config.infohash {
|
for source in &config.infohash {
|
||||||
log::debug!("index source `{source}`...");
|
log::debug!("index source `{source}`...");
|
||||||
// grab latest info-hashes from this source
|
// grab latest info-hashes from this source
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue