handle list and cache duplicate events; log parsed totals

This commit is contained in:
postscriptum 2026-03-23 07:59:18 +02:00
parent cb002bfb92
commit 24cbc3012e

View file

@ -17,15 +17,17 @@ pub struct List {
impl List { impl List {
pub async fn from_opt(list: &Vec<String>, cache: Option<PathBuf>) -> Result<Self> { pub async fn from_opt(list: &Vec<String>, cache: Option<PathBuf>) -> Result<Self> {
fn handle(this: &mut HashSet<Item>, line: &str) { fn handle(this: &mut HashSet<Item>, line: &str) -> Option<bool> {
if line.starts_with("/") || line.starts_with("#") || line.is_empty() { if line.starts_with("/") || line.starts_with("#") || line.is_empty() {
return; return None;
}
if !this.insert(Item::from_line(line)) {
warn!("Duplicated list record: `{line}`")
} }
Some(this.insert(Item::from_line(line)))
} }
let mut index = HashSet::new(); let mut index = HashSet::new();
let mut list_rules_total = 0;
for i in list { for i in list {
for line in if i.contains("://") { for line in if i.contains("://") {
let response = reqwest::get(i).await?; let response = reqwest::get(i).await?;
@ -41,18 +43,31 @@ impl List {
} }
.lines() .lines()
{ {
handle(&mut index, line) if handle(&mut index, line).is_some_and(|status| !status) {
warn!("List `{i}` contains duplicated entry: `{line}`")
}
list_rules_total += 1
} }
} }
let cache = Cache::from_path(cache).await?; let cache = Cache::from_path(cache).await?;
if let Some(data) = cache.read().await? {
for line in data.lines() {
handle(&mut index, line)
}
}
info!("Total list entries parsed: {}", index.len()); let cache_rules_total = cache.read().await?.map(|data| {
let mut t = 0;
for line in data.lines() {
if handle(&mut index, line).is_some_and(|status| !status) {
warn!("Cache file contains duplicated entry: `{line}`")
}
t += 1
}
t
});
let len = index.len();
info!(
"Total rules parsed: {len} (lists: {list_rules_total} / cache: {cache_rules_total:?})",
);
Ok(Self { Ok(Self {
index: RwLock::new(index), index: RwLock::new(index),
cache, cache,