YGGverse/aquatic-crawler
1
0
Fork 0
mirror of https://github.com/YGGverse/aquatic-crawler.git synced 2026-03-31 17:15:35 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix name sanitizing model

Browse source
This commit is contained in:
yggverse 2025-07-09 18:17:48 +03:00
parent 058bc6a07c
commit 5e2dafb898
1 changed files with 8 additions and 20 deletions
Download patch file Download diff file Expand all files Collapse all files

28
src/index/value.rs
View file

@ -1,4 +1,5 @@
use chrono::{DateTime, Utc}; use chrono::{DateTime, Utc};
use voca_rs::Voca;
/// The `Index` value /// The `Index` value
pub struct Value { pub struct Value {
@ -41,32 +42,19 @@ impl Value {
} }
fn filter_name(value: Option<String>) -> Option<String> { fn filter_name(value: Option<String>) -> Option<String> {
value.map(crop) value.map(filter)
} }
fn filter_list(value: Option<Vec<(String, u64)>>) -> Option<Vec<(String, u64)>> { fn filter_list(value: Option<Vec<(String, u64)>>) -> Option<Vec<(String, u64)>> {
value.map(|f| { value.map(|f| f.into_iter().map(|(n, l)| (filter(n), l)).collect())
f.into_iter()
.map(|(n, l)| (crop(sanitize(&n)), l))
.collect()
})
} }
/// Crop long values (prevents unexpected memory pool usage) /// Crop long values (prevents unexpected memory pool usage)
fn crop(value: String) -> String { fn filter(value: String) -> String {
const C: usize = 125; // + 3 for `...` offset, 128 chars max @TODO optional const C: usize = 125; // + 3 for `...` offset, 128 chars max @TODO optional
if value.chars().count() > C { let s = value._strip_bom()._strip_tags();
format!( if s.chars().count() > C {
"{}...", return format!("{}...", s.chars().take(C).collect::<String>());
sanitize(&value.chars().take(C).collect::<String>())
)
} else {
value
} }
} s
/// Strip tags & bom chars from string
fn sanitize(value: &str) -> String {
use voca_rs::strip::*;
strip_tags(&strip_bom(value))
} }