mirror of
https://github.com/YGGverse/aquatic-crawler.git
synced 2026-03-31 17:15:35 +00:00
prevent unexpected memory usage by filter values from unknown source
This commit is contained in:
parent
203c3041d2
commit
57b246a879
3 changed files with 42 additions and 13 deletions
16
src/index.rs
16
src/index.rs
|
|
@ -1,11 +1,7 @@
|
|||
use chrono::{DateTime, Utc};
|
||||
use std::collections::HashMap;
|
||||
mod value;
|
||||
|
||||
pub struct Value {
|
||||
pub time: DateTime<Utc>,
|
||||
pub node: u64,
|
||||
pub name: Option<String>,
|
||||
}
|
||||
use std::collections::HashMap;
|
||||
use value::Value;
|
||||
|
||||
/// Collect processed info hashes to skip on the next iterations (for this session)
|
||||
/// * also contains optional meta info to export index as RSS or any other format
|
||||
|
|
@ -52,11 +48,7 @@ impl Index {
|
|||
.index
|
||||
.insert(
|
||||
infohash,
|
||||
Value {
|
||||
time: Utc::now(),
|
||||
node,
|
||||
name: if self.has_name { name } else { None },
|
||||
},
|
||||
Value::new(node, if self.has_name { name } else { None }),
|
||||
)
|
||||
.is_none()
|
||||
{
|
||||
|
|
|
|||
37
src/index/value.rs
Normal file
37
src/index/value.rs
Normal file
|
|
@ -0,0 +1,37 @@
|
|||
use chrono::{DateTime, Utc};
|
||||
|
||||
const NAME_MAX_LEN: usize = 125; // + 3 bytes for `...` offset @TODO optional
|
||||
|
||||
/// The `Index` value
|
||||
pub struct Value {
|
||||
pub time: DateTime<Utc>,
|
||||
pub node: u64,
|
||||
/// Isolate by applying internal filter on value set
|
||||
name: Option<String>,
|
||||
}
|
||||
|
||||
impl Value {
|
||||
/// Create new `Self` with current timestamp
|
||||
pub fn new(node: u64, name: Option<String>) -> Self {
|
||||
Self {
|
||||
time: Utc::now(),
|
||||
node,
|
||||
name: filter_name(name),
|
||||
}
|
||||
}
|
||||
/// Get reference to the safely constructed `name` member
|
||||
pub fn name(&self) -> Option<&String> {
|
||||
self.name.as_ref()
|
||||
}
|
||||
}
|
||||
|
||||
/// Prevent unexpected memory usage on store values from unknown source
|
||||
fn filter_name(value: Option<String>) -> Option<String> {
|
||||
value.map(|v| {
|
||||
if v.len() > NAME_MAX_LEN {
|
||||
format!("{}...", &v[..NAME_MAX_LEN])
|
||||
} else {
|
||||
v
|
||||
}
|
||||
})
|
||||
}
|
||||
|
|
@ -198,7 +198,7 @@ async fn main() -> Result<()> {
|
|||
for (k, v) in index.list() {
|
||||
rss.push(
|
||||
k,
|
||||
v.name.as_ref().unwrap_or(k),
|
||||
v.name().unwrap_or(k),
|
||||
None, // @TODO
|
||||
Some(&v.time.to_rfc2822()),
|
||||
)?
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue