mirror of
https://github.com/YGGverse/aquatic-crawler.git
synced 2026-04-02 18:15:31 +00:00
prevent unexpected memory usage by filter values from unknown source
This commit is contained in:
parent
203c3041d2
commit
57b246a879
3 changed files with 42 additions and 13 deletions
16
src/index.rs
16
src/index.rs
|
|
@ -1,11 +1,7 @@
|
||||||
use chrono::{DateTime, Utc};
|
mod value;
|
||||||
use std::collections::HashMap;
|
|
||||||
|
|
||||||
pub struct Value {
|
use std::collections::HashMap;
|
||||||
pub time: DateTime<Utc>,
|
use value::Value;
|
||||||
pub node: u64,
|
|
||||||
pub name: Option<String>,
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Collect processed info hashes to skip on the next iterations (for this session)
|
/// Collect processed info hashes to skip on the next iterations (for this session)
|
||||||
/// * also contains optional meta info to export index as RSS or any other format
|
/// * also contains optional meta info to export index as RSS or any other format
|
||||||
|
|
@ -52,11 +48,7 @@ impl Index {
|
||||||
.index
|
.index
|
||||||
.insert(
|
.insert(
|
||||||
infohash,
|
infohash,
|
||||||
Value {
|
Value::new(node, if self.has_name { name } else { None }),
|
||||||
time: Utc::now(),
|
|
||||||
node,
|
|
||||||
name: if self.has_name { name } else { None },
|
|
||||||
},
|
|
||||||
)
|
)
|
||||||
.is_none()
|
.is_none()
|
||||||
{
|
{
|
||||||
|
|
|
||||||
37
src/index/value.rs
Normal file
37
src/index/value.rs
Normal file
|
|
@ -0,0 +1,37 @@
|
||||||
|
use chrono::{DateTime, Utc};
|
||||||
|
|
||||||
|
const NAME_MAX_LEN: usize = 125; // + 3 bytes for `...` offset @TODO optional
|
||||||
|
|
||||||
|
/// The `Index` value
|
||||||
|
pub struct Value {
|
||||||
|
pub time: DateTime<Utc>,
|
||||||
|
pub node: u64,
|
||||||
|
/// Isolate by applying internal filter on value set
|
||||||
|
name: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Value {
|
||||||
|
/// Create new `Self` with current timestamp
|
||||||
|
pub fn new(node: u64, name: Option<String>) -> Self {
|
||||||
|
Self {
|
||||||
|
time: Utc::now(),
|
||||||
|
node,
|
||||||
|
name: filter_name(name),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
/// Get reference to the safely constructed `name` member
|
||||||
|
pub fn name(&self) -> Option<&String> {
|
||||||
|
self.name.as_ref()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Prevent unexpected memory usage on store values from unknown source
|
||||||
|
fn filter_name(value: Option<String>) -> Option<String> {
|
||||||
|
value.map(|v| {
|
||||||
|
if v.len() > NAME_MAX_LEN {
|
||||||
|
format!("{}...", &v[..NAME_MAX_LEN])
|
||||||
|
} else {
|
||||||
|
v
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
@ -198,7 +198,7 @@ async fn main() -> Result<()> {
|
||||||
for (k, v) in index.list() {
|
for (k, v) in index.list() {
|
||||||
rss.push(
|
rss.push(
|
||||||
k,
|
k,
|
||||||
v.name.as_ref().unwrap_or(k),
|
v.name().unwrap_or(k),
|
||||||
None, // @TODO
|
None, // @TODO
|
||||||
Some(&v.time.to_rfc2822()),
|
Some(&v.time.to_rfc2822()),
|
||||||
)?
|
)?
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue