mirror of
https://github.com/YGGverse/aquatic-crawler.git
synced 2026-03-31 17:15:35 +00:00
store index values in memory only in use by the argument options
This commit is contained in:
parent
39e717ec04
commit
e5e4dd0ae6
2 changed files with 7 additions and 3 deletions
|
|
@ -14,13 +14,17 @@ pub struct Index {
|
||||||
/// Track index changes to prevent extra disk write operations (safe SSD life)
|
/// Track index changes to prevent extra disk write operations (safe SSD life)
|
||||||
/// * useful in the static RSS feed generation case, if enabled.
|
/// * useful in the static RSS feed generation case, if enabled.
|
||||||
is_changed: bool,
|
is_changed: bool,
|
||||||
|
/// Store index value in memory only on it is in use
|
||||||
|
/// by the externally defined argument options.
|
||||||
|
has_name: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Index {
|
impl Index {
|
||||||
pub fn init(capacity: usize) -> Self {
|
pub fn init(capacity: usize, has_name: bool) -> Self {
|
||||||
Self {
|
Self {
|
||||||
index: HashMap::with_capacity(capacity),
|
index: HashMap::with_capacity(capacity),
|
||||||
is_changed: false,
|
is_changed: false,
|
||||||
|
has_name,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -52,7 +56,7 @@ impl Index {
|
||||||
Value {
|
Value {
|
||||||
time: Utc::now(),
|
time: Utc::now(),
|
||||||
node,
|
node,
|
||||||
name,
|
name: if self.has_name { name } else { None },
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
.is_none()
|
.is_none()
|
||||||
|
|
|
||||||
|
|
@ -64,7 +64,7 @@ async fn main() -> Result<()> {
|
||||||
|
|
||||||
// begin
|
// begin
|
||||||
debug.info("Crawler started");
|
debug.info("Crawler started");
|
||||||
let mut index = Index::init(config.index_capacity);
|
let mut index = Index::init(config.index_capacity, config.export_rss.is_some());
|
||||||
loop {
|
loop {
|
||||||
debug.info("Index queue begin...");
|
debug.info("Index queue begin...");
|
||||||
index.refresh();
|
index.refresh();
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue