YGGverse/aquatic-crawler
1
0
Fork 0
mirror of https://github.com/YGGverse/aquatic-crawler.git synced 2026-03-31 09:05:33 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

collect unique ids from the multiple source(s) to handle

Browse source
This commit is contained in:
yggverse 2025-08-18 21:00:41 +03:00
parent 3643d456d0
commit 79643b84d2
1 changed files with 94 additions and 93 deletions
Download patch file Download diff file Expand all files Collapse all files

23
src/main.rs
View file

@ -66,6 +66,9 @@ async fn main() -> Result<()> {
loop { loop {
let time_queue = Local::now(); let time_queue = Local::now();
log::debug!("queue crawl begin at {time_queue}..."); log::debug!("queue crawl begin at {time_queue}...");
// build unique ID index from the multiple info-hash sources
let mut queue = HashSet::with_capacity(config.index_capacity);
for source in &config.infohash { for source in &config.infohash {
log::debug!("index source `{source}`..."); log::debug!("index source `{source}`...");
// grab latest info-hashes from this source // grab latest info-hashes from this source
@ -76,13 +79,18 @@ async fn main() -> Result<()> {
i i
} }
None => { None => {
// skip without panic
log::warn!( log::warn!(
"the feed `{source}` has an incomplete format (or is still updating); skip." "the feed `{source}` has an incomplete format (or is still updating); skip."
); );
continue; continue; // skip without panic
} }
} { } {
queue.insert(i);
}
}
// handle
for i in queue {
// convert to string once // convert to string once
let h = i.as_string(); let h = i.as_string();
if preload.contains_torrent(&h)? { if preload.contains_torrent(&h)? {
@ -116,10 +124,7 @@ async fn main() -> Result<()> {
list_only: false, list_only: false,
// the destination folder to preload files match `preload_regex` // the destination folder to preload files match `preload_regex`
// * e.g. images for audio albums // * e.g. images for audio albums
output_folder: preload output_folder: preload.tmp_dir(&h, true)?.to_str().map(|s| s.to_string()),
.tmp_dir(&h, true)?
.to_str()
.map(|s| s.to_string()),
..Default::default() ..Default::default()
}), }),
), ),
@ -194,10 +199,7 @@ async fn main() -> Result<()> {
log::debug!("torrent `{h}` preload completed."); log::debug!("torrent `{h}` preload completed.");
// persist torrent bytes and preloaded content, // persist torrent bytes and preloaded content,
// cleanup tmp (see rqbit#408) // cleanup tmp (see rqbit#408)
log::debug!( log::debug!("persist torrent `{h}` with `{}` files...", keep_files.len());
"persist torrent `{h}` with `{}` files...",
keep_files.len()
);
preload.commit(&h, bytes, Some(keep_files))?; preload.commit(&h, bytes, Some(keep_files))?;
session session
.delete(librqbit::api::TorrentIdOrHash::Id(id), false) .delete(librqbit::api::TorrentIdOrHash::Id(id), false)
@ -220,7 +222,6 @@ async fn main() -> Result<()> {
} }
} }
} }
}
log::info!( log::info!(
"queue completed at {time_queue} (time: {} / uptime: {} / banned: {}) await {} seconds to continue...", "queue completed at {time_queue} (time: {} / uptime: {} / banned: {}) await {} seconds to continue...",
Local::now() Local::now()