YGGverse/aquatic-crawler
1
0
Fork 0
mirror of https://github.com/YGGverse/aquatic-crawler.git synced 2026-04-02 10:05:30 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

calculate regex once

Browse source
This commit is contained in:
yggverse 2025-06-15 21:45:57 +03:00
parent 2df00564d6
commit ecdfe2d134
2 changed files with 8 additions and 8 deletions
Download patch file Download diff file Expand all files Collapse all files

9
src/main.rs
View file

@ -25,6 +25,7 @@ async fn main() -> Result<()> {
let peers = peers::Peers::init(&arg.initial_peer)?; let peers = peers::Peers::init(&arg.initial_peer)?;
let storage = Storage::init(&arg.storage, arg.clear)?; let storage = Storage::init(&arg.storage, arg.clear)?;
let trackers = trackers::Trackers::init(&arg.torrent_tracker)?; let trackers = trackers::Trackers::init(&arg.torrent_tracker)?;
let preload_regex = arg.preload_regex.map(|ref r| regex::Regex::new(r).unwrap());
let session = librqbit::Session::new_with_opts( let session = librqbit::Session::new_with_opts(
storage.path(), storage.path(),
SessionOptions { SessionOptions {
@ -80,11 +81,11 @@ async fn main() -> Result<()> {
overwrite: true, overwrite: true,
disable_trackers: trackers.is_empty(), disable_trackers: trackers.is_empty(),
initial_peers: peers.initial_peers(), initial_peers: peers.initial_peers(),
list_only: arg.preload_regex.is_none(), list_only: preload_regex.is_none(),
// the destination folder to preload files match `only_files_regex` // the destination folder to preload files match `only_files_regex`
// * e.g. images for audio albums // * e.g. images for audio albums
output_folder: storage.output_folder(&i, true).ok(), output_folder: storage.output_folder(&i, true).ok(),
only_files_regex: arg.preload_regex.clone(), only_files_regex: preload_regex.as_ref().map(|r| r.to_string()),
..Default::default() ..Default::default()
}), }),
), ),
@ -125,8 +126,8 @@ async fn main() -> Result<()> {
) )
.await?; .await?;
// cleanup irrelevant files (see rqbit#408) // cleanup irrelevant files (see rqbit#408)
if let Some(ref r) = arg.preload_regex { if let Some(r) = preload_regex.as_ref() {
storage.cleanup(&i, r)?; storage.cleanup(&i, Some(r))?;
} }
// ignore on the next crawl iterations for this session // ignore on the next crawl iterations for this session
index.insert(i); index.insert(i);

7
src/storage.rs
View file

@ -54,15 +54,14 @@ impl Storage {
} }
/// Recursively remove all files under the `infohash` location /// Recursively remove all files under the `infohash` location
/// that do not match the `skip_filename_pattern` (see rqbit#408) /// that do not match the `skip_filename` (see rqbit#408)
pub fn cleanup(&self, infohash: &str, skip_filename_pattern: &str) -> Result<()> { pub fn cleanup(&self, infohash: &str, skip_filename: Option<&regex::Regex>) -> Result<()> {
let r = regex::Regex::new(skip_filename_pattern)?;
for e in walkdir::WalkDir::new(self.output_folder(infohash, false)?) for e in walkdir::WalkDir::new(self.output_folder(infohash, false)?)
.into_iter() .into_iter()
.filter_map(Result::ok) .filter_map(Result::ok)
{ {
let p = e.path(); let p = e.path();
if p.is_file() && !r.is_match(p.to_str().unwrap()) { if p.is_file() && skip_filename.is_none_or(|r| !r.is_match(p.to_str().unwrap())) {
fs::remove_file(p)?; fs::remove_file(p)?;
} }
} }