YGGverse/aquatic-crawler
1
0
Fork 0
mirror of https://github.com/YGGverse/aquatic-crawler.git synced 2026-03-31 17:15:35 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

calculate regex once

Browse source
This commit is contained in:
yggverse 2025-06-15 21:45:57 +03:00
parent 2df00564d6
commit ecdfe2d134
2 changed files with 8 additions and 8 deletions
Download patch file Download diff file Expand all files Collapse all files

9
src/main.rs
View file

@ -25,6 +25,7 @@ async fn main() -> Result<()> {
let peers = peers::Peers::init(&arg.initial_peer)?;
let storage = Storage::init(&arg.storage, arg.clear)?;
let trackers = trackers::Trackers::init(&arg.torrent_tracker)?;
let preload_regex = arg.preload_regex.map(|ref r| regex::Regex::new(r).unwrap());
let session = librqbit::Session::new_with_opts(
storage.path(),
SessionOptions {
@ -80,11 +81,11 @@ async fn main() -> Result<()> {
overwrite: true,
disable_trackers: trackers.is_empty(),
initial_peers: peers.initial_peers(),
list_only: arg.preload_regex.is_none(),
list_only: preload_regex.is_none(),
// the destination folder to preload files match `only_files_regex`
// * e.g. images for audio albums
output_folder: storage.output_folder(&i, true).ok(),
only_files_regex: arg.preload_regex.clone(),
only_files_regex: preload_regex.as_ref().map(|r| r.to_string()),
..Default::default()
}),
),
@ -125,8 +126,8 @@ async fn main() -> Result<()> {
)
.await?;
// cleanup irrelevant files (see rqbit#408)
if let Some(ref r) = arg.preload_regex {
storage.cleanup(&i, r)?;
if let Some(r) = preload_regex.as_ref() {
storage.cleanup(&i, Some(r))?;
}
// ignore on the next crawl iterations for this session
index.insert(i);

7
src/storage.rs
View file

@ -54,15 +54,14 @@ impl Storage {
}
/// Recursively remove all files under the `infohash` location
/// that do not match the `skip_filename_pattern` (see rqbit#408)
pub fn cleanup(&self, infohash: &str, skip_filename_pattern: &str) -> Result<()> {
let r = regex::Regex::new(skip_filename_pattern)?;
/// that do not match the `skip_filename` (see rqbit#408)
pub fn cleanup(&self, infohash: &str, skip_filename: Option<&regex::Regex>) -> Result<()> {
for e in walkdir::WalkDir::new(self.output_folder(infohash, false)?)
.into_iter()
.filter_map(Result::ok)
{
let p = e.path();
if p.is_file() && !r.is_match(p.to_str().unwrap()) {
if p.is_file() && skip_filename.is_none_or(|r| !r.is_match(p.to_str().unwrap())) {
fs::remove_file(p)?;
}
}