mirror of
https://github.com/YGGverse/aquatic-crawler.git
synced 2026-03-31 17:15:35 +00:00
move regex member to preload struct
This commit is contained in:
parent
ee78171a12
commit
bd49ae8c6f
2 changed files with 27 additions and 20 deletions
16
src/main.rs
16
src/main.rs
|
|
@ -15,6 +15,7 @@ use preload::Preload;
|
||||||
use rss::Rss;
|
use rss::Rss;
|
||||||
use std::{collections::HashSet, num::NonZero, path::PathBuf, time::Duration};
|
use std::{collections::HashSet, num::NonZero, path::PathBuf, time::Duration};
|
||||||
use torrent::Torrent;
|
use torrent::Torrent;
|
||||||
|
use trackers::Trackers;
|
||||||
use url::Url;
|
use url::Url;
|
||||||
|
|
||||||
#[tokio::main]
|
#[tokio::main]
|
||||||
|
|
@ -32,12 +33,9 @@ async fn main() -> Result<()> {
|
||||||
let peers = peers::Peers::init(&config.initial_peer)?;
|
let peers = peers::Peers::init(&config.initial_peer)?;
|
||||||
let preload = config
|
let preload = config
|
||||||
.preload
|
.preload
|
||||||
.map(|ref p| Preload::init(p, config.clear).unwrap());
|
.map(|ref p| Preload::init(p, config.preload_regex.as_deref(), config.clear).unwrap());
|
||||||
let trackers = trackers::Trackers::init(&config.tracker)?;
|
let trackers = Trackers::init(&config.tracker)?;
|
||||||
let torrent = config.export_torrents.map(|p| Torrent::init(&p).unwrap());
|
let torrent = config.export_torrents.map(|p| Torrent::init(&p).unwrap());
|
||||||
let preload_regex = config
|
|
||||||
.preload_regex
|
|
||||||
.map(|ref r| regex::Regex::new(r).unwrap());
|
|
||||||
let session = librqbit::Session::new_with_opts(
|
let session = librqbit::Session::new_with_opts(
|
||||||
match preload {
|
match preload {
|
||||||
Some(ref p) => p.path(),
|
Some(ref p) => p.path(),
|
||||||
|
|
@ -99,7 +97,7 @@ async fn main() -> Result<()> {
|
||||||
overwrite: true,
|
overwrite: true,
|
||||||
disable_trackers: trackers.is_empty(),
|
disable_trackers: trackers.is_empty(),
|
||||||
initial_peers: peers.initial_peers(),
|
initial_peers: peers.initial_peers(),
|
||||||
list_only: preload_regex.is_none(),
|
list_only: preload.as_ref().is_none_or(|p| p.regex.is_none()),
|
||||||
// it is important to blacklist all files preload until initiation
|
// it is important to blacklist all files preload until initiation
|
||||||
only_files: Some(Vec::with_capacity(
|
only_files: Some(Vec::with_capacity(
|
||||||
config.preload_max_filecount.unwrap_or_default(),
|
config.preload_max_filecount.unwrap_or_default(),
|
||||||
|
|
@ -128,11 +126,9 @@ async fn main() -> Result<()> {
|
||||||
mt.wait_until_initialized().await?;
|
mt.wait_until_initialized().await?;
|
||||||
let name = mt.with_metadata(|m| {
|
let name = mt.with_metadata(|m| {
|
||||||
// init preload files list
|
// init preload files list
|
||||||
if let Some(ref regex) = preload_regex {
|
if let Some(ref p) = preload {
|
||||||
for (id, info) in m.file_infos.iter().enumerate() {
|
for (id, info) in m.file_infos.iter().enumerate() {
|
||||||
if regex.is_match(
|
if p.matches(info.relative_filename.to_str().unwrap()) {
|
||||||
info.relative_filename.to_str().unwrap(),
|
|
||||||
) {
|
|
||||||
if config.preload_max_filesize.is_some_and(
|
if config.preload_max_filesize.is_some_and(
|
||||||
|limit| only_files_size + info.len > limit,
|
|limit| only_files_size + info.len > limit,
|
||||||
) {
|
) {
|
||||||
|
|
|
||||||
|
|
@ -1,17 +1,21 @@
|
||||||
use anyhow::{Result, bail};
|
use anyhow::{Result, bail};
|
||||||
|
use regex::Regex;
|
||||||
use std::{fs, path::PathBuf, str::FromStr};
|
use std::{fs, path::PathBuf, str::FromStr};
|
||||||
|
|
||||||
pub struct Preload(PathBuf);
|
pub struct Preload {
|
||||||
|
path: PathBuf,
|
||||||
|
pub regex: Option<Regex>,
|
||||||
|
}
|
||||||
|
|
||||||
impl Preload {
|
impl Preload {
|
||||||
pub fn init(directory: &str, clear: bool) -> Result<Self> {
|
pub fn init(directory: &str, regex: Option<&str>, clear: bool) -> Result<Self> {
|
||||||
let p = PathBuf::from_str(directory)?;
|
let path = PathBuf::from_str(directory)?;
|
||||||
if let Ok(t) = fs::metadata(&p) {
|
if let Ok(t) = fs::metadata(&path) {
|
||||||
if t.is_file() {
|
if t.is_file() {
|
||||||
bail!("Storage destination is not directory!");
|
bail!("Storage destination is not directory!");
|
||||||
}
|
}
|
||||||
if t.is_dir() && clear {
|
if t.is_dir() && clear {
|
||||||
for i in fs::read_dir(&p)? {
|
for i in fs::read_dir(&path)? {
|
||||||
let r = i?.path();
|
let r = i?.path();
|
||||||
if r.is_dir() {
|
if r.is_dir() {
|
||||||
fs::remove_dir_all(&r)?;
|
fs::remove_dir_all(&r)?;
|
||||||
|
|
@ -21,13 +25,16 @@ impl Preload {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
fs::create_dir_all(&p)?;
|
fs::create_dir_all(&path)?;
|
||||||
Ok(Self(p))
|
Ok(Self {
|
||||||
|
path,
|
||||||
|
regex: regex.map(|r| Regex::new(r).unwrap()),
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn output_folder(&self, infohash: &str, create: bool) -> Result<String> {
|
pub fn output_folder(&self, infohash: &str, create: bool) -> Result<String> {
|
||||||
let mut p = PathBuf::new();
|
let mut p = PathBuf::new();
|
||||||
p.push(&self.0);
|
p.push(&self.path);
|
||||||
p.push(infohash);
|
p.push(infohash);
|
||||||
if p.is_file() {
|
if p.is_file() {
|
||||||
bail!("File destination is not directory!");
|
bail!("File destination is not directory!");
|
||||||
|
|
@ -43,7 +50,7 @@ impl Preload {
|
||||||
|
|
||||||
pub fn absolute(&self, infohash: &str, file: &PathBuf) -> PathBuf {
|
pub fn absolute(&self, infohash: &str, file: &PathBuf) -> PathBuf {
|
||||||
let mut p = PathBuf::new();
|
let mut p = PathBuf::new();
|
||||||
p.push(&self.0);
|
p.push(&self.path);
|
||||||
p.push(infohash);
|
p.push(infohash);
|
||||||
p.push(file);
|
p.push(file);
|
||||||
p
|
p
|
||||||
|
|
@ -62,6 +69,10 @@ impl Preload {
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn path(&self) -> PathBuf {
|
pub fn path(&self) -> PathBuf {
|
||||||
self.0.clone()
|
self.path.clone()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn matches(&self, pattern: &str) -> bool {
|
||||||
|
self.regex.as_ref().is_some_and(|r| r.is_match(pattern))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue