mirror of
https://github.com/YGGverse/aquatic-crawler.git
synced 2026-04-01 17:45:39 +00:00
count total data size for torrent resolved
This commit is contained in:
parent
27360c4da1
commit
b22695587d
3 changed files with 32 additions and 23 deletions
20
src/index.rs
20
src/index.rs
|
|
@ -15,15 +15,15 @@ pub struct Index {
|
||||||
is_changed: bool,
|
is_changed: bool,
|
||||||
/// Store the index value in memory only when it is in use by the init options
|
/// Store the index value in memory only when it is in use by the init options
|
||||||
has_name: bool,
|
has_name: bool,
|
||||||
has_length: bool,
|
has_size: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Index {
|
impl Index {
|
||||||
pub fn init(capacity: usize, timeout: Option<i64>, has_name: bool, has_length: bool) -> Self {
|
pub fn init(capacity: usize, timeout: Option<i64>, has_name: bool, has_size: bool) -> Self {
|
||||||
Self {
|
Self {
|
||||||
index: HashMap::with_capacity(capacity),
|
index: HashMap::with_capacity(capacity),
|
||||||
timeout: timeout.map(Duration::seconds),
|
timeout: timeout.map(Duration::seconds),
|
||||||
has_length,
|
has_size,
|
||||||
has_name,
|
has_name,
|
||||||
is_changed: false,
|
is_changed: false,
|
||||||
}
|
}
|
||||||
|
|
@ -49,21 +49,15 @@ impl Index {
|
||||||
self.index.values().map(|i| i.node).sum::<u64>()
|
self.index.values().map(|i| i.node).sum::<u64>()
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn insert(
|
pub fn insert(&mut self, infohash: String, node: u64, size: u64, name: Option<String>) {
|
||||||
&mut self,
|
|
||||||
infohash: String,
|
|
||||||
node: u64,
|
|
||||||
name: Option<String>,
|
|
||||||
length: Option<u64>,
|
|
||||||
) {
|
|
||||||
if self
|
if self
|
||||||
.index
|
.index
|
||||||
.insert(
|
.insert(
|
||||||
infohash,
|
infohash,
|
||||||
Value::new(
|
Value::new(
|
||||||
node,
|
node,
|
||||||
|
if self.has_size { Some(size) } else { None },
|
||||||
if self.has_name { name } else { None },
|
if self.has_name { name } else { None },
|
||||||
if self.has_length { length } else { None },
|
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
.is_none()
|
.is_none()
|
||||||
|
|
@ -88,9 +82,9 @@ fn test() {
|
||||||
// test values auto-clean by timeout
|
// test values auto-clean by timeout
|
||||||
let mut i = Index::init(2, Some(3), false, false);
|
let mut i = Index::init(2, Some(3), false, false);
|
||||||
|
|
||||||
i.insert("h1".to_string(), 0, None, None);
|
i.insert("h1".to_string(), 0, 0, None);
|
||||||
sleep(Duration::from_secs(1));
|
sleep(Duration::from_secs(1));
|
||||||
i.insert("h2".to_string(), 0, None, None);
|
i.insert("h2".to_string(), 0, 0, None);
|
||||||
|
|
||||||
i.refresh();
|
i.refresh();
|
||||||
assert_eq!(i.len(), 2);
|
assert_eq!(i.len(), 2);
|
||||||
|
|
|
||||||
|
|
@ -7,17 +7,17 @@ pub struct Value {
|
||||||
pub time: DateTime<Utc>,
|
pub time: DateTime<Utc>,
|
||||||
pub node: u64,
|
pub node: u64,
|
||||||
// Isolate by applying internal filter on value set
|
// Isolate by applying internal filter on value set
|
||||||
length: Option<u64>,
|
size: Option<u64>,
|
||||||
name: Option<String>,
|
name: Option<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Value {
|
impl Value {
|
||||||
/// Create new `Self` with current timestamp
|
/// Create new `Self` with current timestamp
|
||||||
pub fn new(node: u64, name: Option<String>, length: Option<u64>) -> Self {
|
pub fn new(node: u64, size: Option<u64>, name: Option<String>) -> Self {
|
||||||
Self {
|
Self {
|
||||||
time: Utc::now(),
|
time: Utc::now(),
|
||||||
node,
|
node,
|
||||||
length,
|
size,
|
||||||
name: filter_name(name),
|
name: filter_name(name),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -26,8 +26,8 @@ impl Value {
|
||||||
self.name.as_ref()
|
self.name.as_ref()
|
||||||
}
|
}
|
||||||
/// Get reference to the safely constructed `length` member
|
/// Get reference to the safely constructed `length` member
|
||||||
pub fn length(&self) -> Option<u64> {
|
pub fn size(&self) -> Option<u64> {
|
||||||
self.length
|
self.size
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
25
src/main.rs
25
src/main.rs
|
|
@ -150,7 +150,7 @@ async fn main() -> Result<()> {
|
||||||
config.preload_max_filecount.unwrap_or_default(),
|
config.preload_max_filecount.unwrap_or_default(),
|
||||||
);
|
);
|
||||||
mt.wait_until_initialized().await?;
|
mt.wait_until_initialized().await?;
|
||||||
let (name, length) = mt.with_metadata(|m| {
|
let (name, size) = mt.with_metadata(|m| {
|
||||||
// init preload files list
|
// init preload files list
|
||||||
if let Some(ref p) = preload {
|
if let Some(ref p) = preload {
|
||||||
for (id, info) in m.file_infos.iter().enumerate() {
|
for (id, info) in m.file_infos.iter().enumerate() {
|
||||||
|
|
@ -183,7 +183,8 @@ async fn main() -> Result<()> {
|
||||||
if let Some(ref t) = torrent {
|
if let Some(ref t) = torrent {
|
||||||
save_torrent_file(t, &debug, &i, &m.torrent_bytes)
|
save_torrent_file(t, &debug, &i, &m.torrent_bytes)
|
||||||
}
|
}
|
||||||
(m.info.name.as_ref().map(|n| n.to_string()), m.info.length)
|
|
||||||
|
(m.info.name.as_ref().map(|n| n.to_string()), size(&m.info))
|
||||||
})?;
|
})?;
|
||||||
session.update_only_files(&mt, &only_files).await?;
|
session.update_only_files(&mt, &only_files).await?;
|
||||||
session.unpause(&mt).await?;
|
session.unpause(&mt).await?;
|
||||||
|
|
@ -198,7 +199,7 @@ async fn main() -> Result<()> {
|
||||||
p.cleanup(&i, Some(only_files_keep))?
|
p.cleanup(&i, Some(only_files_keep))?
|
||||||
}
|
}
|
||||||
|
|
||||||
index.insert(i, only_files_size, name, length)
|
index.insert(i, only_files_size, size, name)
|
||||||
}
|
}
|
||||||
Ok(AddTorrentResponse::ListOnly(r)) => {
|
Ok(AddTorrentResponse::ListOnly(r)) => {
|
||||||
if let Some(ref t) = torrent {
|
if let Some(ref t) = torrent {
|
||||||
|
|
@ -209,7 +210,7 @@ async fn main() -> Result<()> {
|
||||||
// use `r.info` for Memory, SQLite,
|
// use `r.info` for Memory, SQLite,
|
||||||
// Manticore and other alternative storage type
|
// Manticore and other alternative storage type
|
||||||
|
|
||||||
index.insert(i, 0, r.info.name.map(|n| n.to_string()), r.info.length)
|
index.insert(i, 0, size(&r.info), r.info.name.map(|n| n.to_string()))
|
||||||
}
|
}
|
||||||
// unexpected as should be deleted
|
// unexpected as should be deleted
|
||||||
Ok(AddTorrentResponse::AlreadyManaged(..)) => panic!(),
|
Ok(AddTorrentResponse::AlreadyManaged(..)) => panic!(),
|
||||||
|
|
@ -238,7 +239,7 @@ async fn main() -> Result<()> {
|
||||||
rss.push(
|
rss.push(
|
||||||
k,
|
k,
|
||||||
v.name().unwrap_or(k),
|
v.name().unwrap_or(k),
|
||||||
v.length().map(|l| l.bytes()),
|
v.size().map(|l| l.bytes()),
|
||||||
Some(&v.time.to_rfc2822()),
|
Some(&v.time.to_rfc2822()),
|
||||||
)?
|
)?
|
||||||
}
|
}
|
||||||
|
|
@ -285,3 +286,17 @@ fn magnet(infohash: &str, trackers: Option<&HashSet<Url>>) -> String {
|
||||||
}
|
}
|
||||||
m
|
m
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Count total size, including torrent files
|
||||||
|
fn size(info: &librqbit::TorrentMetaV1Info<librqbit::ByteBufOwned>) -> u64 {
|
||||||
|
let mut t = 0;
|
||||||
|
if let Some(l) = info.length {
|
||||||
|
t += l
|
||||||
|
}
|
||||||
|
if let Some(ref files) = info.files {
|
||||||
|
for f in files {
|
||||||
|
t += f.length
|
||||||
|
}
|
||||||
|
}
|
||||||
|
t
|
||||||
|
}
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue