mirror of
https://github.com/YGGverse/pulsarss.git
synced 2026-03-31 09:05:29 +00:00
update error handle, change version
This commit is contained in:
parent
502a099103
commit
27a418e256
6 changed files with 46 additions and 81 deletions
11
Cargo.toml
11
Cargo.toml
|
|
@ -1,6 +1,6 @@
|
||||||
[package]
|
[package]
|
||||||
name = "pulsarss"
|
name = "pulsarss"
|
||||||
version = "0.1.5"
|
version = "0.2.0"
|
||||||
edition = "2021"
|
edition = "2021"
|
||||||
license = "MIT"
|
license = "MIT"
|
||||||
readme = "README.md"
|
readme = "README.md"
|
||||||
|
|
@ -10,7 +10,10 @@ categories = ["command-line-utilities", "parsing", "text-processing", "value-for
|
||||||
repository = "https://github.com/YGGverse/pulsarss"
|
repository = "https://github.com/YGGverse/pulsarss"
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
|
anyhow = "1.0"
|
||||||
chrono = "0.4.39"
|
chrono = "0.4.39"
|
||||||
clap = { version = "4.5.28", features = ["derive"] }
|
clap = { version = "4.5", features = ["derive"] }
|
||||||
reqwest = { version = "0.12.12", features = ["blocking"] }
|
log = "0.4"
|
||||||
rss = "2.0.11"
|
reqwest = { version = "0.12", features = ["blocking"] }
|
||||||
|
rss = "2.0"
|
||||||
|
tracing-subscriber = "0.3"
|
||||||
24
README.md
24
README.md
|
|
@ -25,15 +25,9 @@ pulsarss --source https://path/to/feed.rss --index index.gmi
|
||||||
|
|
||||||
### Options
|
### Options
|
||||||
|
|
||||||
* `source`, `s` - RSS feed source (required)
|
``` bash
|
||||||
* `target`, `t` - Destination directory (`public` by default)
|
pulsarss --help
|
||||||
* `update`, `u` - Update timeout in seconds (`60` by default)
|
```
|
||||||
* `index`, `i` - Generate index files with given filename (disabled by default)
|
|
||||||
* `limit`, `l` - Limit channel items (unlimited by default)
|
|
||||||
* `output`, `o` - Print output (`dw` by default):
|
|
||||||
* `d` - debug
|
|
||||||
* `w` - warning
|
|
||||||
* `n` - disable
|
|
||||||
|
|
||||||
### Autostart
|
### Autostart
|
||||||
|
|
||||||
|
|
@ -41,7 +35,7 @@ pulsarss --source https://path/to/feed.rss --index index.gmi
|
||||||
|
|
||||||
1. Install `pulsarss` by copy the binary compiled into the native system apps destination:
|
1. Install `pulsarss` by copy the binary compiled into the native system apps destination:
|
||||||
|
|
||||||
* Linux: `sudo cp /home/user/.cargo/bin/pulsarss /usr/local/bin`
|
* Linux: `sudo install /home/user/.cargo/bin/pulsarss /usr/local/bin/pulsarss`
|
||||||
|
|
||||||
2. Create `systemd` configuration file:
|
2. Create `systemd` configuration file:
|
||||||
|
|
||||||
|
|
@ -54,13 +48,23 @@ Wants=network-online.target
|
||||||
|
|
||||||
[Service]
|
[Service]
|
||||||
Type=simple
|
Type=simple
|
||||||
|
|
||||||
User=pulsarss
|
User=pulsarss
|
||||||
Group=pulsarss
|
Group=pulsarss
|
||||||
|
|
||||||
|
# Uncomment for debug
|
||||||
|
# Environment="RUST_LOG=debug"
|
||||||
|
# Environment="NO_COLOR=1"
|
||||||
|
|
||||||
ExecStart=/usr/local/bin/pulsarss -s https://path/to/feed.rss -i index.gmi
|
ExecStart=/usr/local/bin/pulsarss -s https://path/to/feed.rss -i index.gmi
|
||||||
|
|
||||||
|
StandardOutput=file:///home/pulsarss/debug.log
|
||||||
|
StandardError=file:///home/pulsarss/error.log
|
||||||
|
|
||||||
[Install]
|
[Install]
|
||||||
WantedBy=multi-user.target
|
WantedBy=multi-user.target
|
||||||
```
|
```
|
||||||
|
* example above requires new system user (`useradd -m pulsarss`)
|
||||||
|
|
||||||
3. Run in priority:
|
3. Run in priority:
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -2,7 +2,7 @@ use clap::Parser;
|
||||||
|
|
||||||
#[derive(Parser, Debug)]
|
#[derive(Parser, Debug)]
|
||||||
#[command(version, about, long_about = None)]
|
#[command(version, about, long_about = None)]
|
||||||
pub struct Argument {
|
pub struct Config {
|
||||||
/// Generate index files with given filename (disabled by default)
|
/// Generate index files with given filename (disabled by default)
|
||||||
#[arg(short, long)]
|
#[arg(short, long)]
|
||||||
pub index: Option<String>,
|
pub index: Option<String>,
|
||||||
|
|
@ -11,10 +11,6 @@ pub struct Argument {
|
||||||
#[arg(short, long)]
|
#[arg(short, long)]
|
||||||
pub limit: Option<usize>,
|
pub limit: Option<usize>,
|
||||||
|
|
||||||
/// Show output (`dw` by default)
|
|
||||||
#[arg(short, long, default_value_t = String::from("dw"))]
|
|
||||||
pub output: String,
|
|
||||||
|
|
||||||
/// RSS feed source (required)
|
/// RSS feed source (required)
|
||||||
#[arg(short, long)]
|
#[arg(short, long)]
|
||||||
pub source: String,
|
pub source: String,
|
||||||
|
|
@ -1,4 +1,4 @@
|
||||||
use std::error::Error;
|
use anyhow::Result;
|
||||||
use std::path::MAIN_SEPARATOR;
|
use std::path::MAIN_SEPARATOR;
|
||||||
|
|
||||||
pub struct Destination {
|
pub struct Destination {
|
||||||
|
|
@ -9,7 +9,7 @@ pub struct Destination {
|
||||||
impl Destination {
|
impl Destination {
|
||||||
// Constructors
|
// Constructors
|
||||||
|
|
||||||
pub fn build(base: &str, pub_date: &str, mkdir: bool) -> Result<Self, Box<dyn Error>> {
|
pub fn build(base: &str, pub_date: &str, mkdir: bool) -> Result<Self> {
|
||||||
use chrono::{DateTime, Datelike, Timelike};
|
use chrono::{DateTime, Datelike, Timelike};
|
||||||
|
|
||||||
let time = DateTime::parse_from_rfc2822(pub_date)?;
|
let time = DateTime::parse_from_rfc2822(pub_date)?;
|
||||||
|
|
|
||||||
46
src/main.rs
46
src/main.rs
|
|
@ -1,27 +1,25 @@
|
||||||
mod argument;
|
mod config;
|
||||||
mod destination;
|
mod destination;
|
||||||
mod output;
|
|
||||||
|
|
||||||
use argument::Argument;
|
use anyhow::Result;
|
||||||
use output::Output;
|
use config::Config;
|
||||||
use std::error::Error;
|
use log::*;
|
||||||
|
|
||||||
fn main() -> Result<(), Box<dyn Error>> {
|
fn main() -> Result<()> {
|
||||||
use clap::Parser;
|
use clap::Parser;
|
||||||
use std::{thread::sleep, time::Duration};
|
use std::{thread::sleep, time::Duration};
|
||||||
|
|
||||||
let argument = Argument::parse();
|
info!("crawler started");
|
||||||
let output = Output::build(&argument.output);
|
|
||||||
|
|
||||||
output.debug("crawler started");
|
let config = Config::parse();
|
||||||
|
|
||||||
loop {
|
loop {
|
||||||
crawl(&argument, &output)?;
|
crawl(&config)?;
|
||||||
sleep(Duration::from_secs(argument.update));
|
sleep(Duration::from_secs(config.update));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn crawl(argument: &Argument, output: &Output) -> Result<(), Box<dyn Error>> {
|
fn crawl(config: &Config) -> Result<()> {
|
||||||
use destination::Destination;
|
use destination::Destination;
|
||||||
use reqwest::blocking::get;
|
use reqwest::blocking::get;
|
||||||
use rss::Channel;
|
use rss::Channel;
|
||||||
|
|
@ -31,17 +29,17 @@ fn crawl(argument: &Argument, output: &Output) -> Result<(), Box<dyn Error>> {
|
||||||
io::{Read, Write},
|
io::{Read, Write},
|
||||||
};
|
};
|
||||||
|
|
||||||
output.debug("feed update begin");
|
debug!("feed update begin");
|
||||||
|
|
||||||
let mut total = 0;
|
let mut total = 0;
|
||||||
let mut exist = 0;
|
let mut exist = 0;
|
||||||
let mut index = HashSet::new();
|
let mut index = HashSet::new();
|
||||||
|
|
||||||
let channel = Channel::read_from(&get(&argument.source)?.bytes()?[..])?;
|
let channel = Channel::read_from(&get(&config.source)?.bytes()?[..])?;
|
||||||
|
|
||||||
// collect feed items
|
// collect feed items
|
||||||
for item in channel.items().iter() {
|
for item in channel.items().iter() {
|
||||||
if argument.limit.is_some_and(|limit| total >= limit) {
|
if config.limit.is_some_and(|limit| total >= limit) {
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -51,7 +49,7 @@ fn crawl(argument: &Argument, output: &Output) -> Result<(), Box<dyn Error>> {
|
||||||
|
|
||||||
let (destination, pub_date) = match item.pub_date() {
|
let (destination, pub_date) = match item.pub_date() {
|
||||||
Some(pub_date) => {
|
Some(pub_date) => {
|
||||||
let destination = Destination::build(&argument.target, pub_date, true)?;
|
let destination = Destination::build(&config.target, pub_date, true)?;
|
||||||
if metadata(destination.item()).is_ok() {
|
if metadata(destination.item()).is_ok() {
|
||||||
exist += 1;
|
exist += 1;
|
||||||
continue;
|
continue;
|
||||||
|
|
@ -68,7 +66,7 @@ fn crawl(argument: &Argument, output: &Output) -> Result<(), Box<dyn Error>> {
|
||||||
(destination, pub_date)
|
(destination, pub_date)
|
||||||
}
|
}
|
||||||
None => {
|
None => {
|
||||||
output.warning("item skipped as `pub_date` required by application");
|
debug!("item skipped as `pub_date` required by application");
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
@ -87,13 +85,13 @@ fn crawl(argument: &Argument, output: &Output) -> Result<(), Box<dyn Error>> {
|
||||||
|
|
||||||
File::create(destination.item())?.write_all(data.join("\n\n").as_bytes())?;
|
File::create(destination.item())?.write_all(data.join("\n\n").as_bytes())?;
|
||||||
|
|
||||||
if argument.index.is_some() {
|
if config.index.is_some() {
|
||||||
index.insert(destination.path); // request index file update in this location
|
index.insert(destination.path); // request index file update in this location
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// renew pending index files on items crawl completed
|
// renew pending index files on items crawl completed
|
||||||
if let Some(ref index_argument) = argument.index {
|
if let Some(ref index_argument) = config.index {
|
||||||
for path in index {
|
for path in index {
|
||||||
let index_filename = format!("{path}{index_argument}");
|
let index_filename = format!("{path}{index_argument}");
|
||||||
|
|
||||||
|
|
@ -101,7 +99,7 @@ fn crawl(argument: &Argument, output: &Output) -> Result<(), Box<dyn Error>> {
|
||||||
let mut files: Vec<_> = read_dir(&path)?.filter_map(Result::ok).collect();
|
let mut files: Vec<_> = read_dir(&path)?.filter_map(Result::ok).collect();
|
||||||
files.sort_by_key(|f| std::cmp::Reverse(f.file_name()));
|
files.sort_by_key(|f| std::cmp::Reverse(f.file_name()));
|
||||||
|
|
||||||
let mut data = Vec::with_capacity(argument.limit.unwrap_or_default());
|
let mut data = Vec::with_capacity(config.limit.unwrap_or_default());
|
||||||
let mut index = File::create(&index_filename)?;
|
let mut index = File::create(&index_filename)?;
|
||||||
let mut total = 0;
|
let mut total = 0;
|
||||||
|
|
||||||
|
|
@ -112,7 +110,7 @@ fn crawl(argument: &Argument, output: &Output) -> Result<(), Box<dyn Error>> {
|
||||||
}
|
}
|
||||||
|
|
||||||
for file in files {
|
for file in files {
|
||||||
if argument.limit.is_some_and(|limit| total >= limit) {
|
if config.limit.is_some_and(|limit| total >= limit) {
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -131,15 +129,15 @@ fn crawl(argument: &Argument, output: &Output) -> Result<(), Box<dyn Error>> {
|
||||||
|
|
||||||
index.write_all(data.join("\n\n").as_bytes())?;
|
index.write_all(data.join("\n\n").as_bytes())?;
|
||||||
|
|
||||||
output.debug(&format!("renew `{index_filename}` (total: {total})"));
|
debug!("renew `{index_filename}` (total: {total})");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// print totals
|
// print totals
|
||||||
output.debug(&format!(
|
debug!(
|
||||||
"feed update completed (added: {} / exist: {exist} / total: {total})",
|
"feed update completed (added: {} / exist: {exist} / total: {total})",
|
||||||
total - exist
|
total - exist
|
||||||
));
|
);
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,36 +0,0 @@
|
||||||
use chrono::{DateTime, Utc};
|
|
||||||
use std::time::SystemTime;
|
|
||||||
|
|
||||||
struct Level {
|
|
||||||
debug: bool,
|
|
||||||
warning: bool,
|
|
||||||
}
|
|
||||||
|
|
||||||
pub struct Output(Level);
|
|
||||||
|
|
||||||
impl Output {
|
|
||||||
pub fn build(level: &str) -> Self {
|
|
||||||
Self(Level {
|
|
||||||
debug: level.contains("d"),
|
|
||||||
warning: level.contains("w"),
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn debug(&self, message: &str) {
|
|
||||||
if self.0.debug {
|
|
||||||
eprintln!("[debug] [{}] {message}", time());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn warning(&self, message: &str) {
|
|
||||||
if self.0.warning {
|
|
||||||
eprintln!("[warning] [{}] {message}", time());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn time() -> String {
|
|
||||||
let system_time = SystemTime::now();
|
|
||||||
let datetime: DateTime<Utc> = system_time.into();
|
|
||||||
datetime.to_rfc3339()
|
|
||||||
}
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue