mirror of
https://github.com/YGGverse/pulsarss.git
synced 2026-03-31 09:05:29 +00:00
update error handle, change version
This commit is contained in:
parent
502a099103
commit
27a418e256
6 changed files with 46 additions and 81 deletions
11
Cargo.toml
11
Cargo.toml
|
|
@ -1,6 +1,6 @@
|
|||
[package]
|
||||
name = "pulsarss"
|
||||
version = "0.1.5"
|
||||
version = "0.2.0"
|
||||
edition = "2021"
|
||||
license = "MIT"
|
||||
readme = "README.md"
|
||||
|
|
@ -10,7 +10,10 @@ categories = ["command-line-utilities", "parsing", "text-processing", "value-for
|
|||
repository = "https://github.com/YGGverse/pulsarss"
|
||||
|
||||
[dependencies]
|
||||
anyhow = "1.0"
|
||||
chrono = "0.4.39"
|
||||
clap = { version = "4.5.28", features = ["derive"] }
|
||||
reqwest = { version = "0.12.12", features = ["blocking"] }
|
||||
rss = "2.0.11"
|
||||
clap = { version = "4.5", features = ["derive"] }
|
||||
log = "0.4"
|
||||
reqwest = { version = "0.12", features = ["blocking"] }
|
||||
rss = "2.0"
|
||||
tracing-subscriber = "0.3"
|
||||
24
README.md
24
README.md
|
|
@ -25,15 +25,9 @@ pulsarss --source https://path/to/feed.rss --index index.gmi
|
|||
|
||||
### Options
|
||||
|
||||
* `source`, `s` - RSS feed source (required)
|
||||
* `target`, `t` - Destination directory (`public` by default)
|
||||
* `update`, `u` - Update timeout in seconds (`60` by default)
|
||||
* `index`, `i` - Generate index files with given filename (disabled by default)
|
||||
* `limit`, `l` - Limit channel items (unlimited by default)
|
||||
* `output`, `o` - Print output (`dw` by default):
|
||||
* `d` - debug
|
||||
* `w` - warning
|
||||
* `n` - disable
|
||||
``` bash
|
||||
pulsarss --help
|
||||
```
|
||||
|
||||
### Autostart
|
||||
|
||||
|
|
@ -41,7 +35,7 @@ pulsarss --source https://path/to/feed.rss --index index.gmi
|
|||
|
||||
1. Install `pulsarss` by copy the binary compiled into the native system apps destination:
|
||||
|
||||
* Linux: `sudo cp /home/user/.cargo/bin/pulsarss /usr/local/bin`
|
||||
* Linux: `sudo install /home/user/.cargo/bin/pulsarss /usr/local/bin/pulsarss`
|
||||
|
||||
2. Create `systemd` configuration file:
|
||||
|
||||
|
|
@ -54,13 +48,23 @@ Wants=network-online.target
|
|||
|
||||
[Service]
|
||||
Type=simple
|
||||
|
||||
User=pulsarss
|
||||
Group=pulsarss
|
||||
|
||||
# Uncomment for debug
|
||||
# Environment="RUST_LOG=debug"
|
||||
# Environment="NO_COLOR=1"
|
||||
|
||||
ExecStart=/usr/local/bin/pulsarss -s https://path/to/feed.rss -i index.gmi
|
||||
|
||||
StandardOutput=file:///home/pulsarss/debug.log
|
||||
StandardError=file:///home/pulsarss/error.log
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
```
|
||||
* example above requires new system user (`useradd -m pulsarss`)
|
||||
|
||||
3. Run in priority:
|
||||
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ use clap::Parser;
|
|||
|
||||
#[derive(Parser, Debug)]
|
||||
#[command(version, about, long_about = None)]
|
||||
pub struct Argument {
|
||||
pub struct Config {
|
||||
/// Generate index files with given filename (disabled by default)
|
||||
#[arg(short, long)]
|
||||
pub index: Option<String>,
|
||||
|
|
@ -11,10 +11,6 @@ pub struct Argument {
|
|||
#[arg(short, long)]
|
||||
pub limit: Option<usize>,
|
||||
|
||||
/// Show output (`dw` by default)
|
||||
#[arg(short, long, default_value_t = String::from("dw"))]
|
||||
pub output: String,
|
||||
|
||||
/// RSS feed source (required)
|
||||
#[arg(short, long)]
|
||||
pub source: String,
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
use std::error::Error;
|
||||
use anyhow::Result;
|
||||
use std::path::MAIN_SEPARATOR;
|
||||
|
||||
pub struct Destination {
|
||||
|
|
@ -9,7 +9,7 @@ pub struct Destination {
|
|||
impl Destination {
|
||||
// Constructors
|
||||
|
||||
pub fn build(base: &str, pub_date: &str, mkdir: bool) -> Result<Self, Box<dyn Error>> {
|
||||
pub fn build(base: &str, pub_date: &str, mkdir: bool) -> Result<Self> {
|
||||
use chrono::{DateTime, Datelike, Timelike};
|
||||
|
||||
let time = DateTime::parse_from_rfc2822(pub_date)?;
|
||||
|
|
|
|||
46
src/main.rs
46
src/main.rs
|
|
@ -1,27 +1,25 @@
|
|||
mod argument;
|
||||
mod config;
|
||||
mod destination;
|
||||
mod output;
|
||||
|
||||
use argument::Argument;
|
||||
use output::Output;
|
||||
use std::error::Error;
|
||||
use anyhow::Result;
|
||||
use config::Config;
|
||||
use log::*;
|
||||
|
||||
fn main() -> Result<(), Box<dyn Error>> {
|
||||
fn main() -> Result<()> {
|
||||
use clap::Parser;
|
||||
use std::{thread::sleep, time::Duration};
|
||||
|
||||
let argument = Argument::parse();
|
||||
let output = Output::build(&argument.output);
|
||||
info!("crawler started");
|
||||
|
||||
output.debug("crawler started");
|
||||
let config = Config::parse();
|
||||
|
||||
loop {
|
||||
crawl(&argument, &output)?;
|
||||
sleep(Duration::from_secs(argument.update));
|
||||
crawl(&config)?;
|
||||
sleep(Duration::from_secs(config.update));
|
||||
}
|
||||
}
|
||||
|
||||
fn crawl(argument: &Argument, output: &Output) -> Result<(), Box<dyn Error>> {
|
||||
fn crawl(config: &Config) -> Result<()> {
|
||||
use destination::Destination;
|
||||
use reqwest::blocking::get;
|
||||
use rss::Channel;
|
||||
|
|
@ -31,17 +29,17 @@ fn crawl(argument: &Argument, output: &Output) -> Result<(), Box<dyn Error>> {
|
|||
io::{Read, Write},
|
||||
};
|
||||
|
||||
output.debug("feed update begin");
|
||||
debug!("feed update begin");
|
||||
|
||||
let mut total = 0;
|
||||
let mut exist = 0;
|
||||
let mut index = HashSet::new();
|
||||
|
||||
let channel = Channel::read_from(&get(&argument.source)?.bytes()?[..])?;
|
||||
let channel = Channel::read_from(&get(&config.source)?.bytes()?[..])?;
|
||||
|
||||
// collect feed items
|
||||
for item in channel.items().iter() {
|
||||
if argument.limit.is_some_and(|limit| total >= limit) {
|
||||
if config.limit.is_some_and(|limit| total >= limit) {
|
||||
break;
|
||||
}
|
||||
|
||||
|
|
@ -51,7 +49,7 @@ fn crawl(argument: &Argument, output: &Output) -> Result<(), Box<dyn Error>> {
|
|||
|
||||
let (destination, pub_date) = match item.pub_date() {
|
||||
Some(pub_date) => {
|
||||
let destination = Destination::build(&argument.target, pub_date, true)?;
|
||||
let destination = Destination::build(&config.target, pub_date, true)?;
|
||||
if metadata(destination.item()).is_ok() {
|
||||
exist += 1;
|
||||
continue;
|
||||
|
|
@ -68,7 +66,7 @@ fn crawl(argument: &Argument, output: &Output) -> Result<(), Box<dyn Error>> {
|
|||
(destination, pub_date)
|
||||
}
|
||||
None => {
|
||||
output.warning("item skipped as `pub_date` required by application");
|
||||
debug!("item skipped as `pub_date` required by application");
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
|
@ -87,13 +85,13 @@ fn crawl(argument: &Argument, output: &Output) -> Result<(), Box<dyn Error>> {
|
|||
|
||||
File::create(destination.item())?.write_all(data.join("\n\n").as_bytes())?;
|
||||
|
||||
if argument.index.is_some() {
|
||||
if config.index.is_some() {
|
||||
index.insert(destination.path); // request index file update in this location
|
||||
}
|
||||
}
|
||||
|
||||
// renew pending index files on items crawl completed
|
||||
if let Some(ref index_argument) = argument.index {
|
||||
if let Some(ref index_argument) = config.index {
|
||||
for path in index {
|
||||
let index_filename = format!("{path}{index_argument}");
|
||||
|
||||
|
|
@ -101,7 +99,7 @@ fn crawl(argument: &Argument, output: &Output) -> Result<(), Box<dyn Error>> {
|
|||
let mut files: Vec<_> = read_dir(&path)?.filter_map(Result::ok).collect();
|
||||
files.sort_by_key(|f| std::cmp::Reverse(f.file_name()));
|
||||
|
||||
let mut data = Vec::with_capacity(argument.limit.unwrap_or_default());
|
||||
let mut data = Vec::with_capacity(config.limit.unwrap_or_default());
|
||||
let mut index = File::create(&index_filename)?;
|
||||
let mut total = 0;
|
||||
|
||||
|
|
@ -112,7 +110,7 @@ fn crawl(argument: &Argument, output: &Output) -> Result<(), Box<dyn Error>> {
|
|||
}
|
||||
|
||||
for file in files {
|
||||
if argument.limit.is_some_and(|limit| total >= limit) {
|
||||
if config.limit.is_some_and(|limit| total >= limit) {
|
||||
break;
|
||||
}
|
||||
|
||||
|
|
@ -131,15 +129,15 @@ fn crawl(argument: &Argument, output: &Output) -> Result<(), Box<dyn Error>> {
|
|||
|
||||
index.write_all(data.join("\n\n").as_bytes())?;
|
||||
|
||||
output.debug(&format!("renew `{index_filename}` (total: {total})"));
|
||||
debug!("renew `{index_filename}` (total: {total})");
|
||||
}
|
||||
}
|
||||
|
||||
// print totals
|
||||
output.debug(&format!(
|
||||
debug!(
|
||||
"feed update completed (added: {} / exist: {exist} / total: {total})",
|
||||
total - exist
|
||||
));
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,36 +0,0 @@
|
|||
use chrono::{DateTime, Utc};
|
||||
use std::time::SystemTime;
|
||||
|
||||
struct Level {
|
||||
debug: bool,
|
||||
warning: bool,
|
||||
}
|
||||
|
||||
pub struct Output(Level);
|
||||
|
||||
impl Output {
|
||||
pub fn build(level: &str) -> Self {
|
||||
Self(Level {
|
||||
debug: level.contains("d"),
|
||||
warning: level.contains("w"),
|
||||
})
|
||||
}
|
||||
|
||||
pub fn debug(&self, message: &str) {
|
||||
if self.0.debug {
|
||||
eprintln!("[debug] [{}] {message}", time());
|
||||
}
|
||||
}
|
||||
|
||||
pub fn warning(&self, message: &str) {
|
||||
if self.0.warning {
|
||||
eprintln!("[warning] [{}] {message}", time());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn time() -> String {
|
||||
let system_time = SystemTime::now();
|
||||
let datetime: DateTime<Utc> = system_time.into();
|
||||
datetime.to_rfc3339()
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue