initial commit

.gitignore

@@ -0,0 +1,2 @@
+/target
+Cargo.lock

Cargo.toml

@@ -0,0 +1,17 @@
+[package]
+name = "rssto"
+version = "0.1.0"
+edition = "2024"
+license = "MIT"
+readme = "README.md"
+description = "Aggregate RSS feeds into different formats"
+keywords = ["rss", "aggregator", "convertor", "conversion", "static"]
+categories = ["command-line-utilities", "parsing", "text-processing", "value-formatting"]
+repository = "https://github.com/YGGverse/rssto"
+
+[dependencies]
+anyhow = "1.0"
+chrono = "0.4"
+clap = { version = "4.5", features = ["derive"] }
+reqwest = { version = "0.12", features = ["blocking"] }
+rss = "2.0"

README.md

@@ -1,2 +1,86 @@
 # rssto
-Aggregate RSS feeds into different formats
+
+![Build](https://github.com/YGGverse/rssto/actions/workflows/build.yml/badge.svg)
+[![Dependencies](https://deps.rs/repo/github/YGGverse/rssto/status.svg)](https://deps.rs/repo/github/YGGverse/rssto)
+[![crates.io](https://img.shields.io/crates/v/rssto.svg)](https://crates.io/crates/rssto)
+
+## Aggregate RSS feeds into different formats
+
+A simple multi-source feed aggregator that outputs static files in multiple formats.
+
+## Roadmap
+
+* [x] HTML
+* [ ] Markdown
+* [ ] Gemtext
+
+## Install
+
+``` bash
+cargo install rssto
+```
+
+## Launch
+
+``` bash
+rssto --source https://path/to/source1.rss\
+      --target /path/to/source1dir\
+      --source https://path/to/source2.rss\
+      --target /path/to/source2dir\
+      --format html
+```
+
+### Options
+
+``` bash
+-d, --debug <DEBUG>              Show output (`d` - debug, `e` - error, `i` - info) [default: ei]
+-f, --format <FORMAT>            Export formats (`html`,`md`,etc.) [default: html]
+-l, --limit <LIMIT>              Limit channel items (unlimited by default)
+-s, --source <SOURCE>            RSS feed URL(s)
+    --target <TARGET>            Destination directory
+    --template <TEMPLATE>        Path to template directory [default: template]
+    --time-format <TIME_FORMAT>  Use custom time format [default: "%Y/%m/%d %H:%M:%S %z"]
+-u, --update <UPDATE>            Update timeout in seconds [default: 60]
+-h, --help                       Print help
+-V, --version                    Print version
+```
+
+### Autostart
+
+#### systemd
+
+1. Install `rssto` by copy the binary compiled into the native system apps destination:
+
+  * Linux: `sudo cp /home/user/.cargo/bin/rssto /usr/local/bin`
+
+2. Create `systemd` configuration file:
+
+``` rssto.service
+# /etc/systemd/system/rssto.service
+
+[Unit]
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=simple
+User=rssto
+Group=rssto
+ExecStart=/usr/local/bin/rssto  --source https://path/to/source1.rss\
+                                --target /path/to/source1dir\
+                                --source https://path/to/source2.rss\
+                                --target /path/to/source2dir\
+                                --format html
+                                --time-format %%Y/%%m/%%d %%H:%%M:%%S
+
+[Install]
+WantedBy=multi-user.target
+```
+* on format time, make sure `%` is escaped to `%%`
+
+3. Run in priority:
+
+  * `systemctl daemon-reload` - reload systemd configuration
+  * `systemctl enable rssto` - enable new service
+  * `systemctl start rssto` - start the process
+  * `systemctl status rssto` - check process launched

src/argument.rs

@@ -0,0 +1,37 @@
+use clap::Parser;
+
+#[derive(Parser, Debug)]
+#[command(version, about, long_about = None)]
+pub struct Argument {
+    /// Show output (`d` - debug, `e` - error, `i` - info)
+    #[arg(short, long, default_value_t = String::from("ei"))]
+    pub debug: String,
+
+    /// Export formats (`html`,`md`,etc.)
+    #[arg(short, long, default_values_t = [String::from("html")])]
+    pub format: Vec<String>,
+
+    /// Limit channel items (unlimited by default)
+    #[arg(short, long)]
+    pub limit: Option<usize>,
+
+    /// RSS feed URL(s)
+    #[arg(short, long)]
+    pub source: Vec<String>,
+
+    /// Destination directory
+    #[arg(long)]
+    pub target: Vec<String>,
+
+    /// Path to template directory
+    #[arg(long, default_value_t = String::from("template"))]
+    pub template: String,
+
+    /// Use custom time format
+    #[arg(long, default_value_t = String::from("%Y/%m/%d %H:%M:%S %z"))]
+    pub time_format: String,
+
+    /// Update timeout in seconds
+    #[arg(short, long, default_value_t = 60)]
+    pub update: u64,
+}

src/debug.rs

@@ -0,0 +1,48 @@
+use anyhow::{Result, bail};
+
+#[derive(PartialEq)]
+pub enum Level {
+    //Debug,
+    //Error,
+    Info,
+}
+
+impl Level {
+    fn parse(value: char) -> Result<Self> {
+        match value {
+            //'d' => Ok(Self::Debug),
+            //'e' => Ok(Self::Error),
+            'i' => Ok(Self::Info),
+            _ => bail!("Unsupported debug value `{value}`!"),
+        }
+    }
+}
+
+pub struct Debug(Vec<Level>);
+
+impl Debug {
+    pub fn init(values: &str) -> Result<Self> {
+        let mut l = Vec::with_capacity(values.len());
+        for s in values.to_lowercase().chars() {
+            l.push(Level::parse(s)?);
+        }
+        Ok(Self(l))
+    }
+
+    /* @TODO
+    pub fn error(&self, message: &str) {
+        if self.has(Level::Error) {
+            eprintln!("[{}] [error] {message}", t());
+        }
+    } */
+
+    pub fn info(&self, message: &str) {
+        if self.0.contains(&Level::Info) {
+            println!("[{}] [info] {message}", t());
+        }
+    }
+}
+
+fn t() -> String {
+    crate::time::utc().to_rfc3339()
+}

src/format.rs

@@ -0,0 +1,57 @@
+use anyhow::{bail, Result};
+
+pub struct Template {
+    pub index: String,
+    pub index_item: String,
+}
+
+impl Template {
+    pub fn html(template_path: &str) -> Result<Self> {
+        use std::{fs::read_to_string, path::PathBuf, str::FromStr};
+
+        let mut p = PathBuf::from_str(template_path)?;
+        p.push("html");
+
+        Ok(Self {
+            index: read_to_string(&{
+                let mut p = PathBuf::from(&p);
+                p.push("index.html");
+                p
+            })?,
+            index_item: read_to_string(&{
+                let mut p = PathBuf::from(&p);
+                p.push("index");
+                p.push("item.html");
+                p
+            })?,
+        })
+    }
+}
+
+pub enum Type {
+    Html(Template),
+}
+
+impl Type {
+    fn parse(format: &str, template_path: &str) -> Result<Self> {
+        if matches!(format.to_lowercase().as_str(), "html") {
+            return Ok(Self::Html(Template::html(template_path)?));
+        }
+        bail!("Format `{format}` support yet not implemented!")
+    }
+}
+
+pub struct Format(Vec<Type>);
+
+impl Format {
+    pub fn init(values: &Vec<String>, template: &str) -> Result<Self> {
+        let mut f = Vec::with_capacity(values.len());
+        for s in values {
+            f.push(Type::parse(s, template)?);
+        }
+        Ok(Self(f))
+    }
+    pub fn get(&self) -> &Vec<Type> {
+        &self.0
+    }
+}

src/main.rs

@@ -0,0 +1,101 @@
+mod argument;
+mod debug;
+mod format;
+mod target;
+mod time;
+
+use anyhow::{Result, bail};
+use argument::Argument;
+use debug::Debug;
+use format::Format;
+use format::Type;
+use target::Target;
+use time::Time;
+
+fn main() -> Result<()> {
+    use clap::Parser;
+    use std::{thread::sleep, time::Duration};
+
+    let argument = Argument::parse();
+
+    // parse argument values once
+    let debug = Debug::init(&argument.debug)?;
+    let format = Format::init(&argument.format, &argument.template)?;
+    let target = Target::init(&argument.target)?;
+    let time = Time::init(argument.time_format);
+
+    // validate some targets
+    if argument.source.len() != argument.target.len() {
+        bail!("Targets quantity does not match sources!")
+    }
+
+    debug.info("Crawler started");
+
+    loop {
+        debug.info("Begin new crawl queue...");
+        for (i, s) in argument.source.iter().enumerate() {
+            debug.info(&format!("Update {s}..."));
+            crawl((s, i), &format, &target, &time, &argument.limit)?;
+        }
+        debug.info(&format!(
+            "Crawl queue completed, wait {} seconds to continue...",
+            argument.update
+        ));
+        sleep(Duration::from_secs(argument.update));
+    }
+}
+
+fn crawl(
+    source: (&str, usize),
+    format: &Format,
+    target: &Target,
+    time: &Time,
+    limit: &Option<usize>,
+) -> Result<()> {
+    use reqwest::blocking::get;
+    use rss::Channel;
+    use std::{fs::File, io::Write};
+
+    let c = Channel::read_from(&get(source.0)?.bytes()?[..])?;
+    let i = c.items();
+    let l = limit.unwrap_or(i.len());
+
+    for f in format.get() {
+        match f {
+            Type::Html(template) => File::create(target.index(source.1, "html"))?.write_all(
+                template
+                    .index
+                    .replace("{title}", c.title())
+                    .replace("{description}", c.description())
+                    .replace("{link}", c.link())
+                    .replace("{language}", c.language().unwrap_or_default())
+                    .replace("{pub_date}", &time.format(c.pub_date()))
+                    .replace("{last_build_date}", &time.format(c.last_build_date()))
+                    .replace("{time_generated}", &time.now())
+                    .replace("{items}", &{
+                        let mut items = String::with_capacity(l);
+                        for (n, item) in i.iter().enumerate() {
+                            if n > l {
+                                break;
+                            }
+                            items.push_str(
+                                &template
+                                    .index_item
+                                    .replace("{title}", item.title().unwrap_or_default())
+                                    .replace(
+                                        "{description}",
+                                        item.description().unwrap_or_default(),
+                                    )
+                                    .replace("{link}", item.link().unwrap_or_default())
+                                    .replace("{time}", &time.format(item.pub_date())),
+                            )
+                        }
+                        items
+                    })
+                    .as_bytes(),
+            )?,
+        }
+    }
+
+    Ok(())
+}

src/target.rs

@@ -0,0 +1,25 @@
+use anyhow::{bail, Result};
+use std::{fs, path::PathBuf, str::FromStr};
+
+pub struct Target(Vec<PathBuf>);
+
+impl Target {
+    pub fn init(paths: &Vec<String>) -> Result<Self> {
+        let mut t = Vec::with_capacity(paths.len());
+        for path in paths {
+            let p = PathBuf::from_str(path)?;
+            if fs::metadata(&p).is_ok_and(|t| t.is_file()) {
+                bail!("Target destination exists and not directory!")
+            }
+            fs::create_dir_all(&p)?;
+            t.push(p)
+        }
+        Ok(Self(t))
+    }
+
+    pub fn index(&self, index: usize, extension: &str) -> PathBuf {
+        let mut p = PathBuf::from(&self.0[index]);
+        p.push(format!("index.{extension}"));
+        p
+    }
+}

src/time.rs

@@ -0,0 +1,29 @@
+use chrono::{DateTime, Utc};
+
+pub struct Time(String);
+
+impl Time {
+    pub fn init(format: String) -> Self {
+        Self(format)
+    }
+
+    pub fn format(&self, value: Option<&str>) -> String {
+        match value {
+            Some(v) => chrono::DateTime::parse_from_rfc2822(v)
+                .unwrap()
+                .format(&self.0)
+                .to_string(),
+            None => todo!(),
+        }
+    }
+
+    pub fn now(&self) -> String {
+        utc().format(&self.0).to_string()
+    }
+}
+
+pub fn utc() -> DateTime<Utc> {
+    let s = std::time::SystemTime::now();
+    let c: chrono::DateTime<chrono::Utc> = s.into();
+    c
+}

template/html/index.html

@@ -0,0 +1,49 @@
+<!DOCTYPE html>
+<html lang="{language}">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>{title}</title>
+    <style>
+        * {
+            color-scheme: light dark
+        }
+        body {
+            margin: 0 auto;
+            max-width: 1024px
+        }
+        header {
+            border-bottom: 1px #ccc dotted;
+            padding-bottom: 32px;
+        }
+        section > article {
+            border-bottom: 1px #ccc dotted;
+            padding-bottom: 32px;
+        }
+        footer {
+            font-size: small;
+            padding: 16px 0;
+        }
+    </style>
+</head>
+<body>
+    <header>
+        <h1>{title}</h1>
+        {description}
+    </header>
+    <section>
+        {items}
+    </section>
+    <footer>
+        <p>
+            Source: <a href="{link}">{title}</a> |
+            Updated: {pub_date} |
+            Build: {last_build_date} |
+            Generated: {time_generated}
+        </p>
+        <p>
+            Powered by <a href="https://github.com/YGGverse/rssto">rssto</a>.
+        </p>
+    </footer>
+</body>
+</html>

template/html/index/item.html

@@ -0,0 +1,5 @@
+<article>
+    <h2>{title}</h2>
+    <p>{description}</p>
+    <a href="{link}">{time}</a>
+</article>