diff --git a/crates/crawler/config.toml b/crates/crawler/config.toml index 3232c16..ad50346 100644 --- a/crates/crawler/config.toml +++ b/crates/crawler/config.toml @@ -6,7 +6,7 @@ update = 900 [mysql] host = "localhost" port = 3306 -user = "" +username = "" password = "" database = "rssto" diff --git a/crates/crawler/src/config.rs b/crates/crawler/src/config.rs index 701c6e4..b4734cc 100644 --- a/crates/crawler/src/config.rs +++ b/crates/crawler/src/config.rs @@ -8,7 +8,7 @@ pub struct Mysql { pub host: String, pub password: String, pub port: u16, - pub user: String, + pub username: String, } #[derive(Debug, Deserialize)] diff --git a/crates/crawler/src/main.rs b/crates/crawler/src/main.rs index 0767499..881b9b6 100644 --- a/crates/crawler/src/main.rs +++ b/crates/crawler/src/main.rs @@ -30,7 +30,7 @@ fn main() -> Result<()> { let db = mysql::Database::pool( &config.mysql.host, config.mysql.port, - &config.mysql.user, + &config.mysql.username, &config.mysql.password, &config.mysql.database, )?; diff --git a/crates/http/Cargo.toml b/crates/http/Cargo.toml index e2cfbf8..e5b8c8c 100644 --- a/crates/http/Cargo.toml +++ b/crates/http/Cargo.toml @@ -15,3 +15,5 @@ clap = { version = "4.5.54", features = ["derive"] } mysql = { package = "rssto-mysql", version = "0.1.0", path = "../mysql" } rocket = "0.5.1" rocket_dyn_templates = { version = "0.2.0", features = ["tera"] } +serde = { version = "1.0.228", features = ["derive"] } +toml = "0.9.10" \ No newline at end of file diff --git a/crates/http/README.md b/crates/http/README.md index c9edc0e..e9b1b42 100644 --- a/crates/http/README.md +++ b/crates/http/README.md @@ -7,8 +7,5 @@ Web server implementation based on the Rocket engine ``` cd rssto/crates/rssto-http -cargo run -- --mysql-username {USER} \ - --mysql-password {PASS} \ - --mysql-database {NAME} -``` -* optionally, use `--provider-id {ID}` to filter content using post-processing results (e.g. generated by the `rssto-llm` crate) \ No newline at end of file +cargo run -- -c /path/to/config.toml +``` \ No newline at end of file diff --git a/crates/http/config.toml b/crates/http/config.toml new file mode 100644 index 0000000..f4ade36 --- /dev/null +++ b/crates/http/config.toml @@ -0,0 +1,33 @@ +title = "rssto" +#description = "" + +# Replace image sources with local +# * if crawled with the `persist_images_selector` selector +local_images = true + +format_time = "%d/%m/%Y %H:%M" + +# Provider ID (`provider` table) +# * None for the original content +# provider_id = 1 + +# Default listing limit +list_limit = 20 + +# Bind server on given host +host = "127.0.0.1" + +# Bind server on given port +port = 8000 + +#Configure instance in the debug mode +debug = true + +# Database connection setup +# * see crates/mysql/database +[mysql] +host = "localhost" +port = 3306 +username = "" +password = "" +database = "rssto" \ No newline at end of file diff --git a/crates/http/src/argument.rs b/crates/http/src/argument.rs new file mode 100644 index 0000000..3894dd5 --- /dev/null +++ b/crates/http/src/argument.rs @@ -0,0 +1,12 @@ +use clap::Parser; +use std::path::PathBuf; + +#[derive(Parser, Debug)] +#[command(version, about, long_about = None)] +pub struct Argument { + /// Path to config file + /// + /// * see `config.toml` + #[arg(short, long)] + pub config: PathBuf, +} diff --git a/crates/http/src/config.rs b/crates/http/src/config.rs index 9ca317f..a6068dd 100644 --- a/crates/http/src/config.rs +++ b/crates/http/src/config.rs @@ -1,53 +1,24 @@ -use clap::Parser; -use std::net::{IpAddr, Ipv4Addr}; +use serde::Deserialize; +use std::net::IpAddr; -#[derive(Parser, Debug)] -#[command(version, about, long_about = None)] -pub struct Config { - /// Server name - #[arg(long, default_value_t = String::from("rssto"))] - pub title: String, - - /// Server description - #[arg(long)] - pub description: Option, - - /// Format timestamps (on the web view) - /// - /// * tip: escape with `%%d/%%m/%%Y %%H:%%M` in the CLI/bash argument - #[arg(long, default_value_t = String::from("%d/%m/%Y %H:%M"))] - pub format_time: String, - - /// Provider ID (`provider` table) - /// * None for the original content - #[arg(long, short)] - pub provider_id: Option, - - /// Default listing limit - #[arg(long, default_value_t = 20)] - pub list_limit: usize, - - /// Bind server on given host - #[arg(long, default_value_t = IpAddr::V4(Ipv4Addr::LOCALHOST))] - pub host: IpAddr, - - /// Bind server on given port - #[arg(long, default_value_t = 8000)] +#[derive(Debug, Deserialize)] +pub struct Mysql { + pub database: String, + pub host: String, + pub password: String, pub port: u16, - - /// Configure instance in the debug mode - #[arg(long, default_value_t = false)] - pub debug: bool, - - // Database - #[arg(long, default_value_t = String::from("localhost"))] - pub mysql_host: String, - #[arg(long, default_value_t = 3306)] - pub mysql_port: u16, - #[arg(long)] - pub mysql_username: String, - #[arg(long)] - pub mysql_password: String, - #[arg(long)] - pub mysql_database: String, + pub username: String, +} + +#[derive(Debug, Deserialize)] +pub struct Config { + pub mysql: Mysql, + pub title: String, + pub description: Option, + pub format_time: String, + pub provider_id: Option, + pub list_limit: usize, + pub host: IpAddr, + pub port: u16, + pub debug: bool, } diff --git a/crates/http/src/main.rs b/crates/http/src/main.rs index 0803379..df8ecd4 100644 --- a/crates/http/src/main.rs +++ b/crates/http/src/main.rs @@ -1,13 +1,13 @@ #[macro_use] extern crate rocket; +mod argument; mod config; mod feed; mod global; mod meta; use chrono::{DateTime, Utc}; -use config::Config; use feed::Feed; use global::Global; use meta::Meta; @@ -187,7 +187,9 @@ fn rss( #[launch] fn rocket() -> _ { use clap::Parser; - let config = Config::parse(); + let argument = argument::Argument::parse(); + let config: config::Config = + toml::from_str(&std::fs::read_to_string(argument.config).unwrap()).unwrap(); rocket::build() .attach(Template::fairing()) .configure(rocket::Config { @@ -201,11 +203,11 @@ fn rocket() -> _ { }) .manage( Database::pool( - &config.mysql_host, - config.mysql_port, - &config.mysql_username, - &config.mysql_password, - &config.mysql_database, + &config.mysql.host, + config.mysql.port, + &config.mysql.username, + &config.mysql.password, + &config.mysql.database, ) .unwrap(), ) diff --git a/crates/llm/Cargo.toml b/crates/llm/Cargo.toml index 3e371e8..6f62481 100644 --- a/crates/llm/Cargo.toml +++ b/crates/llm/Cargo.toml @@ -16,5 +16,7 @@ clap = { version = "4.5.54", features = ["derive"] } lancor = "0.1.1" log = "0.4.29" mysql = { package = "rssto-mysql", version = "0.1.0", features = ["transaction"], path = "../mysql" } +serde = { version = "1.0.228", features = ["derive"] } tokio = { version = "1.0", features = ["full"] } -tracing-subscriber = { version = "0.3.22", features = ["env-filter"] } \ No newline at end of file +toml = "0.9.10" +tracing-subscriber = { version = "0.3.22", features = ["env-filter"] } diff --git a/crates/llm/README.md b/crates/llm/README.md index ecb64a8..16cb13d 100644 --- a/crates/llm/README.md +++ b/crates/llm/README.md @@ -17,12 +17,6 @@ llama-server -hf ggml-org/gemma-3-1b-it-GGUF ``` cd rssto/crates/rssto-llm -cargo run -- --mysql-username {USER} \ - --mysql-password {PASS} \ - --mysql-database {NAME} \ - --llm-host {HOST} \ - --llm-port {PORT} \ - --llm-model {MODEL} \ - --llm-message {MESSAGE} +cargo run -- -c /path/to/config.toml ``` * see `--help` to display all supported options \ No newline at end of file diff --git a/crates/llm/config.toml b/crates/llm/config.toml new file mode 100644 index 0000000..eb9f926 --- /dev/null +++ b/crates/llm/config.toml @@ -0,0 +1,22 @@ +# Rescan database for new subjects, in seconds +# * process once if not defined +# update = 900 + +# Database connection setup +# * see crates/mysql/database +[mysql] +host = "localhost" +port = 3306 +username = "" +password = "" +database = "rssto" + +# LLM connection setup +[llm] +scheme = "http" +host = "127.0.0.1" +port = 8080 +# Model name +model = "ggml-org/gemma-3-1b-it-GGUF" +# Initial message for the `content` subject (e.g. `translate to...`) +message = "translate to english:" \ No newline at end of file diff --git a/crates/llm/src/argument.rs b/crates/llm/src/argument.rs index a3cc51d..3894dd5 100644 --- a/crates/llm/src/argument.rs +++ b/crates/llm/src/argument.rs @@ -1,37 +1,12 @@ use clap::Parser; +use std::path::PathBuf; #[derive(Parser, Debug)] #[command(version, about, long_about = None)] pub struct Argument { - // LLM - #[arg(long, default_value_t = String::from("http"))] - pub llm_scheme: String, - #[arg(long, default_value_t = String::from("localhost"))] - pub llm_host: String, - #[arg(long, default_value_t = 8080)] - pub llm_port: u16, - - /// Model name (e.g. `INSAIT-Institute/MamayLM-Gemma-2-9B-IT-v0.1-GGUF` or `ggml-org/gemma-3-1b-it-GGUF`) - #[arg(long)] - pub llm_model: String, - - /// Initial message for the `content` subject (e.g. `translate to...`) - #[arg(long)] - pub llm_message: String, - - // Database - #[arg(long, default_value_t = String::from("localhost"))] - pub mysql_host: String, - #[arg(long, default_value_t = 3306)] - pub mysql_port: u16, - #[arg(long)] - pub mysql_username: String, - #[arg(long)] - pub mysql_password: String, - #[arg(long)] - pub mysql_database: String, - /// Loop update in seconds - /// * None to exit on complete - #[arg(long, short)] - pub update: Option, + /// Path to config file + /// + /// * see `config.toml` + #[arg(short, long)] + pub config: PathBuf, } diff --git a/crates/llm/src/config.rs b/crates/llm/src/config.rs new file mode 100644 index 0000000..9655ea6 --- /dev/null +++ b/crates/llm/src/config.rs @@ -0,0 +1,27 @@ +use serde::Deserialize; +use std::net::IpAddr; + +#[derive(Debug, Deserialize)] +pub struct Mysql { + pub database: String, + pub host: IpAddr, + pub password: String, + pub port: u16, + pub username: String, +} + +#[derive(Debug, Deserialize)] +pub struct Llm { + pub scheme: String, + pub host: IpAddr, + pub port: u16, + pub model: String, + pub message: String, +} + +#[derive(Debug, Deserialize)] +pub struct Config { + pub mysql: Mysql, + pub llm: Llm, + pub update: Option, +} diff --git a/crates/llm/src/main.rs b/crates/llm/src/main.rs index 1774e00..188902f 100644 --- a/crates/llm/src/main.rs +++ b/crates/llm/src/main.rs @@ -1,7 +1,7 @@ mod argument; +mod config; use anyhow::Result; -use argument::Argument; use mysql::Database; #[tokio::main] @@ -27,34 +27,36 @@ async fn main() -> Result<()> { .init() } - let arg = Argument::parse(); + let argument = argument::Argument::parse(); + let config: config::Config = toml::from_str(&std::fs::read_to_string(argument.config)?)?; + let llm = LlamaCppClient::new(format!( "{}://{}:{}", - arg.llm_scheme, arg.llm_host, arg.llm_port + config.llm.scheme, config.llm.host, config.llm.port ))?; let db = Database::pool( - &arg.mysql_host, - arg.mysql_port, - &arg.mysql_username, - &arg.mysql_password, - &arg.mysql_database, + &config.mysql.host.to_string(), + config.mysql.port, + &config.mysql.username, + &config.mysql.password, + &config.mysql.database, )?; let provider_id = { let mut conn = db.connection()?; - match conn.provider_id_by_name(&arg.llm_model)? { + match conn.provider_id_by_name(&config.llm.model)? { Some(provider_id) => { debug!( "Use existing DB provider #{} matches model name `{}`", - provider_id, &arg.llm_model + provider_id, &config.llm.model ); provider_id } None => { - let provider_id = conn.insert_provider(&arg.llm_model)?; + let provider_id = conn.insert_provider(&config.llm.model)?; info!( "Provider `{}` not found in database, created new one with ID `{provider_id}`", - &arg.llm_model + &config.llm.model ); provider_id } @@ -71,15 +73,15 @@ async fn main() -> Result<()> { source.content_id ); - let title = - llm.chat_completion(ChatCompletionRequest::new(&arg.llm_model).message( - Message::user(format!("{}\n{}", arg.llm_message, source.title)), + let title = llm + .chat_completion(ChatCompletionRequest::new(&config.llm.model).message( + Message::user(format!("{}\n{}", config.llm.message, source.title)), )) .await?; - let description = - llm.chat_completion(ChatCompletionRequest::new(&arg.llm_model).message( - Message::user(format!("{}\n{}", arg.llm_message, source.description)), + let description = llm + .chat_completion(ChatCompletionRequest::new(&config.llm.model).message( + Message::user(format!("{}\n{}", config.llm.message, source.description)), )) .await?; @@ -97,7 +99,7 @@ async fn main() -> Result<()> { } tx.commit()?; debug!("Queue completed"); - if let Some(update) = arg.update { + if let Some(update) = config.update { debug!("Wait {update} seconds to continue..."); std::thread::sleep(std::time::Duration::from_secs(update)) } else {