implement deleted posts cleanup, extend sync response features

Cargo.toml

@@ -15,3 +15,4 @@ chrono = "^0.4.20"
 clap = { version = "4.5", features = ["derive"] }
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
+walkdir = "2.5"

src/main.rs

@@ -1,9 +1,11 @@
 mod config;
 mod nex;
+mod response;
 mod snac;
 
 use anyhow::Result;
 use nex::Nex;
+use response::Response;
 use snac::Snac;
 
 fn main() -> Result<()> {
@@ -22,26 +24,33 @@ fn main() -> Result<()> {
     let s = Snac::init(c.source, c.user)?;
 
     println!("export begin...");
-    let (mut u, mut t) = sync(&s, &n)?;
+    let mut r = sync(&s, &n)?;
     match c.rotate {
-        Some(r) => loop {
+        Some(t) => loop {
-            println!("queue completed (updated: {u} / total: {t}), await {r} seconds to rotate...");
+            println!(
-            std::thread::sleep(std::time::Duration::from_secs(r));
+                "queue completed:\n\tcreated: {}\n\tupdated: {}\n\tdeleted:\n\t\tfiles: {}\n\t\tdirectories: {}\n\tignored: {}\n\ttotal: {}\nawait {t} seconds to rotate...",
-            (u, t) = sync(&s, &n)?;
+                r.created, r.updated, r.deleted.files, r.deleted.directories, r.ignored, r.total
+            );
+            std::thread::sleep(std::time::Duration::from_secs(t));
+            r = sync(&s, &n)?;
         },
-        None => println!("export completed (updated: {u} / total: {t})."),
+        None => println!(
+            "export completed:\n\tcreated: {}\n\tupdated: {}\n\tdeleted:\n\t\tfiles: {}\n\t\tdirectories: {}\n\tignored: {}\n\ttotal: {}",
+            r.created, r.updated, r.deleted.files, r.deleted.directories, r.ignored, r.total
+        ),
     }
 
     Ok(())
 }
 
-fn sync(snac: &Snac, nex: &Nex) -> Result<(usize, usize)> {
+fn sync(snac: &Snac, nex: &Nex) -> Result<Response> {
-    let mut t = 0; // total
+    use std::collections::HashSet;
-    let mut u = 0; // updated
+    let mut r = Response::default();
     for user in &snac.users {
         println!("\tsync profile for `{}`...", user.name);
+        let mut keep = HashSet::with_capacity(100); // @TODO estimated
         for post in user.public()? {
-            t += 1;
+            r.total += 1;
             // make sure post entry has expected content type
             if !post.is_note() {
                 todo!()
@@ -49,12 +58,12 @@ fn sync(snac: &Snac, nex: &Nex) -> Result<(usize, usize)> {
             // skip non authorized content
             if let Some(content) = post.source_content {
                 println!("\t\tsync post `{}`...", post.id);
-                u += nex.sync(
+                let response = nex.sync(
                     &user.name,
                     content,
                     post.url,
                     post.attachment.map(|a| {
-                        use nex::Source;
+                        use nex::source::Source;
                         let mut attachments = Vec::with_capacity(a.len());
                         for attachment in a {
                             attachments.push((
@@ -81,8 +90,31 @@ fn sync(snac: &Snac, nex: &Nex) -> Result<(usize, usize)> {
                     }),
                     (post.published, post.updated),
                 )?;
+                use nex::response::change::Change;
+                let f = ""; //response.file.to_string_lossy();
+                match response.change {
+                    Change::Created => {
+                        r.created += 1;
+                        println!("\t\t\tcreate new post file `{f}`.")
+                    }
+                    Change::Ignored => {
+                        r.ignored += 1;
+                        println!("\t\t\tpost file `{f}` is up to date.")
+                    }
+                    Change::Updated => {
+                        r.updated += 1;
+                        println!("\t\t\tpost file `{f}` update with new content.")
+                    }
+                }
+                for i in response.keep {
+                    keep.insert(i);
+                }
             }
         }
+        // cleanup removed post entries
+        let d = nex.clean(&user.name, keep)?;
+        r.deleted.directories += d.directories;
+        r.deleted.files += d.files;
     }
-    Ok((u, t))
+    Ok(r)
 }

src/nex.rs

@@ -1,14 +1,18 @@
 mod attachment;
+pub mod response;
+pub mod source;
 
 use anyhow::{Result, bail};
 use attachment::Attachment;
 use chrono::{DateTime, Utc};
-use std::{collections::HashMap, fs, path::PathBuf, str::FromStr};
+use response::{Clean, Sync, change::Change};
-
+use source::Source;
-pub enum Source {
+use std::{
-    Url(String),
+    collections::{HashMap, HashSet},
-    File(PathBuf),
+    fs,
-}
+    path::PathBuf,
+    str::FromStr,
+};
 
 pub struct Nex {
     attachment: Attachment,
@@ -66,19 +70,38 @@ impl Nex {
         attachments: Option<Vec<(String, String, Source)>>,
         tags: Option<Vec<String>>,
         (published, updated): (DateTime<Utc>, Option<DateTime<Utc>>),
-    ) -> Result<usize> {
+    ) -> Result<Sync> {
+        // collect existing entries to ignore on cleanup
+        let mut i = Vec::with_capacity(10); // includes est. attachments len
+
         // prepare destination
-        let mut p = PathBuf::from(self.users.get(name).unwrap());
+        let root = PathBuf::from(self.users.get(name).unwrap());
+        let mut p = PathBuf::from(&root);
+        i.push(root);
+
         p.push(published.format("%Y").to_string());
+        i.push(p.clone());
+
         p.push(published.format("%m").to_string());
+        i.push(p.clone());
+
         p.push(published.format("%d").to_string());
+        i.push(p.clone());
+
         fs::create_dir_all(&p)?;
 
+        // init shared post ID once
+        let id = published.format(&self.filename).to_string();
+
+        // init post filepath
+        let mut f = PathBuf::from(&p);
+        f.push(&id);
+
         // init storage directory for the post state file and attachments
         // * by the current implementation, its name starts with dot
         //   as usually hidden but accessible (in servers like Nexy)
         let mut d = PathBuf::from(&p);
-        d.push(format!(".{}", published.format(&self.filename)));
+        d.push(format!(".{id}"));
         fs::create_dir_all(&d)?;
 
         // create system meta file to track post time updated
@@ -89,10 +112,13 @@ impl Nex {
         {
             fs::remove_dir_all(&d)?;
             fs::create_dir_all(&d)?;
-            fs::write(s, updated.unwrap_or(published).to_string())?
+            fs::write(&s, updated.unwrap_or(published).to_string())?
         } else {
-            println!("\t\t\tpost is up to date.");
+            i.extend([d, f, p, s]); // move all paths processed to cleanup ignore
-            return Ok(0);
+            return Ok(Sync {
+                change: Change::Ignored,
+                keep: i,
+            });
         }
 
         // format content pattern
@@ -112,7 +138,7 @@ impl Nex {
                     .map(|a| {
                         let mut b = Vec::with_capacity(a.len());
                         b.push("\n".to_string());
-                        for (i, (name, media_type, source)) in a.into_iter().enumerate() {
+                        for (n, (name, media_type, source)) in a.into_iter().enumerate() {
                             let mut t = Vec::with_capacity(3);
                             t.push(format!(
                                 "=> {}",
@@ -122,7 +148,7 @@ impl Nex {
                                         let mut to = PathBuf::from(&d);
                                         let f = format!(
                                             "{}{}",
-                                            i + 1,
+                                            n + 1,
                                             from.extension()
                                                 .map(|e| format!(".{}", e.to_string_lossy()))
                                                 .unwrap_or_default(),
@@ -131,6 +157,7 @@ impl Nex {
                                         if !to.exists() {
                                             self.attachment.sync(&from, &to).unwrap()
                                         }
+                                        i.push(to);
                                         format!("{}/{f}", d.file_name().unwrap().to_string_lossy())
                                     }
                                 }
@@ -161,17 +188,46 @@ impl Nex {
                     .unwrap_or_default(),
             );
 
-        // save post file
+        // save post file, set its change status
-        p.push(published.format(&self.filename).to_string());
+        let change = if fs::exists(&f)? {
-        let f = p.to_string_lossy();
+            Change::Updated
-        if fs::exists(&p)? {
-            println!("\t\t\tpost file `{f}` update with new content.")
         } else {
-            println!("\t\t\tcreate new post file `{f}`.")
+            Change::Created
-        }
+        };
-        fs::write(&p, c)?;
+        fs::write(&f, c)?;
 
-        Ok(1)
+        // move all paths processed to cleanup ignore
+        i.extend([d, f, p, s]);
+
+        Ok(Sync { change, keep: i })
+    }
+
+    pub fn clean(&self, name: &str, ignore: HashSet<PathBuf>) -> Result<Clean> {
+        let mut r = Clean::default();
+        for entry in
+            walkdir::WalkDir::new(PathBuf::from(self.users.get(name).unwrap())).follow_links(false)
+        {
+            let e = entry?;
+            let p = e.path();
+            let s = p.to_string_lossy();
+            println!("check `{s}`...");
+            if ignore.contains(p) {
+                continue;
+            }
+            let m = fs::metadata(p)?;
+            if m.is_file() {
+                fs::remove_file(p)?;
+                r.files += 1;
+                println!("\t\t\tdelete file `{s}`");
+            } else if m.is_dir() {
+                fs::remove_dir_all(p)?;
+                r.directories += 1;
+                println!("\t\t\tdelete directory `{s}`");
+            } else {
+                panic!()
+            }
+        }
+        Ok(r)
     }
 
     pub fn is_attachments_disabled(&self) -> bool {

src/nex/response.rs

@@ -0,0 +1,12 @@
+pub mod change;
+
+pub struct Sync {
+    pub change: change::Change,
+    pub keep: Vec<std::path::PathBuf>,
+}
+
+#[derive(Default)]
+pub struct Clean {
+    pub directories: usize,
+    pub files: usize,
+}

src/nex/response/change.rs

@@ -0,0 +1,5 @@
+pub enum Change {
+    Created,
+    Ignored,
+    Updated,
+}

src/nex/source.rs

@@ -0,0 +1,4 @@
+pub enum Source {
+    Url(String),
+    File(std::path::PathBuf),
+}

src/response.rs

@@ -0,0 +1,14 @@
+#[derive(Default)]
+pub struct Deleted {
+    pub files: usize,
+    pub directories: usize,
+}
+
+#[derive(Default)]
+pub struct Response {
+    pub created: usize,
+    pub deleted: Deleted,
+    pub ignored: usize,
+    pub total: usize,
+    pub updated: usize,
+}