feat: count unreadable source files during backup

crates/core/src/archiver.rs

@@ -4,6 +4,7 @@ pub(crate) mod tree;
 pub(crate) mod tree_archiver;
 
 use std::path::{Path, PathBuf};
+use std::sync::atomic::{AtomicU64, Ordering};
 use std::thread::scope;
 
 use jiff::Zoned;
@@ -18,14 +19,20 @@ use crate::{
     },
     backend::{ReadSource, ReadSourceEntry, decrypt::DecryptFullBackend},
     blob::BlobType,
-    error::RusticResult,
+    error::{RusticError, RusticResult},
     index::{
         ReadGlobalIndex,
         indexer::{Indexer, SharedIndexer},
     },
     repofile::{configfile::ConfigFile, snapshotfile::SnapshotFile},
 };
 
+/// Report a source-file error, count it, and continue the backup.
+fn report_source_error(p: &Progress, errors: &AtomicU64, during: &'static str, err: &RusticError) {
+    _ = errors.fetch_add(1, Ordering::Relaxed);
+    p.error(err.context_value("path"), during, &err.display_log());
+}
+
 #[derive(thiserror::Error, Debug, displaydoc::Display)]
 /// Tree stack empty
 pub struct TreeStackEmptyError;
@@ -138,6 +145,8 @@ impl<'a, BE: DecryptFullBackend, I: ReadGlobalIndex> Archiver<'a, BE, I> {
         <R as ReadSource>::Open: Send,
         <R as ReadSource>::Iter: Send,
     {
+        let error_count = AtomicU64::new(0);
+
         scope(|s| -> RusticResult<_> {
             // determine backup size in parallel to running backup
             let src_size_handle = s.spawn(|| {
@@ -153,7 +162,7 @@ impl<'a, BE: DecryptFullBackend, I: ReadGlobalIndex> Archiver<'a, BE, I> {
             // filter out errors and handle as_path
             let iter = src.entries().filter_map(|item| match item {
                 Err(err) => {
-                    warn!("ignoring error: {}", err.display_log());
+                    report_source_error(p, &error_count, "scan", &err);
                     None
                 }
                 Ok(ReadSourceEntry { path, node, open }) => {
@@ -197,7 +206,7 @@ impl<'a, BE: DecryptFullBackend, I: ReadGlobalIndex> Archiver<'a, BE, I> {
             .filter_map(|item| match item {
                 Ok(item) => Some(item),
                 Err(err) => {
-                    warn!("ignoring error: {}", err.display_log());
+                    report_source_error(p, &error_count, "archival", &err);
                     None
                 }
             })
@@ -213,6 +222,7 @@ impl<'a, BE: DecryptFullBackend, I: ReadGlobalIndex> Archiver<'a, BE, I> {
         let stats = self.file_archiver.finalize()?;
         let (id, mut summary) = self.tree_archiver.finalize(self.parent.tree_id())?;
         stats.apply(&mut summary, BlobType::Data);
+        summary.error_count = error_count.load(Ordering::Relaxed);
         self.snap.tree = id;
 
         self.indexer.write().unwrap().finalize()?;

crates/core/src/backend/ignore.rs

@@ -283,6 +283,17 @@ impl ReadSource for LocalSource {
     }
 }
 
+fn ignore_error_path(err: &ignore::Error) -> Option<String> {
+    match err {
+        ignore::Error::WithPath { path, .. } => Some(path.display().to_string()),
+        ignore::Error::WithDepth { err, .. } | ignore::Error::WithLineNumber { err, .. } => {
+            ignore_error_path(err)
+        }
+        ignore::Error::Loop { child, .. } => Some(child.display().to_string()),
+        _ => None,
+    }
+}
+
 // Walk doesn't implement Debug
 #[allow(missing_debug_implementations)]
 pub struct LocalSourceWalker {
@@ -304,23 +315,35 @@ impl Iterator for LocalSourceWalker {
             item => item,
         }
         .map(|e| {
-            self.save_opts
-                .map_entry(e.map_err(|err| {
+            let entry = e.map_err(|err| {
+                let path = ignore_error_path(&err);
+                let rustic_err = if path.is_some() {
                     RusticError::with_source(
-                        ErrorKind::Internal,
-                        "Failed to get next entry from walk iterator.",
+                        ErrorKind::InputOutput,
+                        "Failed to read source path `{path}`.",
                         err,
                     )
-                    .ask_report()
-                })?)
-                .map_err(|err| {
+                } else {
                     RusticError::with_source(
-                        ErrorKind::Internal,
-                        "Failed to map Directory entry to ReadSourceEntry.",
+                        ErrorKind::InputOutput,
+                        "Failed to read source path.",
                         err,
                     )
-                    .ask_report()
-                })
+                };
+                match path {
+                    Some(path) => rustic_err.attach_context("path", path),
+                    None => rustic_err,
+                }
+            })?;
+            let path = entry.path().display().to_string();
+            self.save_opts.map_entry(entry).map_err(|err| {
+                RusticError::with_source(
+                    ErrorKind::InputOutput,
+                    "Failed to map directory entry `{path}` to a backup source entry.",
+                    err,
+                )
+                .attach_context("path", path)
+            })
         })
     }
 }

crates/core/src/error.rs

@@ -369,6 +369,14 @@ impl RusticError {
         Self::with_source(kind, error.to_string(), error)
     }
 
+    /// Get a context value by key, if present.
+    #[must_use]
+    pub fn context_value(&self, key: &str) -> Option<&str> {
+        self.context
+            .iter()
+            .find_map(|(k, v)| (k == key).then_some(v.as_str()))
+    }
+
     /// Returns a String representation for logging purposes.
     ///
     /// This is a more concise version of the error message.

crates/core/src/progress.rs

@@ -1,6 +1,6 @@
 use std::sync::Arc;
 
-use log::info;
+use log::{info, warn};
 
 /// A progress used to indicate/update the status of something which is being processed
 #[derive(Debug, Clone)]
@@ -55,6 +55,17 @@ impl Progress {
     pub fn finish(&self) {
         self.0.finish();
     }
+
+    /// Report an error encountered while processing an item.
+    ///
+    /// # Arguments
+    ///
+    /// * `item` - Usually the path of the problematic file or directory
+    /// * `during` - What was being attempted, e.g. `"scan"` or `"archival"`
+    /// * `message` - Error message
+    pub fn error(&self, item: Option<&str>, during: &str, message: &str) {
+        self.0.error(item, during, message);
+    }
 }
 
 /// Trait to report progress information for any rustic action which supports that.
@@ -87,6 +98,23 @@ pub trait RusticProgress: Send + Sync + 'static + std::fmt::Debug {
 
     /// Finish the progress
     fn finish(&self);
+
+    /// Report an error encountered while processing an item.
+    ///
+    /// The default implementation logs a warning. JSON progress implementations
+    /// emit a restic-compatible error message instead.
+    ///
+    /// # Arguments
+    ///
+    /// * `item` - Usually the path of the problematic file or directory
+    /// * `during` - What was being attempted, e.g. `"scan"` or `"archival"`
+    /// * `message` - Error message
+    fn error(&self, item: Option<&str>, during: &str, message: &str) {
+        match item {
+            Some(item) => warn!("error: {during} {item}: {message}"),
+            None => warn!("{during}: {message}"),
+        }
+    }
 }
 
 /// Type of progress

crates/core/src/repofile/snapshotfile.rs

@@ -247,6 +247,18 @@ pub struct SnapshotSummary {
 
     /// Total duration that the rustic command ran in seconds
     pub total_duration: f64,
+
+    /// Number of source files/directories that could not be read during this backup run.
+    ///
+    /// Serialized only when non-zero so existing restic/rustic snapshots stay unchanged.
+    #[serde(default, skip_serializing_if = "u64_is_zero")]
+    pub error_count: u64,
+}
+
+// serde `skip_serializing_if` always passes a reference.
+#[allow(clippy::trivially_copy_pass_by_ref)]
+const fn u64_is_zero(value: &u64) -> bool {
+    *value == 0
 }
 
 impl Default for SnapshotSummary {
@@ -275,6 +287,7 @@ impl Default for SnapshotSummary {
             backup_end: Zoned::now(),
             backup_duration: Default::default(),
             total_duration: Default::default(),
+            error_count: Default::default(),
         }
     }
 }
@@ -1610,4 +1623,24 @@ mod tests {
         let ids: Vec<_> = snaps.iter().map(|sn| *sn.id).collect();
         assert_eq!(ids, vec![id3, id1, id3]);
     }
+
+    #[test]
+    fn error_count_omitted_from_json_when_zero() {
+        let summary = SnapshotSummary::default();
+        let json = serde_json::to_value(&summary).unwrap();
+        assert!(json.get("error_count").is_none());
+    }
+
+    #[test]
+    fn error_count_serialized_when_nonzero() {
+        let summary = SnapshotSummary {
+            error_count: 3,
+            ..Default::default()
+        };
+        let json = serde_json::to_value(&summary).unwrap();
+        assert_eq!(json["error_count"], 3);
+
+        let decoded: SnapshotSummary = serde_json::from_value(json).unwrap();
+        assert_eq!(decoded.error_count, 3);
+    }
 }

crates/core/tests/integration/backup.rs

@@ -307,3 +307,45 @@ fn test_backup_excludes_xattr_entries(set_up_repo: Result<RepoOpen>) -> Result<(
 
     Ok(())
 }
+
+#[cfg(unix)]
+#[rstest]
+fn test_backup_unreadable_file_sets_error_count(set_up_repo: Result<RepoOpen>) -> Result<()> {
+    use std::fs::{self, File, Permissions};
+    use std::os::unix::fs::PermissionsExt;
+
+    let tmp = tempfile::tempdir()?;
+    let base = tmp.path();
+    fs::write(base.join("ok.txt"), "ok")?;
+    let secret = base.join("secret.txt");
+    fs::write(&secret, "secret")?;
+    fs::set_permissions(&secret, Permissions::from_mode(0o000))?;
+
+    if File::open(&secret).is_ok() {
+        // e.g. running as root; mode bits are not enforced
+        return Ok(());
+    }
+
+    let repo = set_up_repo?.to_indexed_ids()?;
+    let paths = PathList::from_iter(Some(base.to_path_buf()));
+    let opts = BackupOptions::default().as_path(PathBuf::from("test"));
+    let snapshot = repo.backup(&opts, &paths, SnapshotFile::default())?;
+
+    let summary = snapshot.summary.as_ref().expect("backup sets a summary");
+    assert!(
+        summary.error_count >= 1,
+        "expected at least one source-file error, got {}",
+        summary.error_count
+    );
+    assert_ne!(snapshot.id, SnapshotFile::default().id);
+    assert!(summary.total_files_processed >= 1);
+
+    let loaded = repo.get_snapshot_from_str(&snapshot.id.to_string(), |_| true)?;
+    assert_eq!(
+        loaded.summary.as_ref().map(|s| s.error_count),
+        Some(summary.error_count),
+        "error_count should be persisted in the snapshot file when non-zero"
+    );
+
+    Ok(())
+}

crates/core/tests/integration/snapshots/integration__integration__chunker__chunker-fixedsize.snap

@@ -38,6 +38,7 @@ SnapshotFile(
     backup_end: "[backup_end]",
     backup_duration: "[backup_duration]",
     total_duration: "[total_duration]",
+    error_count: 0,
   )),
   id: "[id]",
 )

crates/core/tests/snapshots/integration__backup-tar-groups-nix.snap

@@ -45,6 +45,7 @@ expression: snap
           backup_end: "[backup_end]",
           backup_duration: "[backup_duration]",
           total_duration: "[total_duration]",
+          error_count: 0,
         )),
         id: "[id]",
       ),
@@ -87,6 +88,7 @@ expression: snap
           backup_end: "[backup_end]",
           backup_duration: "[backup_duration]",
           total_duration: "[total_duration]",
+          error_count: 0,
         )),
         id: "[id]",
       ),
@@ -142,6 +144,7 @@ expression: snap
           backup_end: "[backup_end]",
           backup_duration: "[backup_duration]",
           total_duration: "[total_duration]",
+          error_count: 0,
         )),
         id: "[id]",
       ),

crates/core/tests/snapshots/integration__backup-tar-groups-windows.snap

@@ -45,6 +45,7 @@ expression: snap
           backup_end: "[backup_end]",
           backup_duration: "[backup_duration]",
           total_duration: "[total_duration]",
+          error_count: 0,
         )),
         id: "[id]",
       ),
@@ -87,6 +88,7 @@ expression: snap
           backup_end: "[backup_end]",
           backup_duration: "[backup_duration]",
           total_duration: "[total_duration]",
+          error_count: 0,
         )),
         id: "[id]",
       ),
@@ -142,6 +144,7 @@ expression: snap
           backup_end: "[backup_end]",
           backup_duration: "[backup_duration]",
           total_duration: "[total_duration]",
+          error_count: 0,
         )),
         id: "[id]",
       ),

crates/core/tests/snapshots/integration__backup-tar-matching-snaps-nix.snap

@@ -45,6 +45,7 @@ expression: snap
       backup_end: "[backup_end]",
       backup_duration: "[backup_duration]",
       total_duration: "[total_duration]",
+      error_count: 0,
     )),
     id: "[id]",
   ),

crates/core/tests/snapshots/integration__backup-tar-matching-snaps-windows.snap

@@ -45,6 +45,7 @@ expression: snap
       backup_end: "[backup_end]",
       backup_duration: "[backup_duration]",
       total_duration: "[total_duration]",
+      error_count: 0,
     )),
     id: "[id]",
   ),

crates/core/tests/snapshots/integration__backup-tar-summary-first-nix.snap

@@ -38,6 +38,7 @@ SnapshotFile(
     backup_end: "[backup_end]",
     backup_duration: "[backup_duration]",
     total_duration: "[total_duration]",
+    error_count: 0,
   )),
   id: "[id]",
 )

crates/core/tests/snapshots/integration__backup-tar-summary-first-windows.snap

@@ -38,6 +38,7 @@ SnapshotFile(
     backup_end: "[backup_end]",
     backup_duration: "[backup_duration]",
     total_duration: "[total_duration]",
+    error_count: 0,
   )),
   id: "[id]",
 )

crates/core/tests/snapshots/integration__backup-tar-summary-second-nix.snap

@@ -40,6 +40,7 @@ SnapshotFile(
     backup_end: "[backup_end]",
     backup_duration: "[backup_duration]",
     total_duration: "[total_duration]",
+    error_count: 0,
   )),
   id: "[id]",
 )

crates/core/tests/snapshots/integration__backup-tar-summary-second-windows.snap

@@ -40,6 +40,7 @@ SnapshotFile(
     backup_end: "[backup_end]",
     backup_duration: "[backup_duration]",
     total_duration: "[total_duration]",
+    error_count: 0,
   )),
   id: "[id]",
 )