fix: Reduce amount of rescans in giant $HOME like folders

Makefile

@@ -14,7 +14,7 @@ SHELL := bash
 # string rather than the literal `-o` / `pipefail` tokens.
 .SHELLFLAGS := -o pipefail -euc
 
-.PHONY: build build-c-lib install uninstall test test-rust test-c-smoke test-c-api test-lua test-lua-snap test-version test-bun test-node prepare-bun prepare-bun-packaged prepare-node set-npm-version header test-stress test-stress-seeded test-stress-random test-stress-regressions test-stress-repos test-node-stress sync-js-api sync-js-api-check bump-homebrew-formula bump-install-mcp-sh test-bun-compile
+.PHONY: build build-c-lib install uninstall test test-rust test-rescan test-rescan-known-defects rescan-probe test-c-smoke test-c-api test-lua test-lua-snap test-version test-bun test-node prepare-bun prepare-bun-packaged prepare-node set-npm-version header test-stress test-stress-seeded test-stress-random test-stress-regressions test-stress-repos test-node-stress sync-js-api sync-js-api-check bump-homebrew-formula bump-install-mcp-sh test-bun-compile
 
 all: format test lint
 
@@ -92,6 +92,25 @@ test-setup:
 test-rust:
 	cargo test --workspace --no-default-features --features zlob --exclude fff-nvim
 
+# Watcher rescan harness: asserts that editing, build output, git activity and
+# preview reads all stay on the incremental path instead of re-walking the tree.
+test-rescan:
+	cargo test -p fff-search --no-default-features --features zlob \
+		--lib --test rescan_regression -- rescan
+
+# Live probe for watcher rescan requests and their causes.
+# Usage: make rescan-probe DIR=~/some/repo [SECONDS=120]
+rescan-probe:
+	cargo run --release -p fff-nvim --bin rescan_probe \
+		--no-default-features --features zlob,rescan-stats -- \
+		$(or $(DIR),.) $(if $(SECONDS),--seconds $(SECONDS),)
+
+# The same harness, restricted to cases that currently fail on purpose. Each
+# `#[ignore]` reason names the code that causes the unnecessary rescan.
+test-rescan-known-defects:
+	cargo test --no-fail-fast -p fff-search --no-default-features --features zlob \
+		--lib --test rescan_regression -- --ignored --nocapture
+
 CC ?= cc
 CFLAGS ?= -O0 -g -Wall -Wextra -std=c99
 TARGET_DIR ?= target/release

crates/fff-core/Cargo.toml

@@ -41,6 +41,9 @@ harness = false
 default = ["ripgrep"]
 # Enable C FFI exports
 ffi = []
+# Count full rescans and their causes. Always on in debug builds; enable this
+# to keep the accounting in a release build (used by the rescan_probe binary).
+rescan-stats = []
 # Enables POC definition classification for grep result matched lines
 definitions = []
 # Pure-Rust filesystem walker + glob matcher (ignore + globset crates).

crates/fff-core/build.rs

@@ -3,6 +3,15 @@ fn main() {
     // used by tests/fuzz_git_watcher_stress.rs
     println!("cargo::rustc-check-cfg=cfg(stress)");
 
+    // Full-rescan accounting. Debug builds get it for free; a release build has
+    // to opt in with `--features rescan-stats` (what the rescan_probe needs).
+    println!("cargo::rustc-check-cfg=cfg(rescan_stats)");
+    if std::env::var("DEBUG").is_ok_and(|debug| debug != "false")
+        || std::env::var("CARGO_FEATURE_RESCAN_STATS").is_ok()
+    {
+        println!("cargo::rustc-cfg=rescan_stats");
+    }
+
     // When the `zlob` feature is enabled (Zig-compiled C library):
     // On Windows MSVC, explicitly link the C runtime libraries.
     // Zig-compiled static libraries don't emit /DEFAULTLIB directives for the

crates/fff-core/src/constants.rs

@@ -12,16 +12,25 @@ pub const MAX_INDEXABLE_FILE_SIZE: usize = 2 * 1024 * 1024;
 pub const MAX_CACHED_CONTENT_BYTES: u64 = 512 * 1024 * 1024;
 
 /// Files below one page waste the remainder when mmapped, so the cache skips
-/// them and falls back to chunked reads. Unused on Windows (no content cache).
+/// them and falls back to chunked reads. Unused on Windows (no content cache)
 #[cfg(all(not(target_os = "windows"), target_arch = "aarch64"))]
 pub const MMAP_THRESHOLD: u64 = 16 * 1024;
 #[cfg(all(not(target_os = "windows"), not(target_arch = "aarch64")))]
 pub const MMAP_THRESHOLD: u64 = 4 * 1024;
 
-/// Capacity reserved for files the watcher discovers after the initial scan;
-/// exceeding it forces a full rescan.
+/// Watcher overflow capacity reserved after the initial scan
 pub const MAX_OVERFLOW_FILES: usize = 1024;
 
+/// Minimum delay between watcher-initiated rescans.
+pub const RESCAN_MIN_INTERVAL: std::time::Duration = std::time::Duration::from_secs(30);
+
+/// Rescan delay for large indexes.
+pub const RESCAN_MIN_INTERVAL_LARGE_INDEX: std::time::Duration =
+    std::time::Duration::from_secs(5 * 60);
+
+/// Live-file count at which [`RESCAN_MIN_INTERVAL_LARGE_INDEX`] takes over.
+pub const LARGE_INDEX_FILE_COUNT: usize = 1_000_000;
+
 /// Fresh-mmap threshold: files at or above this size get mmapped directly on
 /// cache miss instead of chunked reads into Vec. Empirically tuned per-platform.
 /// Only referenced on Unix; Windows uses the `std::fs::read` fallback so this

crates/fff-core/src/file_picker.rs

@@ -629,6 +629,10 @@ impl FilePicker {
         &self.base_path
     }
 
+    pub fn has_git_repo(&self) -> bool {
+        self.sync_data.git_workdir.is_some()
+    }
+
     /// Ignore rules the walker assembled during the last scan (zlob backend
     /// only). The background watcher uses these to filter events without
     /// libgit2. `None` when the backend doesn't surface rules or no ignore

crates/fff-core/src/ignore.rs

@@ -3,28 +3,53 @@ use std::path::Path;
 /// Directories excluded when walking a non-git root. Entries are `cfg`-gated
 /// so a single iteration covers standard + platform-specific overrides.
 pub(crate) const IGNORED_DIRS: &[&str] = &[
+    // various dev tools that can be meet in the developer app
     "node_modules",
     "__pycache__",
     "venv",
     ".venv",
-    // Rust (glob-only patterns for non_git_repo_overrides; is_non_code_directory
-    // matches the "target" component separately).
     "target/debug",
     "target/release",
     "target/rust-analyzer",
     "target/criterion",
+    // Language package caches in non-git roots.
+    "go/pkg/mod",
+    ".cargo/registry",
+    ".rustup/toolchains",
+    ".gradle/caches",
+    ".m2/repository",
+    ".npm/_cacache",
+    ".pub-cache",
+    #[cfg(not(target_os = "windows"))]
+    ".local/state", // this contains tons of logs which generate too much watcher noise
     #[cfg(target_os = "macos")]
     "Library/Application Support",
     #[cfg(target_os = "macos")]
     "Library/Caches",
-    // App-group sandbox storage — used by iMessage, Photos, Notes, Calendar,
-    // Electron apps, etc. for SQLite-WAL, LevelDB, protobuf files. These are
-    // almost entirely extension-less binary files (~80k on a typical $HOME)
-    // that never need to appear in a fuzzy or grep search.
     #[cfg(target_os = "macos")]
-    "Library/Group Containers",
+    "Library/Containers", // sandboxed apps data
     #[cfg(target_os = "macos")]
-    "Library/Containers",
+    "Library/Group Containers", // random application data and networking
+    #[cfg(target_os = "macos")]
+    "Library/pnpm",
+    #[cfg(target_os = "macos")]
+    "Library/Metadata",
+    #[cfg(target_os = "macos")]
+    "Library/Developer/CoreSimulator",
+    #[cfg(target_os = "macos")]
+    "Library/Android",
+    #[cfg(target_os = "macos")]
+    "Library/Logs",
+    #[cfg(target_os = "macos")]
+    "Library/Daemon Containers",
+    #[cfg(target_os = "macos")]
+    "Library/Trial",
+    #[cfg(target_os = "macos")]
+    "Library/Preferences",
+    #[cfg(target_os = "macos")]
+    "Library/Messages",
+    #[cfg(target_os = "macos")]
+    "Library/IdentityServices",
     #[cfg(target_os = "windows")]
     "bin/Debug",
     #[cfg(target_os = "windows")]
@@ -57,6 +82,10 @@ pub(crate) fn non_git_repo_overrides(base_path: &Path) -> Option<ignore::overrid
 pub(crate) fn is_non_code_directory(path: &Path) -> bool {
     let path_str = path.as_os_str().to_str().unwrap_or("");
     IGNORED_DIRS.iter().any(|&dir| {
+        // Entries are gitignore patterns for the walkers; here they are matched
+        // as substrings, so a leading `*` wildcard has to come off first.
+        let dir = dir.strip_prefix('*').unwrap_or(dir);
+
         #[cfg(target_os = "windows")]
         let dir = dir.replace('/', std::path::MAIN_SEPARATOR_STR);
         #[cfg(target_os = "windows")]
@@ -66,3 +95,42 @@ pub(crate) fn is_non_code_directory(path: &Path) -> bool {
         path_str.contains(dir)
     })
 }
+
+#[cfg(all(test, target_os = "macos"))]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn home_machine_state_is_excluded_but_source_trees_are_not() {
+        // Representative machine state from a home index.
+        for rel in [
+            "Library/pnpm/store/v3/files/00/abcdef",
+            "Library/Preferences/com.apple.finder.plist",
+            "Library/Messages/prewarm.db-shm",
+            "Library/IdentityServices/TetraDB-identityservicesd.db-wal",
+            "Library/Developer/CoreSimulator/Devices/X/data/f",
+            "go/pkg/mod/github.com/x/y@v1/main.go",
+            ".cargo/registry/src/index.crates.io-1/serde-1.0/src/lib.rs",
+            "Library/Android/sdk/platforms/android-34/data/x",
+            ".local/state/nvim/fff+123+456.log",
+        ] {
+            assert!(
+                is_non_code_directory(Path::new(rel)),
+                "{rel} must not reach the index"
+            );
+        }
+
+        // Source trees under $HOME stay searchable.
+        for rel in [
+            "dev/chromium/third_party/blink/renderer/core/dom/node.cc",
+            "dev/fff.nvim/crates/fff-core/src/lib.rs",
+            "Documents/notes/todo.md",
+            "dev/myproj/pkg/mod/thing.go",
+        ] {
+            assert!(
+                !is_non_code_directory(Path::new(rel)),
+                "{rel} must stay searchable"
+            );
+        }
+    }
+}

crates/fff-core/src/lib.rs

@@ -135,6 +135,12 @@ pub use types::*;
 
 pub mod constants;
 
+/// Watcher rescan request accounting.
+pub mod rescan_stats;
+pub use rescan_stats::{RESCAN_STATS_ENABLED, RescanReason, RescanStats};
+
+mod rescan_throttle;
+
 // ==================================
 // these are public only for benchmarks, no backward compatibility guaranteed
 #[doc(hidden)]